├── CHANGES.txt ├── LICENSE ├── NOTICE ├── README ├── bin ├── find-new-patch-available-jiras ├── raw-check-patch ├── test-patch ├── test-patch-00-clean ├── test-patch-05-patch-raw-analysis ├── test-patch-08-rat ├── test-patch-09-javadoc ├── test-patch-10-compile ├── test-patch-11-findbugs ├── test-patch-20-tests ├── test-patch-30-dist └── update-master-docs ├── bookkeeper-benchmark ├── bin │ └── benchmark ├── conf │ └── log4j.properties ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── bookkeeper │ │ └── benchmark │ │ ├── BenchBookie.java │ │ ├── BenchReadThroughputLatency.java │ │ ├── BenchThroughputLatency.java │ │ ├── MySqlClient.java │ │ └── TestClient.java │ └── test │ ├── java │ └── org │ │ └── apache │ │ └── bookkeeper │ │ └── benchmark │ │ └── TestBenchmark.java │ └── resources │ └── log4j.properties ├── bookkeeper-server ├── bin │ ├── bookkeeper │ ├── bookkeeper-cluster.sh │ └── bookkeeper-daemon.sh ├── conf │ ├── bk_server.conf │ ├── bkenv.sh │ ├── log4j.properties │ └── log4j.shell.properties ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── apache │ │ │ └── bookkeeper │ │ │ ├── auth │ │ │ ├── AuthProviderFactoryFactory.java │ │ │ ├── BookieAuthProvider.java │ │ │ └── ClientAuthProvider.java │ │ │ ├── bookie │ │ │ ├── BookKeeperServerStats.java │ │ │ ├── Bookie.java │ │ │ ├── BookieBean.java │ │ │ ├── BookieCriticalThread.java │ │ │ ├── BookieException.java │ │ │ ├── BookieMXBean.java │ │ │ ├── BookieShell.java │ │ │ ├── BookieThread.java │ │ │ ├── BufferedChannel.java │ │ │ ├── BufferedChannelBase.java │ │ │ ├── BufferedReadChannel.java │ │ │ ├── CacheCallback.java │ │ │ ├── CheckpointSource.java │ │ │ ├── CompactableLedgerStorage.java │ │ │ ├── Cookie.java │ │ │ ├── EntryKey.java │ │ │ ├── EntryKeyValue.java │ │ │ ├── EntryLocation.java │ │ │ ├── EntryLogMetadata.java │ │ │ ├── EntryLogger.java │ │ │ ├── EntryMemTable.java │ │ │ ├── ExitCode.java │ │ │ ├── FileInfo.java │ │ │ ├── FileSystemUpgrade.java │ │ │ ├── GarbageCollector.java │ │ │ ├── GarbageCollectorThread.java │ │ │ ├── HandleFactory.java │ │ │ ├── HandleFactoryImpl.java │ │ │ ├── IndexInMemPageMgr.java │ │ │ ├── IndexPersistenceMgr.java │ │ │ ├── InterleavedLedgerStorage.java │ │ │ ├── Journal.java │ │ │ ├── JournalChannel.java │ │ │ ├── LEPStateChangeCallback.java │ │ │ ├── LedgerCache.java │ │ │ ├── LedgerCacheBean.java │ │ │ ├── LedgerCacheImpl.java │ │ │ ├── LedgerCacheMXBean.java │ │ │ ├── LedgerDescriptor.java │ │ │ ├── LedgerDescriptorImpl.java │ │ │ ├── LedgerDescriptorReadOnlyImpl.java │ │ │ ├── LedgerDirsManager.java │ │ │ ├── LedgerEntryPage.java │ │ │ ├── LedgerStorage.java │ │ │ ├── LedgerStorageFactory.java │ │ │ ├── LocalBookieEnsemblePlacementPolicy.java │ │ │ ├── LogMark.java │ │ │ ├── MarkerFileChannel.java │ │ │ ├── ReadOnlyBookie.java │ │ │ ├── ReadOnlyEntryLogger.java │ │ │ ├── ReadOnlyFileInfo.java │ │ │ ├── ScanAndCompareGarbageCollector.java │ │ │ ├── SkipListArena.java │ │ │ ├── SkipListFlusher.java │ │ │ ├── SortedLedgerStorage.java │ │ │ └── SyncThread.java │ │ │ ├── client │ │ │ ├── AsyncCallback.java │ │ │ ├── BKException.java │ │ │ ├── BookKeeper.java │ │ │ ├── BookKeeperAdmin.java │ │ │ ├── BookKeeperClientStats.java │ │ │ ├── BookieWatcher.java │ │ │ ├── BookiesListener.java │ │ │ ├── CRC32DigestManager.java │ │ │ ├── DefaultEnsemblePlacementPolicy.java │ │ │ ├── DigestManager.java │ │ │ ├── DistributionSchedule.java │ │ │ ├── EnsemblePlacementPolicy.java │ │ │ ├── LedgerChecker.java │ │ │ ├── LedgerCreateOp.java │ │ │ ├── LedgerDeleteOp.java │ │ │ ├── LedgerEntry.java │ │ │ ├── LedgerFragment.java │ │ │ ├── LedgerFragmentReplicator.java │ │ │ ├── LedgerHandle.java │ │ │ ├── LedgerHandleAdv.java │ │ │ ├── LedgerMetadata.java │ │ │ ├── LedgerOpenOp.java │ │ │ ├── LedgerRecoveryOp.java │ │ │ ├── MacDigestManager.java │ │ │ ├── PendingAddOp.java │ │ │ ├── PendingReadOp.java │ │ │ ├── RackawareEnsemblePlacementPolicy.java │ │ │ ├── ReadLastConfirmedOp.java │ │ │ ├── ReadOnlyLedgerHandle.java │ │ │ ├── RoundRobinDistributionSchedule.java │ │ │ ├── SyncCounter.java │ │ │ ├── TryReadLastConfirmedOp.java │ │ │ └── UpdateLedgerOp.java │ │ │ ├── conf │ │ │ ├── AbstractConfiguration.java │ │ │ ├── ClientConfiguration.java │ │ │ ├── Configurable.java │ │ │ └── ServerConfiguration.java │ │ │ ├── jmx │ │ │ ├── BKMBeanInfo.java │ │ │ └── BKMBeanRegistry.java │ │ │ ├── meta │ │ │ ├── AbstractZkLedgerManager.java │ │ │ ├── CleanupLedgerManager.java │ │ │ ├── FlatLedgerManager.java │ │ │ ├── FlatLedgerManagerFactory.java │ │ │ ├── HierarchicalLedgerManager.java │ │ │ ├── HierarchicalLedgerManagerFactory.java │ │ │ ├── LedgerIdGenerator.java │ │ │ ├── LedgerLayout.java │ │ │ ├── LedgerManager.java │ │ │ ├── LedgerManagerFactory.java │ │ │ ├── LedgerUnderreplicationManager.java │ │ │ ├── MSLedgerManagerFactory.java │ │ │ ├── ZkLedgerIdGenerator.java │ │ │ ├── ZkLedgerUnderreplicationManager.java │ │ │ └── ZkVersion.java │ │ │ ├── metastore │ │ │ ├── InMemoryMetaStore.java │ │ │ ├── InMemoryMetastoreCursor.java │ │ │ ├── InMemoryMetastoreTable.java │ │ │ ├── MSException.java │ │ │ ├── MSWatchedEvent.java │ │ │ ├── MetaStore.java │ │ │ ├── MetastoreCallback.java │ │ │ ├── MetastoreCursor.java │ │ │ ├── MetastoreException.java │ │ │ ├── MetastoreFactory.java │ │ │ ├── MetastoreScannableTable.java │ │ │ ├── MetastoreTable.java │ │ │ ├── MetastoreTableItem.java │ │ │ ├── MetastoreUtils.java │ │ │ ├── MetastoreWatcher.java │ │ │ └── Value.java │ │ │ ├── net │ │ │ ├── AbstractDNSToSwitchMapping.java │ │ │ ├── BookieSocketAddress.java │ │ │ ├── CachedDNSToSwitchMapping.java │ │ │ ├── CommonConfigurationKeys.java │ │ │ ├── DNS.java │ │ │ ├── DNSToSwitchMapping.java │ │ │ ├── NetUtils.java │ │ │ ├── NetworkTopology.java │ │ │ ├── Node.java │ │ │ ├── NodeBase.java │ │ │ └── ScriptBasedMapping.java │ │ │ ├── processor │ │ │ └── RequestProcessor.java │ │ │ ├── proto │ │ │ ├── AuthHandler.java │ │ │ ├── BKStats.java │ │ │ ├── BookieClient.java │ │ │ ├── BookieNettyServer.java │ │ │ ├── BookieProtoEncoding.java │ │ │ ├── BookieProtocol.java │ │ │ ├── BookieRequestHandler.java │ │ │ ├── BookieRequestProcessor.java │ │ │ ├── BookieServer.java │ │ │ ├── BookieServerBean.java │ │ │ ├── BookieServerMXBean.java │ │ │ ├── BookkeeperInternalCallbacks.java │ │ │ ├── BookkeeperProtocol.java │ │ │ ├── DataFormats.java │ │ │ ├── DefaultPerChannelBookieClientPool.java │ │ │ ├── LocalBookiesRegistry.java │ │ │ ├── PacketProcessorBase.java │ │ │ ├── PacketProcessorBaseV3.java │ │ │ ├── PerChannelBookieClient.java │ │ │ ├── PerChannelBookieClientFactory.java │ │ │ ├── PerChannelBookieClientPool.java │ │ │ ├── ReadEntryProcessor.java │ │ │ ├── ReadEntryProcessorV3.java │ │ │ ├── ResponseBuilder.java │ │ │ ├── ServerStats.java │ │ │ ├── WriteEntryProcessor.java │ │ │ └── WriteEntryProcessorV3.java │ │ │ ├── replication │ │ │ ├── Auditor.java │ │ │ ├── AuditorElector.java │ │ │ ├── AutoRecoveryMain.java │ │ │ ├── BookieLedgerIndexer.java │ │ │ ├── ReplicationEnableCb.java │ │ │ ├── ReplicationException.java │ │ │ ├── ReplicationStats.java │ │ │ └── ReplicationWorker.java │ │ │ ├── streaming │ │ │ ├── LedgerInputStream.java │ │ │ └── LedgerOutputStream.java │ │ │ ├── tools │ │ │ └── BookKeeperTools.java │ │ │ ├── util │ │ │ ├── BookKeeperConstants.java │ │ │ ├── DaemonThreadFactory.java │ │ │ ├── DirectMemoryUtils.java │ │ │ ├── DiskChecker.java │ │ │ ├── EntryFormatter.java │ │ │ ├── HardLink.java │ │ │ ├── IOUtils.java │ │ │ ├── LocalBookKeeper.java │ │ │ ├── Main.java │ │ │ ├── MathUtils.java │ │ │ ├── NativeIO.java │ │ │ ├── OrderedSafeExecutor.java │ │ │ ├── ReflectionUtils.java │ │ │ ├── SafeRunnable.java │ │ │ ├── Shell.java │ │ │ ├── SnapshotMap.java │ │ │ ├── StringEntryFormatter.java │ │ │ ├── StringUtils.java │ │ │ ├── Tool.java │ │ │ ├── ZeroBuffer.java │ │ │ └── ZkUtils.java │ │ │ ├── versioning │ │ │ ├── Version.java │ │ │ └── Versioned.java │ │ │ └── zookeeper │ │ │ ├── BoundExponentialBackoffRetryPolicy.java │ │ │ ├── ExponentialBackoffRetryPolicy.java │ │ │ ├── RetryPolicy.java │ │ │ ├── ZooKeeperClient.java │ │ │ ├── ZooKeeperWatcherBase.java │ │ │ └── ZooWorker.java │ ├── proto │ │ ├── BookkeeperProtocol.proto │ │ └── DataFormats.proto │ └── resources │ │ ├── LICENSE.bin.txt │ │ ├── NOTICE.bin.txt │ │ └── findbugsExclude.xml │ └── test │ ├── java │ └── org │ │ └── apache │ │ └── bookkeeper │ │ ├── auth │ │ └── TestAuth.java │ │ ├── bookie │ │ ├── BookieAccessor.java │ │ ├── BookieInitializationTest.java │ │ ├── BookieJournalTest.java │ │ ├── BookieShutdownTest.java │ │ ├── BookieThreadTest.java │ │ ├── CompactionTest.java │ │ ├── CookieTest.java │ │ ├── CreateNewLogTest.java │ │ ├── EntryLogTest.java │ │ ├── IndexCorruptionTest.java │ │ ├── IndexPersistenceMgrTest.java │ │ ├── LedgerCacheTest.java │ │ ├── TestGcOverreplicatedLedger.java │ │ ├── TestLedgerDirsManager.java │ │ ├── TestSyncThread.java │ │ ├── UpdateCookieCmdTest.java │ │ └── UpgradeTest.java │ │ ├── client │ │ ├── BookKeeperClientZKSessionExpiry.java │ │ ├── BookKeeperCloseTest.java │ │ ├── BookKeeperTest.java │ │ ├── BookKeeperTestClient.java │ │ ├── BookieRecoveryTest.java │ │ ├── BookieWriteLedgerTest.java │ │ ├── ClientUtil.java │ │ ├── LedgerCloseTest.java │ │ ├── LedgerHandleAdapter.java │ │ ├── LedgerRecoveryTest.java │ │ ├── ListLedgersTest.java │ │ ├── LocalBookKeeperTest.java │ │ ├── RoundRobinDistributionScheduleTest.java │ │ ├── SlowBookieTest.java │ │ ├── TestAddEntryQuorumTimeout.java │ │ ├── TestBookieHealthCheck.java │ │ ├── TestBookieWatcher.java │ │ ├── TestFencing.java │ │ ├── TestLedgerChecker.java │ │ ├── TestLedgerFragmentReplication.java │ │ ├── TestRackawareEnsemblePlacementPolicy.java │ │ ├── TestReadTimeout.java │ │ ├── TestSpeculativeRead.java │ │ ├── TestTryReadLastConfirmed.java │ │ ├── TestWatchEnsembleChange.java │ │ ├── UpdateLedgerCmdTest.java │ │ └── UpdateLedgerOpTest.java │ │ ├── conf │ │ └── TestBKConfiguration.java │ │ ├── meta │ │ ├── GcLedgersTest.java │ │ ├── LedgerLayoutTest.java │ │ ├── LedgerManagerIteratorTest.java │ │ ├── LedgerManagerTestCase.java │ │ ├── TestLedgerManager.java │ │ ├── TestZkLedgerIdGenerator.java │ │ └── TestZkVersion.java │ │ ├── metastore │ │ ├── MetastoreScannableTableAsyncToSyncConverter.java │ │ ├── MetastoreTableAsyncToSyncConverter.java │ │ └── TestMetaStore.java │ │ ├── proto │ │ ├── TestBKStats.java │ │ ├── TestBackwardCompatCMS42.java │ │ ├── TestDataFormats.java │ │ ├── TestDeathwatcher.java │ │ └── TestPerChannelBookieClient.java │ │ ├── replication │ │ ├── AuditorBookieTest.java │ │ ├── AuditorLedgerCheckerTest.java │ │ ├── AuditorPeriodicBookieCheckTest.java │ │ ├── AuditorPeriodicCheckTest.java │ │ ├── AuditorRollingRestartTest.java │ │ ├── AutoRecoveryMainTest.java │ │ ├── BookieAutoRecoveryTest.java │ │ ├── BookieLedgerIndexTest.java │ │ ├── ReplicationTestUtil.java │ │ ├── TestAutoRecoveryAlongWithBookieServers.java │ │ ├── TestLedgerUnderreplicationManager.java │ │ └── TestReplicationWorker.java │ │ ├── test │ │ ├── AsyncLedgerOpsTest.java │ │ ├── BaseTestCase.java │ │ ├── BookKeeperClusterTestCase.java │ │ ├── BookieClientTest.java │ │ ├── BookieFailureTest.java │ │ ├── BookieJournalRollingTest.java │ │ ├── BookieReadWriteTest.java │ │ ├── BookieZKExpireTest.java │ │ ├── CloseTest.java │ │ ├── ConcurrentLedgerTest.java │ │ ├── ConditionalSetTest.java │ │ ├── ConfigurationTest.java │ │ ├── ForceReadOnlyBookieTest.java │ │ ├── LedgerCreateDeleteTest.java │ │ ├── LedgerDeleteTest.java │ │ ├── LoopbackClient.java │ │ ├── MultiLedgerManagerMultiDigestTestCase.java │ │ ├── MultiLedgerManagerTestCase.java │ │ ├── MultipleThreadReadTest.java │ │ ├── PortManager.java │ │ ├── ReadOnlyBookieTest.java │ │ ├── TestBackwardCompat.java │ │ ├── TestCallbacks.java │ │ └── ZooKeeperUtil.java │ │ ├── util │ │ ├── StaticDNSResolver.java │ │ ├── TestDiskChecker.java │ │ └── TestUtils.java │ │ └── zookeeper │ │ └── TestZooKeeperClient.java │ ├── proto │ └── TestDataFormats.proto │ └── resources │ └── log4j.properties ├── bookkeeper-stats-providers ├── codahale-metrics-provider │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── bookkeeper │ │ │ └── stats │ │ │ ├── CodahaleMetricsProvider.java │ │ │ ├── CodahaleOpStatsLogger.java │ │ │ └── CodahaleStatsLogger.java │ │ └── test │ │ └── java │ │ └── org │ │ └── apache │ │ └── bookkeeper │ │ └── stats │ │ └── CodahaleOpStatsTest.java ├── pom.xml ├── twitter-ostrich-provider │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── apache │ │ └── bookkeeper │ │ └── stats │ │ └── twitter │ │ └── ostrich │ │ ├── CounterImpl.java │ │ ├── OpStatsLoggerImpl.java │ │ ├── OstrichProvider.java │ │ └── OstrichStatsLoggerImpl.java └── twitter-science-provider │ ├── pom.xml │ └── src │ └── main │ └── java │ └── org │ └── apache │ └── bookkeeper │ └── stats │ └── twitter │ └── science │ ├── CounterImpl.java │ ├── HTTPStatsExporter.java │ ├── OpStatsLoggerImpl.java │ ├── TwitterStatsLoggerImpl.java │ └── TwitterStatsProvider.java ├── bookkeeper-stats ├── pom.xml └── src │ └── main │ └── java │ └── org │ └── apache │ └── bookkeeper │ └── stats │ ├── Counter.java │ ├── Gauge.java │ ├── NullStatsLogger.java │ ├── NullStatsProvider.java │ ├── OpStatsData.java │ ├── OpStatsLogger.java │ ├── Stats.java │ ├── StatsLogger.java │ └── StatsProvider.java ├── compat-deps ├── bookkeeper-server-compat-4.0.0 │ └── pom.xml ├── bookkeeper-server-compat-4.1.0 │ └── pom.xml ├── bookkeeper-server-compat-4.2.0 │ └── pom.xml └── pom.xml ├── dev └── bk-merge-pr.py ├── doc ├── bookieConfigParams.textile ├── bookieRecovery.textile ├── bookkeeperConfig.textile ├── bookkeeperConfigParams.textile ├── bookkeeperInternals.textile ├── bookkeeperJMX.textile ├── bookkeeperLedgers2Logs.textile ├── bookkeeperMetadata.textile ├── bookkeeperOverview.textile ├── bookkeeperProgrammer.textile ├── bookkeeperProtocol.textile ├── bookkeeperStarted.textile ├── bookkeeperStream.textile ├── bookkeeperTutorial.textile ├── doc.textile ├── index.textile └── metastore.textile ├── formatter.xml ├── patch-review.py ├── pom.xml └── src └── assemble ├── bin.xml └── src.xml /NOTICE: -------------------------------------------------------------------------------- 1 | Apache BookKeeper 2 | Copyright 2011-2015 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | 7 | -------------------------------------------------------------------------------- /bin/raw-check-patch: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | printTrailingSpaces() { 16 | PATCH=$1 17 | cat $PATCH | awk '/^+/ { if (/ $/) { print "\tL" NR ":" $0} }' 18 | } 19 | 20 | printTabs() { 21 | PATCH=$1 22 | cat $PATCH | awk '/^+/ { if (/\t/) { print "\tL" NR ":" $0 } }' 23 | } 24 | 25 | printAuthors() { 26 | PATCH=$1 27 | cat $PATCH | awk '/^+/ { L=tolower($0); if (L ~ /.*\*.* @author/) { print "\tL" NR ":" $0 } }' 28 | } 29 | 30 | printLongLines() { 31 | PATCH=$1 32 | cat $PATCH | awk '/^+/ { if ( length > 121 ) { print "\tL" NR ":" $0 } }' 33 | } 34 | 35 | if [[ "X$(basename -- "$0")" = "Xraw-check-patch" ]]; then 36 | echo Trailing spaces 37 | printTrailingSpaces $1 38 | echo 39 | echo Tabs 40 | printTabs $1 41 | echo 42 | echo Authors 43 | printAuthors $1 44 | echo 45 | echo Long lines 46 | printLongLines $1 47 | fi 48 | -------------------------------------------------------------------------------- /bin/update-master-docs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | set +x 16 | 17 | BASEDIR=/tmp/update-master-docs 18 | mkdir -p $BASEDIR 19 | 20 | SVNDIR=$BASEDIR/svn 21 | 22 | GITDIR=$BASEDIR/git 23 | 24 | rm -rf $SVNDIR 25 | rm -rf $GITDIR 26 | 27 | svn checkout https://svn.apache.org/repos/asf/bookkeeper/site/trunk/content/docs/master $SVNDIR 28 | svn checkout https://github.com/apache/bookkeeper/trunk $GITDIR 29 | 30 | mvn -f $GITDIR/pom.xml -Dnotimestamp javadoc:aggregate 31 | 32 | rsync -avP --exclude=.svn $GITDIR/doc/ $SVNDIR/ 33 | rsync -avP --exclude=.svn $GITDIR/target/site/apidocs/ $SVNDIR/apidocs/ 34 | svn add --force $SVNDIR/* 35 | 36 | CHANGES=$(svn status $SVNDIR| awk 'BEGIN { COUNT = 0; } /^[A|M]/ { COUNT = COUNT + 1; } END { print COUNT }') 37 | if [ $CHANGES -gt 0 ]; then 38 | svn commit -m "Syncing website with master documentation" $SVNDIR 39 | echo 40 | echo "Now go review on http://bookkeeper.staging.apache.org, and publish if it's ok" 41 | echo 42 | else 43 | echo "No changes, submitting nothing" 44 | fi 45 | -------------------------------------------------------------------------------- /bookkeeper-server/conf/bkenv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | #/** 4 | # * Copyright 2007 The Apache Software Foundation 5 | # * 6 | # * Licensed to the Apache Software Foundation (ASF) under one 7 | # * or more contributor license agreements. See the NOTICE file 8 | # * distributed with this work for additional information 9 | # * regarding copyright ownership. The ASF licenses this file 10 | # * to you under the Apache License, Version 2.0 (the 11 | # * "License"); you may not use this file except in compliance 12 | # * with the License. You may obtain a copy of the License at 13 | # * 14 | # * http://www.apache.org/licenses/LICENSE-2.0 15 | # * 16 | # * Unless required by applicable law or agreed to in writing, software 17 | # * distributed under the License is distributed on an "AS IS" BASIS, 18 | # * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | # * See the License for the specific language governing permissions and 20 | # * limitations under the License. 21 | # */ 22 | 23 | # Set JAVA_HOME here to override the environment setting 24 | # JAVA_HOME= 25 | 26 | # default settings for starting bookkeeper 27 | 28 | # Configuration file of settings used in bookie server 29 | # BOOKIE_CONF= 30 | 31 | # Log4j configuration file 32 | # BOOKIE_LOG_CONF= 33 | 34 | # Logs location 35 | # BOOKIE_LOG_DIR= 36 | 37 | # Extra options to be passed to the jvm 38 | # BOOKIE_EXTRA_OPTS= 39 | 40 | # Add extra paths to the bookkeeper classpath 41 | # BOOKIE_EXTRA_CLASSPATH= 42 | 43 | #Folder where the Bookie server PID file should be stored 44 | #BOOKIE_PID_DIR= 45 | 46 | #Wait time before forcefully kill the Bookie server instance, if the stop is not successful 47 | #BOOKIE_STOP_TIMEOUT= 48 | 49 | #Entry formatter class to format entries. 50 | #ENTRY_FORMATTER_CLASS= 51 | -------------------------------------------------------------------------------- /bookkeeper-server/conf/log4j.shell.properties: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # 20 | # 21 | 22 | # 23 | # BookieShell configuration 24 | 25 | # DEFAULT: console appender only 26 | # Define some default values that can be overridden by system properties 27 | bookkeeper.root.logger=ERROR,CONSOLE 28 | 29 | log4j.rootLogger=${bookkeeper.root.logger} 30 | 31 | # 32 | # Log INFO level and above messages to the console 33 | # 34 | log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender 35 | log4j.appender.CONSOLE.Threshold=INFO 36 | log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout 37 | log4j.appender.CONSOLE.layout.ConversionPattern=%d{ABSOLUTE} %-5p %m%n 38 | 39 | log4j.logger.org.apache.zookeeper=ERROR 40 | log4j.logger.org.apache.bookkeeper=ERROR 41 | log4j.logger.org.apache.bookkeeper.bookie.BookieShell=INFO 42 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.bookkeeper.bookie; 20 | 21 | import java.io.File; 22 | 23 | import org.apache.bookkeeper.jmx.BKMBeanInfo; 24 | 25 | /** 26 | * Bookie Bean 27 | */ 28 | public class BookieBean implements BookieMXBean, BKMBeanInfo { 29 | 30 | protected Bookie bk; 31 | 32 | public BookieBean(Bookie bk) { 33 | this.bk = bk; 34 | } 35 | 36 | @Override 37 | public String getName() { 38 | return "Bookie"; 39 | } 40 | 41 | @Override 42 | public boolean isHidden() { 43 | return false; 44 | } 45 | 46 | @Override 47 | public int getQueueLength() { 48 | return bk.journal.getJournalQueueLength(); 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieCriticalThread.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.bookkeeper.bookie; 19 | 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | /** 24 | * Thread is marked as critical and will exit, when there is an uncaught 25 | * exception occurred in thread. 26 | */ 27 | public class BookieCriticalThread extends BookieThread { 28 | private static final Logger LOG = LoggerFactory 29 | .getLogger(BookieCriticalThread.class); 30 | 31 | public BookieCriticalThread(String name) { 32 | super(name); 33 | } 34 | 35 | public BookieCriticalThread(Runnable thread, String name) { 36 | super(thread, name); 37 | } 38 | 39 | @Override 40 | protected void handleException(Thread t, Throwable e) { 41 | LOG.error("Uncaught exception in thread {} and is exiting!", 42 | t.getName(), e); 43 | Runtime.getRuntime().exit(ExitCode.BOOKIE_EXCEPTION); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieMXBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.bookkeeper.bookie; 20 | 21 | import java.io.File; 22 | 23 | /** 24 | * Bookie MBean 25 | */ 26 | public interface BookieMXBean { 27 | /** 28 | * @return log entry queue length 29 | */ 30 | public int getQueueLength(); 31 | } 32 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieThread.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.bookkeeper.bookie; 19 | 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | /* 24 | * Wrapper that wraps bookie threads 25 | * Any common handing that we require for all bookie threads 26 | * should be implemented here 27 | */ 28 | public class BookieThread extends Thread implements 29 | Thread.UncaughtExceptionHandler { 30 | 31 | private static final Logger LOG = LoggerFactory 32 | .getLogger(BookieThread.class); 33 | 34 | @Override 35 | public void uncaughtException(Thread t, Throwable e) { 36 | handleException(t, e); 37 | } 38 | 39 | public BookieThread(String name) { 40 | super(name); 41 | setUncaughtExceptionHandler(this); 42 | } 43 | 44 | public BookieThread(Runnable thread, String name) { 45 | super(thread, name); 46 | setUncaughtExceptionHandler(this); 47 | } 48 | 49 | /** 50 | * Handles uncaught exception occurred in thread 51 | */ 52 | protected void handleException(Thread t, Throwable e) { 53 | LOG.error("Uncaught exception in thread {}", t.getName(), e); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BufferedChannelBase.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.bookkeeper.bookie; 19 | 20 | import java.io.IOException; 21 | import java.nio.channels.FileChannel; 22 | 23 | public abstract class BufferedChannelBase { 24 | protected final FileChannel fileChannel; 25 | 26 | protected BufferedChannelBase(FileChannel fc) { 27 | this.fileChannel = fc; 28 | } 29 | 30 | protected FileChannel validateAndGetFileChannel() throws IOException { 31 | // Even if we have BufferedChannelBase objects in the cache, higher layers should 32 | // guarantee that once a log file has been closed and possibly deleted during garbage 33 | // collection, attempts will not be made to read from it 34 | if (!fileChannel.isOpen()) { 35 | throw new IOException("Attempting to access a file channel that has already been closed"); 36 | } 37 | return fileChannel; 38 | } 39 | 40 | /** 41 | * Get the current size of the underlying FileChannel. 42 | * @return 43 | */ 44 | public long size() throws IOException { 45 | return validateAndGetFileChannel().size(); 46 | } 47 | 48 | /** 49 | * Get the {@link FileChannel} that this BufferedChannel wraps around. 50 | * @return 51 | */ 52 | public FileChannel getFileChannel() { 53 | return fileChannel; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/CacheCallback.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.bookie; 23 | 24 | import java.io.IOException; 25 | 26 | /** 27 | * Interface plugged into caching to receive callback notifications 28 | */ 29 | public interface CacheCallback { 30 | /** 31 | * Process notification that cache size limit reached. 32 | */ 33 | public void onSizeLimitReached() throws IOException; 34 | } 35 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/CompactableLedgerStorage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.bookie; 23 | 24 | import java.io.IOException; 25 | 26 | /** 27 | * Interface that identifies LedgerStorage implementations using EntryLogger and running periodic entries compaction 28 | */ 29 | public interface CompactableLedgerStorage extends LedgerStorage { 30 | 31 | /** 32 | * @return the EntryLogger used by the ledger storage 33 | */ 34 | EntryLogger getEntryLogger(); 35 | 36 | /** 37 | * Get an iterator over a range of ledger ids stored in the bookie. 38 | * 39 | * @param firstLedgerId first ledger id in the sequence (included) 40 | * @param lastLedgerId last ledger id in the sequence (not included) 41 | * @return 42 | */ 43 | Iterable getActiveLedgersInRange(long firstLedgerId, long lastLedgerId) 44 | throws IOException; 45 | 46 | /** 47 | * Update the location of several entries 48 | * 49 | * @param locations 50 | * the list of locations to update 51 | * @throws IOException 52 | */ 53 | void updateEntriesLocations(Iterable locations) throws IOException; 54 | 55 | /** 56 | * Flush the entries locations index for the compacted entries 57 | * 58 | * @throws IOException 59 | */ 60 | void flushEntriesLocationsIndex() throws IOException; 61 | } -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryKey.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright The Apache Software Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package org.apache.bookkeeper.bookie; 21 | 22 | import java.io.Serializable; 23 | import java.util.Comparator; 24 | 25 | public class EntryKey { 26 | long ledgerId; 27 | long entryId; 28 | 29 | public EntryKey() { 30 | this(0, 0); 31 | } 32 | 33 | public EntryKey(long ledgerId, long entryId) { 34 | this.ledgerId = ledgerId; 35 | this.entryId = entryId; 36 | } 37 | 38 | public long getLedgerId() { 39 | return ledgerId; 40 | } 41 | 42 | public long getEntryId() { 43 | return entryId; 44 | } 45 | 46 | /** 47 | * Comparator for the key portion 48 | */ 49 | public static final KeyComparator COMPARATOR = new KeyComparator(); 50 | 51 | // Only compares the key portion 52 | @Override 53 | public boolean equals(Object other) { 54 | if (!(other instanceof EntryKey)) { 55 | return false; 56 | } 57 | EntryKey key = (EntryKey)other; 58 | return ledgerId == key.ledgerId && 59 | entryId == key.entryId; 60 | } 61 | 62 | @Override 63 | public int hashCode() { 64 | return (int)(ledgerId * 13 ^ entryId * 17); 65 | } 66 | } 67 | 68 | /** 69 | * Compare EntryKey. 70 | */ 71 | class KeyComparator implements Comparator, Serializable { 72 | 73 | private static final long serialVersionUID = 0L; 74 | 75 | @Override 76 | public int compare(EntryKey left, EntryKey right) { 77 | long ret = left.ledgerId - right.ledgerId; 78 | if (ret == 0) { 79 | ret = left.entryId - right.entryId; 80 | } 81 | return (ret < 0)? -1 : ((ret > 0)? 1 : 0); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLocation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.bookie; 23 | 24 | public class EntryLocation { 25 | public final long ledger; 26 | public final long entry; 27 | public final long location; 28 | 29 | public EntryLocation(long ledger, long entry, long location) { 30 | this.ledger = ledger; 31 | this.entry = entry; 32 | this.location = location; 33 | } 34 | } -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ExitCode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.bookie; 23 | 24 | /** 25 | * Exit code used to exit bookie server 26 | */ 27 | public class ExitCode { 28 | // normal quit 29 | public final static int OK = 0; 30 | // invalid configuration 31 | public final static int INVALID_CONF = 1; 32 | // exception running bookie server 33 | public final static int SERVER_EXCEPTION = 2; 34 | // zookeeper is expired 35 | public final static int ZK_EXPIRED = 3; 36 | // register bookie on zookeeper failed 37 | public final static int ZK_REG_FAIL = 4; 38 | // exception running bookie 39 | public final static int BOOKIE_EXCEPTION = 5; 40 | } 41 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollector.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.bookie; 23 | 24 | /** 25 | * This is the garbage collector interface, garbage collector implementers 26 | * need to extends this class to remove the deleted ledgers. 27 | */ 28 | public interface GarbageCollector { 29 | /** 30 | * Do the garbage collector work 31 | * 32 | * @param garbageCleaner 33 | * cleaner used to clean selected garbages 34 | */ 35 | public abstract void gc(GarbageCleaner garbageCleaner); 36 | 37 | /** 38 | * A interface used to define customised garbage cleaner 39 | */ 40 | public interface GarbageCleaner { 41 | 42 | /** 43 | * Clean a specific ledger 44 | * 45 | * @param ledgerId 46 | * Ledger ID to be cleaned 47 | */ 48 | public void clean(final long ledgerId) ; 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/HandleFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.bookie; 23 | 24 | import java.io.IOException; 25 | 26 | interface HandleFactory { 27 | LedgerDescriptor getHandle(long ledgerId, byte[] masterKey) 28 | throws IOException, BookieException; 29 | 30 | LedgerDescriptor getReadOnlyHandle(long ledgerId) 31 | throws IOException, Bookie.NoLedgerException; 32 | } -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LEPStateChangeCallback.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.bookie; 22 | 23 | /** 24 | * Callback interface when state of ledger entry page changed. 25 | */ 26 | interface LEPStateChangeCallback { 27 | public void onSetInUse(LedgerEntryPage lep); 28 | public void onResetInUse(LedgerEntryPage lep); 29 | public void onSetClean(LedgerEntryPage lep); 30 | public void onSetDirty(LedgerEntryPage lep); 31 | } 32 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCache.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.bookie; 23 | 24 | import java.io.Closeable; 25 | import java.io.IOException; 26 | 27 | /** 28 | * This class maps a ledger entry number into a location (entrylogid, offset) in 29 | * an entry log file. It does user level caching to more efficiently manage disk 30 | * head scheduling. 31 | */ 32 | interface LedgerCache extends Closeable { 33 | 34 | boolean setFenced(long ledgerId) throws IOException; 35 | boolean isFenced(long ledgerId) throws IOException; 36 | 37 | void setMasterKey(long ledgerId, byte[] masterKey) throws IOException; 38 | byte[] readMasterKey(long ledgerId) throws IOException, BookieException; 39 | boolean ledgerExists(long ledgerId) throws IOException; 40 | 41 | void putEntryOffset(long ledger, long entry, long offset) throws IOException; 42 | long getEntryOffset(long ledger, long entry) throws IOException; 43 | 44 | void flushLedger(boolean doAll) throws IOException; 45 | long getLastEntry(long ledgerId) throws IOException; 46 | 47 | void deleteLedger(long ledgerId) throws IOException; 48 | 49 | LedgerCacheBean getJMXBean(); 50 | } 51 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCacheBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.bookkeeper.bookie; 20 | 21 | import org.apache.bookkeeper.jmx.BKMBeanInfo; 22 | 23 | /** 24 | * Ledger Cache Bean 25 | */ 26 | public interface LedgerCacheBean extends LedgerCacheMXBean, BKMBeanInfo { 27 | } 28 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCacheMXBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.bookkeeper.bookie; 20 | 21 | /** 22 | * Ledger Cache MBean 23 | */ 24 | public interface LedgerCacheMXBean { 25 | 26 | /** 27 | * @return number of page used in cache 28 | */ 29 | public int getPageCount(); 30 | 31 | /** 32 | * @return page size 33 | */ 34 | public int getPageSize(); 35 | 36 | /** 37 | * @return the limit of open files 38 | */ 39 | public int getOpenFileLimit(); 40 | 41 | /** 42 | * @return the limit number of pages 43 | */ 44 | public int getPageLimit(); 45 | 46 | /** 47 | * @return number of open ledgers 48 | */ 49 | public int getNumOpenLedgers(); 50 | } 51 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDescriptor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.bookie; 23 | 24 | import java.io.IOException; 25 | import java.nio.ByteBuffer; 26 | import java.util.Arrays; 27 | 28 | import org.slf4j.Logger; 29 | import org.slf4j.LoggerFactory; 30 | 31 | /** 32 | * Implements a ledger inside a bookie. In particular, it implements operations 33 | * to write entries to a ledger and read entries from a ledger. 34 | */ 35 | public abstract class LedgerDescriptor { 36 | static LedgerDescriptor create(byte[] masterKey, 37 | long ledgerId, 38 | LedgerStorage ledgerStorage) throws IOException { 39 | LedgerDescriptor ledger = new LedgerDescriptorImpl(masterKey, ledgerId, ledgerStorage); 40 | ledgerStorage.setMasterKey(ledgerId, masterKey); 41 | return ledger; 42 | } 43 | 44 | static LedgerDescriptor createReadOnly(long ledgerId, 45 | LedgerStorage ledgerStorage) 46 | throws IOException, Bookie.NoLedgerException { 47 | if (!ledgerStorage.ledgerExists(ledgerId)) { 48 | throw new Bookie.NoLedgerException(ledgerId); 49 | } 50 | return new LedgerDescriptorReadOnlyImpl(ledgerId, ledgerStorage); 51 | } 52 | 53 | abstract void checkAccess(byte masterKey[]) throws BookieException, IOException; 54 | 55 | abstract long getLedgerId(); 56 | 57 | abstract boolean setFenced() throws IOException; 58 | abstract boolean isFenced() throws IOException; 59 | 60 | abstract long addEntry(ByteBuffer entry) throws IOException; 61 | abstract ByteBuffer readEntry(long entryId) throws IOException; 62 | } 63 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDescriptorReadOnlyImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.bookie; 23 | 24 | import java.io.IOException; 25 | import java.nio.ByteBuffer; 26 | /** 27 | * Implements a ledger inside a bookie. In particular, it implements operations 28 | * to write entries to a ledger and read entries from a ledger. 29 | */ 30 | public class LedgerDescriptorReadOnlyImpl extends LedgerDescriptorImpl { 31 | LedgerDescriptorReadOnlyImpl(long ledgerId, LedgerStorage storage) { 32 | super(null, ledgerId, storage); 33 | } 34 | 35 | @Override 36 | boolean setFenced() throws IOException { 37 | assert false; 38 | throw new IOException("Invalid action on read only descriptor"); 39 | } 40 | 41 | @Override 42 | long addEntry(ByteBuffer entry) throws IOException { 43 | assert false; 44 | throw new IOException("Invalid action on read only descriptor"); 45 | } 46 | 47 | @Override 48 | void checkAccess(byte masterKey[]) throws BookieException, IOException { 49 | assert false; 50 | throw new IOException("Invalid action on read only descriptor"); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerStorageFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.bookie; 22 | 23 | import java.io.IOException; 24 | 25 | import org.apache.bookkeeper.util.ReflectionUtils; 26 | 27 | public class LedgerStorageFactory { 28 | public static LedgerStorage createLedgerStorage(String name) throws IOException { 29 | try { 30 | return ReflectionUtils.newInstance(name, LedgerStorage.class); 31 | } catch (Throwable t) { 32 | throw new IOException("Failed to instantiate ledger storage : " + name, t); 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LogMark.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright The Apache Software Foundation 3 | * 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | package org.apache.bookkeeper.bookie; 21 | 22 | import java.nio.ByteBuffer; 23 | 24 | /** 25 | * Journal stream position 26 | */ 27 | class LogMark { 28 | long logFileId; 29 | long logFileOffset; 30 | 31 | public LogMark() { 32 | setLogMark(0, 0); 33 | } 34 | 35 | public LogMark(LogMark other) { 36 | setLogMark(other.getLogFileId(), other.getLogFileOffset()); 37 | } 38 | 39 | public LogMark(long logFileId, long logFileOffset) { 40 | setLogMark(logFileId, logFileOffset); 41 | } 42 | 43 | public synchronized long getLogFileId() { 44 | return logFileId; 45 | } 46 | 47 | public synchronized long getLogFileOffset() { 48 | return logFileOffset; 49 | } 50 | 51 | public synchronized void readLogMark(ByteBuffer bb) { 52 | logFileId = bb.getLong(); 53 | logFileOffset = bb.getLong(); 54 | } 55 | 56 | public synchronized void writeLogMark(ByteBuffer bb) { 57 | bb.putLong(logFileId); 58 | bb.putLong(logFileOffset); 59 | } 60 | 61 | public synchronized void setLogMark(long logFileId, long logFileOffset) { 62 | this.logFileId = logFileId; 63 | this.logFileOffset = logFileOffset; 64 | } 65 | 66 | public synchronized int compare(LogMark other) { 67 | long ret = this.logFileId - other.getLogFileId(); 68 | if (ret == 0) { 69 | ret = this.logFileOffset - other.getLogFileOffset(); 70 | } 71 | return (ret < 0)? -1 : ((ret > 0)? 1 : 0); 72 | } 73 | 74 | @Override 75 | public synchronized String toString() { 76 | StringBuilder sb = new StringBuilder(); 77 | 78 | sb.append("LogMark: logFileId - ").append(logFileId) 79 | .append(" , logFileOffset - ").append(logFileOffset); 80 | 81 | return sb.toString(); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ReadOnlyBookie.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.bookie; 23 | 24 | import java.io.IOException; 25 | 26 | import org.apache.bookkeeper.conf.ServerConfiguration; 27 | import org.apache.bookkeeper.stats.StatsLogger; 28 | import org.apache.zookeeper.KeeperException; 29 | import org.slf4j.Logger; 30 | import org.slf4j.LoggerFactory; 31 | 32 | /** 33 | * Implements a read only bookie. 34 | * 35 | * ReadOnlyBookie is force started as readonly, and will not change to writable. 36 | * 37 | */ 38 | public class ReadOnlyBookie extends Bookie { 39 | 40 | private final static Logger LOG = LoggerFactory.getLogger(ReadOnlyBookie.class); 41 | 42 | public ReadOnlyBookie(ServerConfiguration conf, StatsLogger statsLogger) 43 | throws IOException, KeeperException, InterruptedException, BookieException { 44 | super(conf, statsLogger); 45 | if (conf.isReadOnlyModeEnabled()) { 46 | readOnly.set(true); 47 | } else { 48 | String err = "Try to init ReadOnly Bookie, while ReadOnly mode is not enabled"; 49 | LOG.error(err); 50 | throw new IOException(err); 51 | } 52 | LOG.info("Running bookie in readonly mode."); 53 | } 54 | 55 | @Override 56 | public void doTransitionToWritableMode() { 57 | // no-op 58 | LOG.info("Skip transition to writable mode for readonly bookie"); 59 | } 60 | 61 | @Override 62 | public void doTransitionToReadOnlyMode() { 63 | // no-op 64 | LOG.info("Skip transition to readonly mode for readonly bookie"); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ReadOnlyEntryLogger.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.bookie; 23 | 24 | import java.io.IOException; 25 | import java.nio.ByteBuffer; 26 | 27 | import org.apache.bookkeeper.conf.ServerConfiguration; 28 | 29 | /** 30 | * Read Only Entry Logger 31 | */ 32 | public class ReadOnlyEntryLogger extends EntryLogger { 33 | 34 | public ReadOnlyEntryLogger(ServerConfiguration conf) throws IOException { 35 | super(conf, new LedgerDirsManager(conf, conf.getLedgerDirs())); 36 | } 37 | 38 | @Override 39 | protected void initialize() throws IOException { 40 | // do nothing for read only entry logger 41 | } 42 | 43 | @Override 44 | void createNewLog() throws IOException { 45 | throw new IOException("Can't create new entry log using a readonly entry logger."); 46 | } 47 | 48 | @Override 49 | protected boolean removeEntryLog(long entryLogId) { 50 | // can't remove entry log in readonly mode 51 | return false; 52 | } 53 | 54 | @Override 55 | public synchronized long addEntry(long ledger, ByteBuffer entry) throws IOException { 56 | throw new IOException("Can't add entry to a readonly entry logger."); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ReadOnlyFileInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.bookie; 23 | 24 | import java.io.File; 25 | import java.io.IOException; 26 | import java.io.RandomAccessFile; 27 | import java.nio.ByteBuffer; 28 | import java.nio.BufferUnderflowException; 29 | import java.nio.channels.FileChannel; 30 | 31 | import org.slf4j.Logger; 32 | import org.slf4j.LoggerFactory; 33 | 34 | /** 35 | * Provide a readonly file info. 36 | */ 37 | class ReadOnlyFileInfo extends FileInfo { 38 | 39 | public ReadOnlyFileInfo(File lf, byte[] masterKey) throws IOException { 40 | super(lf, masterKey); 41 | mode = "r"; 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/SkipListFlusher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.bookie; 23 | 24 | import java.io.IOException; 25 | import java.nio.ByteBuffer; 26 | 27 | /** 28 | * Flush entries from skip list 29 | */ 30 | public interface SkipListFlusher { 31 | /** 32 | * Process an entry. 33 | * 34 | * @param ledgerId 35 | * Ledger ID. 36 | * @param entryId 37 | * The entry id this entry. 38 | * @param entry 39 | * Entry ByteBuffer 40 | * @throws IOException 41 | */ 42 | public void process(long ledgerId, long entryId, ByteBuffer entry) throws IOException; 43 | } 44 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperClientStats.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.client; 23 | 24 | public interface BookKeeperClientStats { 25 | public final static String CLIENT_SCOPE = "bookkeeper_client"; 26 | public final static String CREATE_OP = "LEDGER_CREATE"; 27 | public final static String DELETE_OP = "LEDGER_DELETE"; 28 | public final static String OPEN_OP = "LEDGER_OPEN"; 29 | public final static String ADD_OP = "ADD_ENTRY"; 30 | public final static String READ_OP = "READ_ENTRY"; 31 | public final static String PENDING_ADDS = "NUM_PENDING_ADD"; 32 | public final static String ENSEMBLE_CHANGES = "NUM_ENSEMBLE_CHANGE"; 33 | public final static String LAC_UPDATE_HITS = "LAC_UPDATE_HITS"; 34 | public final static String LAC_UPDATE_MISSES = "LAC_UPDATE_MISSES"; 35 | 36 | // per channel stats 37 | public final static String CHANNEL_SCOPE = "per_channel_bookie_client"; 38 | 39 | public final static String CHANNEL_READ_OP = "READ_ENTRY"; 40 | public final static String CHANNEL_TIMEOUT_READ = "TIMEOUT_READ_ENTRY"; 41 | public final static String CHANNEL_ADD_OP = "ADD_ENTRY"; 42 | public final static String CHANNEL_TIMEOUT_ADD = "TIMEOUT_ADD_ENTRY"; 43 | } 44 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookiesListener.java: -------------------------------------------------------------------------------- 1 | package org.apache.bookkeeper.client; 2 | 3 | /** 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | 21 | 22 | /** 23 | * Listener for the the available bookies changes. 24 | */ 25 | public interface BookiesListener { 26 | void availableBookiesChanged(); 27 | } 28 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/client/CRC32DigestManager.java: -------------------------------------------------------------------------------- 1 | package org.apache.bookkeeper.client; 2 | 3 | /* 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | 21 | 22 | import java.nio.ByteBuffer; 23 | import java.util.zip.CRC32; 24 | 25 | class CRC32DigestManager extends DigestManager { 26 | private final ThreadLocal crc = new ThreadLocal() { 27 | @Override 28 | protected CRC32 initialValue() { 29 | return new CRC32(); 30 | } 31 | }; 32 | 33 | public CRC32DigestManager(long ledgerId) { 34 | super(ledgerId); 35 | } 36 | 37 | @Override 38 | int getMacCodeLength() { 39 | return 8; 40 | } 41 | 42 | @Override 43 | byte[] getValueAndReset() { 44 | byte[] value = new byte[8]; 45 | ByteBuffer buf = ByteBuffer.wrap(value); 46 | buf.putLong(crc.get().getValue()); 47 | crc.get().reset(); 48 | return value; 49 | } 50 | 51 | @Override 52 | void update(byte[] data, int offset, int length) { 53 | crc.get().update(data, offset, length); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/client/SyncCounter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.client; 23 | 24 | import java.util.Enumeration; 25 | 26 | /** 27 | * Implements objects to help with the synchronization of asynchronous calls 28 | * 29 | */ 30 | 31 | class SyncCounter { 32 | int i; 33 | int rc; 34 | int total; 35 | Enumeration seq = null; 36 | LedgerHandle lh = null; 37 | 38 | synchronized void inc() { 39 | i++; 40 | total++; 41 | } 42 | 43 | synchronized void dec() { 44 | i--; 45 | notifyAll(); 46 | } 47 | 48 | synchronized void block(int limit) throws InterruptedException { 49 | while (i > limit) { 50 | int prev = i; 51 | wait(); 52 | if (i == prev) { 53 | break; 54 | } 55 | } 56 | } 57 | 58 | synchronized int total() { 59 | return total; 60 | } 61 | 62 | void setrc(int rc) { 63 | this.rc = rc; 64 | } 65 | 66 | int getrc() { 67 | return rc; 68 | } 69 | 70 | void setSequence(Enumeration seq) { 71 | this.seq = seq; 72 | } 73 | 74 | Enumeration getSequence() { 75 | return seq; 76 | } 77 | 78 | void setLh(LedgerHandle lh) { 79 | this.lh = lh; 80 | } 81 | 82 | LedgerHandle getLh() { 83 | return lh; 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/Configurable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.conf; 22 | 23 | import org.apache.commons.configuration.Configuration; 24 | 25 | import com.google.common.annotations.Beta; 26 | 27 | /** 28 | * Class that may be configured with a {@link Configuration}. 29 | */ 30 | @Beta 31 | public interface Configurable { 32 | 33 | /** 34 | * Set the configuration to be used by this object. 35 | * 36 | * @param conf 37 | * Configuration object to use 38 | */ 39 | public void setConf(Configuration conf); 40 | 41 | /** 42 | * Return the configuration used by this object. 43 | * 44 | * @return configuration used by this object. 45 | */ 46 | public Configuration getConf(); 47 | } 48 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/jmx/BKMBeanInfo.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.bookkeeper.jmx; 20 | 21 | import org.apache.zookeeper.jmx.ZKMBeanInfo; 22 | 23 | /** 24 | * BookKeeper MBean info interface. 25 | */ 26 | public interface BKMBeanInfo extends ZKMBeanInfo { 27 | } 28 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerIdGenerator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.bookkeeper.meta; 19 | 20 | import java.io.Closeable; 21 | 22 | import org.apache.bookkeeper.client.BKException; 23 | import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.GenericCallback; 24 | 25 | /** 26 | * The interface for global unique ledger ID generation 27 | */ 28 | public interface LedgerIdGenerator extends Closeable { 29 | 30 | /** 31 | * generate a global unique ledger id 32 | * 33 | * @param cb 34 | * Callback when a new ledger id is generated, return code:
    35 | *
  • {@link BKException.Code.OK} if success
    • 36 | *
  • {@link BKException.Code.ZKException} when can't generate new ledger id
    • 37 | *
38 | */ 39 | public void generateLedgerId(GenericCallback cb); 40 | 41 | } 42 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/ZkVersion.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.meta; 22 | 23 | import org.apache.bookkeeper.versioning.Version; 24 | import org.apache.bookkeeper.versioning.Version.Occurred; 25 | 26 | public class ZkVersion implements Version { 27 | int znodeVersion; 28 | 29 | public ZkVersion(int version) { 30 | znodeVersion = version; 31 | } 32 | 33 | @Override 34 | public Occurred compare(Version v) { 35 | if (null == v) { 36 | throw new NullPointerException("Version is not allowed to be null."); 37 | } 38 | if (v == Version.NEW) { 39 | return Occurred.AFTER; 40 | } else if (v == Version.ANY) { 41 | return Occurred.CONCURRENTLY; 42 | } else if (!(v instanceof ZkVersion)) { 43 | throw new IllegalArgumentException("Invalid version type"); 44 | } 45 | ZkVersion zv = (ZkVersion)v; 46 | int res = znodeVersion - zv.znodeVersion; 47 | if (res == 0) { 48 | return Occurred.CONCURRENTLY; 49 | } else if (res < 0) { 50 | return Occurred.BEFORE; 51 | } else { 52 | return Occurred.AFTER; 53 | } 54 | } 55 | 56 | public int getZnodeVersion() { 57 | return znodeVersion; 58 | } 59 | 60 | public ZkVersion setZnodeVersion(int znodeVersion) { 61 | this.znodeVersion = znodeVersion; 62 | return this; 63 | } 64 | 65 | @Override 66 | public String toString() { 67 | return Integer.toString(znodeVersion, 10); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/metastore/InMemoryMetaStore.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.bookkeeper.metastore; 19 | 20 | import java.util.HashMap; 21 | import java.util.Map; 22 | 23 | import org.apache.commons.configuration.Configuration; 24 | 25 | public class InMemoryMetaStore implements MetaStore { 26 | 27 | static final int CUR_VERSION = 1; 28 | 29 | static Map tables = 30 | new HashMap(); 31 | 32 | // for test 33 | public static void reset() { 34 | tables.clear(); 35 | } 36 | 37 | @Override 38 | public String getName() { 39 | return getClass().getName(); 40 | } 41 | 42 | @Override 43 | public int getVersion() { 44 | return CUR_VERSION; 45 | } 46 | 47 | @Override 48 | public void init(Configuration conf, int msVersion) 49 | throws MetastoreException { 50 | // do nothing 51 | } 52 | 53 | @Override 54 | public void close() { 55 | // do nothing 56 | } 57 | 58 | @Override 59 | public MetastoreTable createTable(String name) { 60 | return createInMemoryTable(name); 61 | } 62 | 63 | @Override 64 | public MetastoreScannableTable createScannableTable(String name) { 65 | return createInMemoryTable(name); 66 | } 67 | 68 | private InMemoryMetastoreTable createInMemoryTable(String name) { 69 | InMemoryMetastoreTable t = tables.get(name); 70 | if (t == null) { 71 | t = new InMemoryMetastoreTable(this, name); 72 | tables.put(name, t); 73 | } 74 | return t; 75 | } 76 | 77 | } 78 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/metastore/MSWatchedEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.bookkeeper.metastore; 19 | 20 | public class MSWatchedEvent { 21 | public enum EventType {CHANGED, REMOVED}; 22 | 23 | String key; 24 | EventType type; 25 | 26 | public MSWatchedEvent(String key, EventType type) { 27 | this.key = key; 28 | this.type = type; 29 | } 30 | 31 | public EventType getType() { 32 | return type; 33 | } 34 | 35 | public String getKey(){ 36 | return key; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/metastore/MetaStore.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.bookkeeper.metastore; 19 | 20 | import org.apache.commons.configuration.Configuration; 21 | 22 | /** 23 | * Metadata Store Interface. 24 | */ 25 | public interface MetaStore { 26 | /** 27 | * Return the name of the plugin. 28 | * 29 | * @return the plugin name. 30 | */ 31 | public String getName(); 32 | 33 | /** 34 | * Get the plugin verison. 35 | * 36 | * @return the plugin version. 37 | */ 38 | public int getVersion(); 39 | 40 | /** 41 | * Initialize the meta store. 42 | * 43 | * @param config 44 | * Configuration object passed to metastore 45 | * @param msVersion 46 | * Version to initialize the metastore 47 | * @throws MetastoreException when failed to initialize 48 | */ 49 | public void init(Configuration config, int msVersion) 50 | throws MetastoreException; 51 | 52 | /** 53 | * Close the meta store. 54 | */ 55 | public void close(); 56 | 57 | /** 58 | * Create a metastore table. 59 | * 60 | * @param name 61 | * Table name. 62 | * @return a metastore table 63 | * @throws MetastoreException when failed to create the metastore table. 64 | */ 65 | public MetastoreTable createTable(String name) 66 | throws MetastoreException; 67 | 68 | /** 69 | * Create a scannable metastore table. 70 | * 71 | * @param name 72 | * Table name. 73 | * @return a metastore scannable table 74 | * @throws MetastoreException when failed to create the metastore table. 75 | */ 76 | public MetastoreScannableTable createScannableTable(String name) 77 | throws MetastoreException; 78 | 79 | } 80 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/metastore/MetastoreCallback.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.bookkeeper.metastore; 19 | 20 | public interface MetastoreCallback { 21 | /** 22 | * @see MSException.Code 23 | */ 24 | public void complete(int rc, T value, Object ctx); 25 | } 26 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/metastore/MetastoreException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.bookkeeper.metastore; 19 | 20 | @SuppressWarnings("serial") 21 | public class MetastoreException extends Exception { 22 | 23 | public MetastoreException(String message) { 24 | super(message); 25 | } 26 | 27 | public MetastoreException(String message, Throwable t) { 28 | super(message, t); 29 | } 30 | 31 | public MetastoreException(Throwable t) { 32 | super(t); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/metastore/MetastoreFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.bookkeeper.metastore; 19 | 20 | import org.apache.bookkeeper.util.ReflectionUtils; 21 | 22 | public class MetastoreFactory { 23 | 24 | public static MetaStore createMetaStore(String name) 25 | throws MetastoreException { 26 | try { 27 | return ReflectionUtils.newInstance(name, MetaStore.class); 28 | } catch (Throwable t) { 29 | throw new MetastoreException("Failed to instantiate metastore : " + name); 30 | } 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/metastore/MetastoreTableItem.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.bookkeeper.metastore; 19 | 20 | import org.apache.bookkeeper.versioning.Versioned; 21 | 22 | /** 23 | * Identify an item in a metastore table. 24 | */ 25 | public class MetastoreTableItem { 26 | 27 | private String key; 28 | private Versioned value; 29 | 30 | public MetastoreTableItem(String key, Versioned value) { 31 | this.key = key; 32 | this.value = value; 33 | } 34 | 35 | /** 36 | * Get the key of the table item. 37 | * 38 | * @return key of table item. 39 | */ 40 | public String getKey() { 41 | return key; 42 | } 43 | 44 | /** 45 | * Set the key of the item. 46 | * 47 | * @param key Key 48 | */ 49 | public void setKey(String key) { 50 | this.key = key; 51 | } 52 | 53 | /** 54 | * Get the value of the item. 55 | * 56 | * @return value of the item. 57 | */ 58 | public Versioned getValue() { 59 | return value; 60 | } 61 | 62 | /** 63 | * Set the value of the item. 64 | * 65 | * @return value of the item. 66 | */ 67 | public void setValue(Versioned value) { 68 | this.value = value; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/metastore/MetastoreWatcher.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.bookkeeper.metastore; 19 | 20 | public interface MetastoreWatcher { 21 | public void process(MSWatchedEvent e); 22 | } 23 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/net/CommonConfigurationKeys.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.net; 22 | 23 | public interface CommonConfigurationKeys { 24 | 25 | // script file name to resolve network topology 26 | public static final String NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY = "networkTopologyScriptFileName"; 27 | // number of arguments that network topology resolve script used 28 | public static final String NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY = "networkTopologyScriptNumberArgs"; 29 | // default value of NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY 30 | public static final int NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_DEFAULT = 100; 31 | } 32 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/net/DNSToSwitchMapping.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | // This code has been copied from hadoop-common 0.23.1 19 | package org.apache.bookkeeper.net; 20 | 21 | import java.util.List; 22 | 23 | import com.google.common.annotations.Beta; 24 | 25 | /** 26 | * An interface that must be implemented to allow pluggable 27 | * DNS-name/IP-address to RackID resolvers. 28 | * 29 | */ 30 | @Beta 31 | public interface DNSToSwitchMapping { 32 | /** 33 | * Resolves a list of DNS-names/IP-addresses and returns back a list of 34 | * switch information (network paths). One-to-one correspondence must be 35 | * maintained between the elements in the lists. 36 | * Consider an element in the argument list - x.y.com. The switch information 37 | * that is returned must be a network path of the form /foo/rack, 38 | * where / is the root, and 'foo' is the switch where 'rack' is connected. 39 | * Note the hostname/ip-address is not part of the returned path. 40 | * The network topology of the cluster would determine the number of 41 | * components in the network path. 42 | *

43 | * 44 | * If a name cannot be resolved to a rack, the implementation 45 | * should return {@link NetworkTopology#DEFAULT_RACK}. This 46 | * is what the bundled implementations do, though it is not a formal requirement 47 | * 48 | * @param names the list of hosts to resolve (can be empty) 49 | * @return list of resolved network paths. 50 | * If names is empty, the returned list is also empty 51 | */ 52 | public List resolve(List names); 53 | 54 | /** 55 | * Reload all of the cached mappings. 56 | * 57 | * If there is a cache, this method will clear it, so that future accesses 58 | * will get a chance to see the new data. 59 | */ 60 | public void reloadCachedMappings(); 61 | } 62 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/net/NetUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | // This code has been copied from hadoop-common 0.23.1 19 | package org.apache.bookkeeper.net; 20 | 21 | import java.net.InetAddress; 22 | import java.net.UnknownHostException; 23 | import java.util.ArrayList; 24 | import java.util.Collection; 25 | import java.util.List; 26 | 27 | public class NetUtils { 28 | 29 | /** 30 | * Given a string representation of a host, return its ip address 31 | * in textual presentation. 32 | * 33 | * @param name a string representation of a host: 34 | * either a textual representation its IP address or its host name 35 | * @return its IP address in the string format 36 | */ 37 | public static String normalizeHostName(String name) { 38 | try { 39 | return InetAddress.getByName(name).getHostAddress(); 40 | } catch (UnknownHostException e) { 41 | return name; 42 | } 43 | } 44 | 45 | /** 46 | * Given a collection of string representation of hosts, return a list of 47 | * corresponding IP addresses in the textual representation. 48 | * 49 | * @param names a collection of string representations of hosts 50 | * @return a list of corresponding IP addresses in the string format 51 | * @see #normalizeHostName(String) 52 | */ 53 | public static List normalizeHostNames(Collection names) { 54 | List hostNames = new ArrayList(names.size()); 55 | for (String name : names) { 56 | hostNames.add(normalizeHostName(name)); 57 | } 58 | return hostNames; 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/net/Node.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | // This code has been copied from hadoop-common 0.23.1 19 | package org.apache.bookkeeper.net; 20 | 21 | import com.google.common.annotations.Beta; 22 | 23 | /** The interface defines a node in a network topology. 24 | * A node may be a leave representing a data node or an inner 25 | * node representing a datacenter or rack. 26 | * Each data has a name and its location in the network is 27 | * decided by a string with syntax similar to a file name. 28 | * For example, a data node's name is hostname:port# and if it's located at 29 | * rack "orange" in datacenter "dog", the string representation of its 30 | * network location is /dog/orange 31 | */ 32 | @Beta 33 | public interface Node { 34 | /** @return the string representation of this node's network location */ 35 | public String getNetworkLocation(); 36 | 37 | /** Set this node's network location 38 | * @param location the location 39 | */ 40 | public void setNetworkLocation(String location); 41 | 42 | /** @return this node's name */ 43 | public String getName(); 44 | 45 | /** @return this node's parent */ 46 | public Node getParent(); 47 | 48 | /** Set this node's parent 49 | * @param parent the parent 50 | */ 51 | public void setParent(Node parent); 52 | 53 | /** @return this node's level in the tree. 54 | * E.g. the root of a tree returns 0 and its children return 1 55 | */ 56 | public int getLevel(); 57 | 58 | /** Set this node's level in the tree 59 | * @param i the level 60 | */ 61 | public void setLevel(int i); 62 | } 63 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/processor/RequestProcessor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.processor; 22 | 23 | import org.jboss.netty.channel.Channel; 24 | 25 | public interface RequestProcessor { 26 | 27 | /** 28 | * Close the request processor. 29 | */ 30 | public void close(); 31 | 32 | /** 33 | * Process request. 34 | * 35 | * @param r 36 | * request to process 37 | * @param channel 38 | * channel received the given request r 39 | */ 40 | public void processRequest(Object r, Channel channel); 41 | 42 | } 43 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServerBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.bookkeeper.proto; 20 | 21 | import java.net.UnknownHostException; 22 | 23 | import org.apache.bookkeeper.bookie.Bookie; 24 | import org.apache.bookkeeper.conf.ServerConfiguration; 25 | import org.apache.bookkeeper.jmx.BKMBeanInfo; 26 | import org.apache.bookkeeper.proto.BKStats.OpStatData; 27 | 28 | /** 29 | * Bookie Server Bean 30 | */ 31 | public class BookieServerBean implements BookieServerMXBean, BKMBeanInfo { 32 | 33 | protected final BookieServer bks; 34 | protected final ServerConfiguration conf; 35 | private final String name; 36 | 37 | public BookieServerBean(ServerConfiguration conf, BookieServer bks) { 38 | this.conf = conf; 39 | this.bks = bks; 40 | name = "BookieServer_" + conf.getBookiePort(); 41 | } 42 | 43 | @Override 44 | public String getName() { 45 | return name; 46 | } 47 | 48 | @Override 49 | public boolean isHidden() { 50 | return false; 51 | } 52 | 53 | @Override 54 | public long getNumPacketsReceived() { 55 | return ServerStats.getInstance().getPacketsReceived(); 56 | } 57 | 58 | @Override 59 | public long getNumPacketsSent() { 60 | return ServerStats.getInstance().getPacketsSent(); 61 | } 62 | 63 | @Override 64 | public OpStatData getAddStats() { 65 | return BKStats.getInstance().getOpStats(BKStats.STATS_ADD).toOpStatData(); 66 | } 67 | 68 | @Override 69 | public OpStatData getReadStats() { 70 | return BKStats.getInstance().getOpStats(BKStats.STATS_READ).toOpStatData(); 71 | } 72 | 73 | @Override 74 | public String getServerPort() { 75 | try { 76 | return Bookie.getBookieAddress(conf).toString(); 77 | } catch (UnknownHostException e) { 78 | return "localhost:" + conf.getBookiePort(); 79 | } 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServerMXBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.bookkeeper.proto; 20 | 21 | import org.apache.bookkeeper.proto.BKStats.OpStatData; 22 | 23 | /** 24 | * Bookie Server MBean 25 | */ 26 | public interface BookieServerMXBean { 27 | 28 | /** 29 | * @return packets received 30 | */ 31 | public long getNumPacketsReceived(); 32 | 33 | /** 34 | * @return packets sent 35 | */ 36 | public long getNumPacketsSent(); 37 | 38 | /** 39 | * @return add stats 40 | */ 41 | public OpStatData getAddStats(); 42 | 43 | /** 44 | * @return read stats 45 | */ 46 | public OpStatData getReadStats(); 47 | 48 | /** 49 | * @return server port 50 | */ 51 | public String getServerPort(); 52 | 53 | } 54 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/LocalBookiesRegistry.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.proto; 22 | 23 | import java.util.concurrent.ConcurrentHashMap; 24 | import org.apache.bookkeeper.net.BookieSocketAddress; 25 | 26 | /** 27 | * Local registry for embedded Bookies 28 | */ 29 | class LocalBookiesRegistry { 30 | 31 | private final static ConcurrentHashMap localBookiesRegistry 32 | = new ConcurrentHashMap<>(); 33 | 34 | static void registerLocalBookieAddress(BookieSocketAddress address) { 35 | localBookiesRegistry.put(address,Boolean.TRUE); 36 | } 37 | static void unregisterLocalBookieAddress(BookieSocketAddress address) { 38 | localBookiesRegistry.remove(address); 39 | } 40 | static boolean isLocalBookie(BookieSocketAddress address) { 41 | return localBookiesRegistry.containsKey(address); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClientFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.proto; 22 | 23 | import org.apache.bookkeeper.net.BookieSocketAddress; 24 | 25 | /** 26 | * Factory to create {@link org.apache.bookkeeper.proto.PerChannelBookieClient}. 27 | */ 28 | interface PerChannelBookieClientFactory { 29 | 30 | /** 31 | * Create a {@link org.apache.bookkeeper.proto.PerChannelBookieClient} to 32 | * address. 33 | * 34 | * @return the client connected to address. 35 | */ 36 | PerChannelBookieClient create(BookieSocketAddress address, PerChannelBookieClientPool pcbcPool); 37 | } 38 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClientPool.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.proto; 22 | 23 | import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.GenericCallback; 24 | 25 | /** 26 | * An interface to manage channel pooling for bookie client. 27 | */ 28 | interface PerChannelBookieClientPool { 29 | 30 | /** 31 | * intialize the pool. the implementation should not be blocked. 32 | */ 33 | void intialize(); 34 | 35 | /** 36 | * Obtain a channel from channel pool to execute operations. 37 | * 38 | * @param callback 39 | * callback to return channel from channel pool. 40 | */ 41 | void obtain(GenericCallback callback); 42 | 43 | /** 44 | * record any read/write error on {@link PerChannelBookieClientPool} 45 | */ 46 | void recordError(); 47 | 48 | /** 49 | * Disconnect the connections in the pool. 50 | * 51 | * @param wait 52 | * whether need to wait until pool disconnected. 53 | */ 54 | void disconnect(boolean wait); 55 | 56 | /** 57 | * Close the pool. 58 | * 59 | * @param wait 60 | * whether need to wait until pool closed. 61 | */ 62 | void close(boolean wait); 63 | 64 | } 65 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ResponseBuilder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.proto; 22 | 23 | import java.nio.ByteBuffer; 24 | import org.jboss.netty.buffer.ChannelBuffers; 25 | 26 | class ResponseBuilder { 27 | static BookieProtocol.Response buildErrorResponse(int errorCode, BookieProtocol.Request r) { 28 | if (r.getOpCode() == BookieProtocol.ADDENTRY) { 29 | return new BookieProtocol.AddResponse(r.getProtocolVersion(), errorCode, 30 | r.getLedgerId(), r.getEntryId()); 31 | } else { 32 | assert(r.getOpCode() == BookieProtocol.READENTRY); 33 | return new BookieProtocol.ReadResponse(r.getProtocolVersion(), errorCode, 34 | r.getLedgerId(), r.getEntryId()); 35 | } 36 | } 37 | 38 | static BookieProtocol.Response buildAddResponse(BookieProtocol.Request r) { 39 | return new BookieProtocol.AddResponse(r.getProtocolVersion(), BookieProtocol.EOK, r.getLedgerId(), 40 | r.getEntryId()); 41 | } 42 | 43 | static BookieProtocol.Response buildReadResponse(ByteBuffer data, BookieProtocol.Request r) { 44 | return new BookieProtocol.ReadResponse(r.getProtocolVersion(), BookieProtocol.EOK, 45 | r.getLedgerId(), r.getEntryId(), ChannelBuffers.wrappedBuffer(data)); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationEnableCb.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.replication; 22 | 23 | import java.util.concurrent.CountDownLatch; 24 | 25 | import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.GenericCallback; 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | 29 | /** 30 | * Callback which is getting notified when the replication process is enabled 31 | */ 32 | public class ReplicationEnableCb implements GenericCallback { 33 | 34 | private static final Logger LOG = LoggerFactory 35 | .getLogger(ReplicationEnableCb.class); 36 | private final CountDownLatch latch = new CountDownLatch(1); 37 | 38 | @Override 39 | public void operationComplete(int rc, Void result) { 40 | latch.countDown(); 41 | LOG.debug("Automatic ledger re-replication is enabled"); 42 | } 43 | 44 | /** 45 | * This is a blocking call and causes the current thread to wait until the 46 | * replication process is enabled 47 | * 48 | * @throws InterruptedException 49 | * interrupted while waiting 50 | */ 51 | public void await() throws InterruptedException { 52 | LOG.debug("Automatic ledger re-replication is disabled. " 53 | + "Hence waiting until its enabled!"); 54 | latch.await(); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.bookkeeper.replication; 20 | 21 | /** 22 | * Exceptions for use within the replication service 23 | */ 24 | public abstract class ReplicationException extends Exception { 25 | protected ReplicationException(String message, Throwable cause) { 26 | super(message, cause); 27 | } 28 | 29 | protected ReplicationException(String message) { 30 | super(message); 31 | } 32 | 33 | /** 34 | * The replication service has become unavailable 35 | */ 36 | public static class UnavailableException extends ReplicationException { 37 | private static final long serialVersionUID = 31872209L; 38 | 39 | public UnavailableException(String message, Throwable cause) { 40 | super(message, cause); 41 | } 42 | 43 | public UnavailableException(String message) { 44 | super(message); 45 | } 46 | } 47 | 48 | /** 49 | * Compatibility error. This version of the code, doesn't know how to 50 | * deal with the metadata it has found. 51 | */ 52 | public static class CompatibilityException extends ReplicationException { 53 | private static final long serialVersionUID = 98551903L; 54 | 55 | public CompatibilityException(String message, Throwable cause) { 56 | super(message, cause); 57 | } 58 | 59 | public CompatibilityException(String message) { 60 | super(message); 61 | } 62 | } 63 | 64 | /** 65 | * Exception while auditing bookie-ledgers 66 | */ 67 | static class BKAuditException extends ReplicationException { 68 | private static final long serialVersionUID = 95551905L; 69 | 70 | BKAuditException(String message, Throwable cause) { 71 | super(message, cause); 72 | } 73 | 74 | BKAuditException(String message) { 75 | super(message); 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationStats.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.replication; 22 | 23 | public interface ReplicationStats { 24 | 25 | public final static String REPLICATION_SCOPE = "replication"; 26 | 27 | public final static String AUDITOR_SCOPE = "auditor"; 28 | public final static String ELECTION_ATTEMPTS = "election_attempts"; 29 | 30 | public final static String REPLICATION_WORKER_SCOPE = "replication_worker"; 31 | public final static String REREPLICATE_OP = "rereplicate"; 32 | 33 | public final static String BK_CLIENT_SCOPE = "bk_client"; 34 | 35 | } 36 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.bookkeeper.util; 19 | 20 | /** 21 | * This class contains constants used in BookKeeper 22 | */ 23 | public class BookKeeperConstants { 24 | 25 | // ////////////////////////// 26 | // /////Basic constants////// 27 | // ////////////////////////// 28 | public static final String LEDGER_NODE_PREFIX = "L"; 29 | public static final String COLON = ":"; 30 | public static final String VERSION_FILENAME = "VERSION"; 31 | public final static String PASSWD = "passwd"; 32 | public static final String CURRENT_DIR = "current"; 33 | public static final String READONLY = "readonly"; 34 | 35 | // ////////////////////////// 36 | // ///// Znodes////////////// 37 | // ////////////////////////// 38 | public static final String AVAILABLE_NODE = "available"; 39 | public static final String COOKIE_NODE = "cookies"; 40 | public static final String UNDER_REPLICATION_NODE = "underreplication"; 41 | public static final String UNDER_REPLICATION_LOCK = "locks"; 42 | public static final String DISABLE_NODE = "disable"; 43 | public static final String DEFAULT_ZK_LEDGERS_ROOT_PATH = "/ledgers"; 44 | public static final String LAYOUT_ZNODE = "LAYOUT"; 45 | public static final String INSTANCEID = "INSTANCEID"; 46 | 47 | /** 48 | * Set the max log size limit to 1GB. It makes extra room for entry log file before 49 | * hitting hard limit '2GB'. So we don't need to force roll entry log file when flushing 50 | * memtable (for performance consideration) 51 | */ 52 | public static final long MAX_LOG_SIZE_LIMIT = 1 * 1024 * 1024 * 1024; 53 | } 54 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/util/DaemonThreadFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.bookkeeper.util; 20 | 21 | import java.util.concurrent.ThreadFactory; 22 | import java.util.concurrent.Executors; 23 | 24 | public class DaemonThreadFactory implements ThreadFactory { 25 | private ThreadFactory defaultThreadFactory = Executors.defaultThreadFactory(); 26 | private int priority = Thread.NORM_PRIORITY; 27 | public DaemonThreadFactory() { 28 | } 29 | public DaemonThreadFactory(int priority) { 30 | assert priority >= Thread.MIN_PRIORITY && priority <= Thread.MAX_PRIORITY; 31 | this.priority = priority; 32 | } 33 | public Thread newThread(Runnable r) { 34 | Thread thread = defaultThreadFactory.newThread(r); 35 | thread.setDaemon(true); 36 | thread.setPriority(priority); 37 | return thread; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/util/DirectMemoryUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.util; 22 | 23 | import static com.google.common.base.Preconditions.checkArgument; 24 | import static com.google.common.base.Preconditions.checkNotNull; 25 | 26 | import java.lang.reflect.Method; 27 | 28 | import org.slf4j.Logger; 29 | import org.slf4j.LoggerFactory; 30 | 31 | public class DirectMemoryUtils { 32 | /** 33 | * Returns the max configured size of direct memory for the JVM process. 34 | * 35 | * Direct memory can be specified with the flag -XX:MaxDirectMemorySize=8G on the command line. If not 36 | * specified, the default value will be set to the max size of the JVM heap. 37 | */ 38 | public static long maxDirectMemory() { 39 | try { 40 | 41 | Class VM = Class.forName("sun.misc.VM"); 42 | Method maxDirectMemory = VM.getDeclaredMethod("maxDirectMemory"); 43 | Object result = maxDirectMemory.invoke(null, (Object[]) null); 44 | 45 | checkNotNull(result); 46 | checkArgument(result instanceof Long); 47 | return (Long) result; 48 | } catch (Exception e) { 49 | if (LOG.isDebugEnabled()) { 50 | LOG.debug("Failed to get maxDirectMemory size from sun.misc.VM, falling back to max heap size", e); 51 | } 52 | return Runtime.getRuntime().maxMemory(); 53 | } 54 | } 55 | 56 | private static final Logger LOG = LoggerFactory.getLogger(DirectMemoryUtils.class); 57 | } 58 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/util/EntryFormatter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.util; 23 | 24 | import java.io.IOException; 25 | 26 | import org.apache.commons.configuration.Configuration; 27 | 28 | import org.slf4j.Logger; 29 | import org.slf4j.LoggerFactory; 30 | 31 | /** 32 | * Formatter to format an entry 33 | */ 34 | public abstract class EntryFormatter { 35 | 36 | private final static Logger LOG = LoggerFactory.getLogger(EntryFormatter.class); 37 | 38 | protected Configuration conf; 39 | 40 | public void setConf(Configuration conf) { 41 | this.conf = conf; 42 | } 43 | 44 | /** 45 | * Format an entry into a readable format 46 | * 47 | * @param data 48 | * Data Payload 49 | */ 50 | public abstract void formatEntry(byte[] data); 51 | 52 | /** 53 | * Format an entry from a string into a readable format 54 | * 55 | * @param input 56 | * Input Stream 57 | */ 58 | public abstract void formatEntry(java.io.InputStream input); 59 | 60 | public final static EntryFormatter STRING_FORMATTER = new StringEntryFormatter(); 61 | 62 | public static EntryFormatter newEntryFormatter(Configuration conf, String clsProperty) { 63 | String cls = conf.getString(clsProperty, StringEntryFormatter.class.getName()); 64 | ClassLoader classLoader = EntryFormatter.class.getClassLoader(); 65 | EntryFormatter formatter; 66 | try { 67 | Class aCls = classLoader.loadClass(cls); 68 | formatter = (EntryFormatter) aCls.newInstance(); 69 | formatter.setConf(conf); 70 | } catch (Exception e) { 71 | LOG.warn("No formatter class found : " + cls, e); 72 | LOG.warn("Using Default String Formatter."); 73 | formatter = STRING_FORMATTER; 74 | } 75 | return formatter; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/util/Main.java: -------------------------------------------------------------------------------- 1 | package org.apache.bookkeeper.util; 2 | 3 | /* 4 | * 5 | * Licensed to the Apache Software Foundation (ASF) under one 6 | * or more contributor license agreements. See the NOTICE file 7 | * distributed with this work for additional information 8 | * regarding copyright ownership. The ASF licenses this file 9 | * to you under the Apache License, Version 2.0 (the 10 | * "License"); you may not use this file except in compliance 11 | * with the License. You may obtain a copy of the License at 12 | * 13 | * http://www.apache.org/licenses/LICENSE-2.0 14 | * 15 | * Unless required by applicable law or agreed to in writing, 16 | * software distributed under the License is distributed on an 17 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 18 | * KIND, either express or implied. See the License for the 19 | * specific language governing permissions and limitations 20 | * under the License. 21 | * 22 | */ 23 | 24 | import java.io.IOException; 25 | 26 | import org.apache.bookkeeper.proto.BookieClient; 27 | import org.apache.bookkeeper.proto.BookieServer; 28 | 29 | public class Main { 30 | 31 | static void usage() { 32 | System.err.println("USAGE: bookeeper client|bookie"); 33 | } 34 | 35 | /** 36 | * @param args 37 | * @throws InterruptedException 38 | * @throws IOException 39 | */ 40 | public static void main(String[] args) throws Exception { 41 | if (args.length < 1 || !(args[0].equals("client") || args[0].equals("bookie"))) { 42 | usage(); 43 | return; 44 | } 45 | String newArgs[] = new String[args.length - 1]; 46 | System.arraycopy(args, 1, newArgs, 0, newArgs.length); 47 | if (args[0].equals("bookie")) { 48 | BookieServer.main(newArgs); 49 | } else { 50 | BookieClient.main(newArgs); 51 | } 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/util/SafeRunnable.java: -------------------------------------------------------------------------------- 1 | package org.apache.bookkeeper.util; 2 | 3 | /** 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | public abstract class SafeRunnable implements Runnable { 25 | 26 | static final Logger logger = LoggerFactory.getLogger(SafeRunnable.class); 27 | 28 | @Override 29 | public void run() { 30 | try { 31 | safeRun(); 32 | } catch(Throwable t) { 33 | logger.error("Unexpected throwable caught ", t); 34 | } 35 | } 36 | 37 | public abstract void safeRun(); 38 | 39 | } 40 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/util/StringEntryFormatter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.util; 23 | 24 | import java.io.IOException; 25 | 26 | import org.apache.commons.configuration.Configuration; 27 | 28 | import com.google.protobuf.ByteString; 29 | 30 | public class StringEntryFormatter extends EntryFormatter { 31 | @Override 32 | public void formatEntry(byte[] data) { 33 | System.out.println(ByteString.copyFrom(data).toStringUtf8()); 34 | } 35 | 36 | @Override 37 | public void formatEntry(java.io.InputStream input) { 38 | try { 39 | byte[] data = new byte[input.available()]; 40 | input.read(data, 0, data.length); 41 | formatEntry(data); 42 | } catch (IOException ie) { 43 | System.out.println("Warn: Unreadable entry : " + ie.getMessage()); 44 | } 45 | } 46 | 47 | }; 48 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/util/Tool.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.bookkeeper.util; 20 | 21 | import org.apache.commons.configuration.Configuration; 22 | 23 | /** 24 | * A tool interface that supports handling of generic command-line options. 25 | */ 26 | public interface Tool { 27 | /** 28 | * Exectue the command with given arguments 29 | * 30 | * @param args command specific arguments 31 | * @return exit code. 32 | * @throws Exception 33 | */ 34 | public int run(String[] args) throws Exception; 35 | 36 | /** 37 | * Passe a configuration object to the tool. 38 | * 39 | * @param conf configuration object 40 | * @throws Exception 41 | */ 42 | public void setConf(Configuration conf) throws Exception; 43 | } 44 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/util/ZeroBuffer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.util; 23 | 24 | import java.nio.ByteBuffer; 25 | 26 | /** 27 | * Zero buffer utility. 28 | * 29 | */ 30 | 31 | public class ZeroBuffer { 32 | static final byte zeroBytes[] = new byte[64*1024]; 33 | 34 | /** 35 | * fill zeros into given buffer 36 | * @param dst 37 | */ 38 | public static void put(ByteBuffer dst) { 39 | put(dst, dst.remaining()); 40 | } 41 | 42 | /** 43 | * fill zeros into given buffer up to given length 44 | * @param dst 45 | * @param length 46 | */ 47 | public static void put(ByteBuffer dst, int length) { 48 | while (length > zeroBytes.length) { 49 | dst.put(zeroBytes); 50 | length -= zeroBytes.length; 51 | } 52 | if (length > 0) { 53 | dst.put(zeroBytes, 0, length); 54 | } 55 | } 56 | 57 | /** 58 | * returns read-only zero-filled buffer, 59 | * @param length 60 | * @return ByteBuffer 61 | */ 62 | public static ByteBuffer readOnlyBuffer(int length) { 63 | ByteBuffer buffer; 64 | if (length <= zeroBytes.length) { 65 | buffer = ByteBuffer.wrap(zeroBytes, 0, length); 66 | } 67 | else { 68 | buffer = ByteBuffer.allocate(length); 69 | put(buffer); 70 | } 71 | return buffer.asReadOnlyBuffer(); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/versioning/Version.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.versioning; 22 | 23 | /** 24 | * An interface that allows us to determine if a given version happened before or after another version. 25 | */ 26 | public interface Version { 27 | 28 | /** 29 | * Initial version. 30 | */ 31 | public static final Version NEW = new Version() { 32 | @Override 33 | public Occurred compare(Version v) { 34 | if (null == v) { 35 | throw new NullPointerException("Version is not allowed to be null."); 36 | } 37 | if (this == v) { 38 | return Occurred.CONCURRENTLY; 39 | } 40 | return Occurred.BEFORE; 41 | } 42 | }; 43 | 44 | /** 45 | * Match any version. 46 | */ 47 | public static final Version ANY = new Version() { 48 | @Override 49 | public Occurred compare(Version v) { 50 | if (null == v) { 51 | throw new NullPointerException("Version is not allowed to be null."); 52 | } 53 | return Occurred.CONCURRENTLY; 54 | } 55 | }; 56 | 57 | public static enum Occurred { 58 | BEFORE, AFTER, CONCURRENTLY 59 | } 60 | 61 | public Occurred compare(Version v); 62 | } 63 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/versioning/Versioned.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.bookkeeper.versioning; 20 | 21 | public class Versioned { 22 | T value; 23 | Version version; 24 | 25 | public Versioned(T value, Version version) { 26 | this.value = value; 27 | this.version = version; 28 | } 29 | 30 | public void setValue(T value) { 31 | this.value = value; 32 | } 33 | 34 | public T getValue() { 35 | return value; 36 | } 37 | 38 | public void setVersion(Version version) { 39 | this.version = version; 40 | } 41 | 42 | public Version getVersion() { 43 | return version; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/zookeeper/BoundExponentialBackoffRetryPolicy.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.zookeeper; 22 | 23 | /** 24 | * Retry policy that retries a set number of times with an increasing (up to a 25 | * maximum bound) backoff time between retries. 26 | */ 27 | public class BoundExponentialBackoffRetryPolicy extends ExponentialBackoffRetryPolicy { 28 | 29 | private final long maxBackoffTime; 30 | 31 | public BoundExponentialBackoffRetryPolicy(long baseBackoffTime, long maxBackoffTime, int maxRetries) { 32 | super(baseBackoffTime, maxRetries); 33 | this.maxBackoffTime = maxBackoffTime; 34 | } 35 | 36 | @Override 37 | public long nextRetryWaitTime(int retryCount, long elapsedRetryTime) { 38 | return Math.min(maxBackoffTime, super.nextRetryWaitTime(retryCount, elapsedRetryTime)); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/zookeeper/ExponentialBackoffRetryPolicy.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.zookeeper; 22 | 23 | import java.util.Random; 24 | 25 | public class ExponentialBackoffRetryPolicy implements RetryPolicy { 26 | 27 | private final Random random; 28 | private final int maxRetries; 29 | private final long baseBackoffTime; 30 | 31 | public ExponentialBackoffRetryPolicy(long baseBackoffTime, int maxRetries) { 32 | this.maxRetries = maxRetries; 33 | this.baseBackoffTime = baseBackoffTime; 34 | this.random = new Random(System.currentTimeMillis()); 35 | } 36 | 37 | @Override 38 | public boolean allowRetry(int retryCount, long elapsedRetryTime) { 39 | return retryCount <= maxRetries; 40 | } 41 | 42 | @Override 43 | public long nextRetryWaitTime(int retryCount, long elapsedRetryTime) { 44 | return baseBackoffTime * Math.max(1, random.nextInt(Math.max(1, 1 << (retryCount + 1)))); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/java/org/apache/bookkeeper/zookeeper/RetryPolicy.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.zookeeper; 22 | 23 | /** 24 | * Interface of the policy to use when retrying operations. 25 | */ 26 | public interface RetryPolicy { 27 | 28 | /** 29 | * Called when retrying an operation failed for some reason. Return true if 30 | * another attempt is allowed to make. 31 | * 32 | * @param retryCount 33 | * The number of times retried so far (1 for the first time). 34 | * @param elapsedRetryTime 35 | * The elapsed time since the operation attempted. (in 36 | * milliseconds) 37 | * @return true if anther attempt is allowed to make. otherwise, false. 38 | */ 39 | public boolean allowRetry(int retryCount, long elapsedRetryTime); 40 | 41 | /** 42 | * Called before making an attempt to retry a failed operation. Return 0 if 43 | * an attempt needs to be made immediately. 44 | * 45 | * @param retryCount 46 | * The number of times retried so far (0 for the first time). 47 | * @param elapsedRetryTime 48 | * The elapsed time since the operation attempted. (in 49 | * milliseconds) 50 | * @return the elapsed time that the attempt needs to wait before retrying. 51 | * (in milliseconds) 52 | */ 53 | public long nextRetryWaitTime(int retryCount, long elapsedRetryTime); 54 | 55 | } 56 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/proto/DataFormats.proto: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | option java_package = "org.apache.bookkeeper.proto"; 19 | option optimize_for = SPEED; 20 | 21 | /** 22 | * Metadata format for storing ledger information 23 | */ 24 | message LedgerMetadataFormat { 25 | required int32 quorumSize = 1; 26 | required int32 ensembleSize = 2; 27 | required int64 length = 3; 28 | optional int64 lastEntryId = 4; 29 | 30 | enum State { 31 | OPEN = 1; 32 | IN_RECOVERY = 2; 33 | CLOSED = 3; 34 | } 35 | required State state = 5 [default = OPEN]; 36 | 37 | message Segment { 38 | repeated string ensembleMember = 1; 39 | required int64 firstEntryId = 2; 40 | } 41 | repeated Segment segment = 6; 42 | 43 | enum DigestType { 44 | CRC32 = 1; 45 | HMAC = 2; 46 | } 47 | optional DigestType digestType = 7; 48 | optional bytes password = 8; 49 | 50 | optional int32 ackQuorumSize = 9; 51 | 52 | optional int64 ctime = 10; 53 | } 54 | 55 | message LedgerRereplicationLayoutFormat { 56 | required string type = 1; 57 | required int32 version = 2; 58 | } 59 | 60 | message UnderreplicatedLedgerFormat { 61 | repeated string replica = 1; 62 | } 63 | 64 | /** 65 | * Cookie format for storing cookie information 66 | */ 67 | message CookieFormat { 68 | required string bookieHost = 1; 69 | required string journalDir = 2; 70 | required string ledgerDirs = 3; 71 | optional string instanceId = 4; 72 | } 73 | 74 | /** 75 | * Debug information for locks 76 | */ 77 | message LockDataFormat { 78 | optional string bookieId = 1; 79 | } 80 | 81 | /** 82 | * Debug information for auditor votes 83 | */ 84 | message AuditorVoteFormat { 85 | optional string bookieId = 1; 86 | } -------------------------------------------------------------------------------- /bookkeeper-server/src/main/resources/NOTICE.bin.txt: -------------------------------------------------------------------------------- 1 | Apache BookKeeper 2 | Copyright 2011-2015 The Apache Software Foundation 3 | 4 | Licensed under the Apache License, Version 2.0 (the 5 | "License"); you may not use this file except in compliance 6 | with the License. You may obtain a copy of the License at: 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, 11 | software distributed under the License is distributed on 12 | an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 13 | KIND, either express or implied. See the License for the 14 | specific language governing permissions and limitations 15 | under the License. 16 | 17 | This project includes: 18 | Apache Log4j under The Apache Software License, Version 2.0 19 | Commons BeanUtils Core under The Apache Software License, Version 2.0 20 | Commons CLI under The Apache Software License, Version 2.0 21 | Commons Codec under The Apache Software License, Version 2.0 22 | Commons Collections under The Apache Software License, Version 2.0 23 | Commons Configuration under The Apache Software License, Version 2.0 24 | Commons IO under The Apache Software License, Version 2.0 25 | Commons Lang under The Apache Software License, Version 2.0 26 | Commons Logging under The Apache Software License, Version 2.0 27 | commons Beanutils under Apache License, Version 2.0 28 | Commons Digester under The Apache Software License, Version 2.0 29 | Java Native Access under Apache License, Version 2.0 30 | JLine under BSD 31 | SLF4J API Module under MIT License 32 | SLF4J LOG4J-12 Binding under MIT License 33 | The Netty Project under Apache License, Version 2.0 34 | ZooKeeper under Apache License, Version 2.0 35 | Protocol Buffer Java API under New BSD license 36 | Guava under The Apache Software License, Version 2.0 37 | -------------------------------------------------------------------------------- /bookkeeper-server/src/main/resources/findbugsExclude.xml: -------------------------------------------------------------------------------- 1 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 30 | 31 | 32 | 33 | 34 | 36 | 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieAccessor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.bookie; 22 | 23 | import java.io.IOException; 24 | 25 | import org.apache.bookkeeper.bookie.CheckpointSource.Checkpoint; 26 | 27 | /** 28 | * Accessor class to avoid making Bookie internals public 29 | */ 30 | public class BookieAccessor { 31 | /** 32 | * Force a bookie to flush its ledger storage 33 | */ 34 | public static void forceFlush(Bookie b) throws IOException { 35 | Checkpoint cp = b.journal.newCheckpoint(); 36 | b.ledgerStorage.flush(); 37 | b.journal.checkpointComplete(cp, true); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperTestClient.java: -------------------------------------------------------------------------------- 1 | package org.apache.bookkeeper.client; 2 | 3 | /* 4 | * 5 | * Licensed to the Apache Software Foundation (ASF) under one 6 | * or more contributor license agreements. See the NOTICE file 7 | * distributed with this work for additional information 8 | * regarding copyright ownership. The ASF licenses this file 9 | * to you under the Apache License, Version 2.0 (the 10 | * "License"); you may not use this file except in compliance 11 | * with the License. You may obtain a copy of the License at 12 | * 13 | * http://www.apache.org/licenses/LICENSE-2.0 14 | * 15 | * Unless required by applicable law or agreed to in writing, 16 | * software distributed under the License is distributed on an 17 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 18 | * KIND, either express or implied. See the License for the 19 | * specific language governing permissions and limitations 20 | * under the License. 21 | * 22 | */ 23 | 24 | import java.io.IOException; 25 | import java.util.concurrent.Executors; 26 | 27 | import org.apache.bookkeeper.conf.ClientConfiguration; 28 | import org.apache.bookkeeper.client.AsyncCallback.CreateCallback; 29 | import org.apache.bookkeeper.client.AsyncCallback.DeleteCallback; 30 | import org.apache.bookkeeper.client.AsyncCallback.OpenCallback; 31 | import org.apache.bookkeeper.client.BKException.Code; 32 | import org.apache.bookkeeper.proto.BookieClient; 33 | import org.apache.bookkeeper.util.OrderedSafeExecutor; 34 | import org.slf4j.Logger; 35 | import org.slf4j.LoggerFactory; 36 | import org.apache.zookeeper.KeeperException; 37 | import org.apache.zookeeper.WatchedEvent; 38 | import org.apache.zookeeper.Watcher; 39 | import org.apache.zookeeper.ZooKeeper; 40 | import org.jboss.netty.channel.socket.ClientSocketChannelFactory; 41 | import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory; 42 | 43 | /** 44 | * Test BookKeeperClient which allows access to members we don't 45 | * wish to expose in the public API. 46 | */ 47 | public class BookKeeperTestClient extends BookKeeper { 48 | public BookKeeperTestClient(ClientConfiguration conf) 49 | throws IOException, InterruptedException, KeeperException { 50 | super(conf); 51 | } 52 | 53 | public ZooKeeper getZkHandle() { 54 | return super.getZkHandle(); 55 | } 56 | 57 | public ClientConfiguration getConf() { 58 | return super.getConf(); 59 | } 60 | 61 | /** 62 | * Force a read to zookeeper to get list of bookies. 63 | * 64 | * @throws InterruptedException 65 | * @throws KeeperException 66 | */ 67 | public void readBookiesBlocking() 68 | throws InterruptedException, KeeperException { 69 | bookieWatcher.readBookiesBlocking(); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ClientUtil.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.bookkeeper.client; 19 | 20 | import org.jboss.netty.buffer.ChannelBuffer; 21 | 22 | public class ClientUtil { 23 | public static ChannelBuffer generatePacket(long ledgerId, long entryId, long lastAddConfirmed, 24 | long length, byte[] data) { 25 | return generatePacket(ledgerId, entryId, lastAddConfirmed, length, data, 0, data.length); 26 | } 27 | 28 | public static ChannelBuffer generatePacket(long ledgerId, long entryId, long lastAddConfirmed, 29 | long length, byte[] data, int offset, int len) { 30 | CRC32DigestManager dm = new CRC32DigestManager(ledgerId); 31 | return dm.computeDigestAndPackageForSending(entryId, lastAddConfirmed, length, 32 | data, offset, len); 33 | } 34 | 35 | /** Returns that whether ledger is in open state */ 36 | public static boolean isLedgerOpen(LedgerHandle handle) { 37 | return !handle.metadata.isClosed(); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /bookkeeper-server/src/test/java/org/apache/bookkeeper/client/LedgerHandleAdapter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | * 19 | */ 20 | package org.apache.bookkeeper.client; 21 | 22 | /** 23 | * Adapter for tests to get the public access from LedgerHandle for its default 24 | * scope 25 | */ 26 | public class LedgerHandleAdapter { 27 | 28 | /** get the ledger handle */ 29 | public static LedgerMetadata getLedgerMetadata(LedgerHandle lh) { 30 | return lh.getLedgerMetadata(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /bookkeeper-server/src/test/java/org/apache/bookkeeper/client/LocalBookKeeperTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.client; 23 | 24 | import java.util.concurrent.CountDownLatch; 25 | 26 | import org.apache.bookkeeper.conf.ClientConfiguration; 27 | import org.apache.bookkeeper.client.BookKeeper.DigestType; 28 | import org.apache.bookkeeper.conf.ServerConfiguration; 29 | import org.apache.bookkeeper.test.BaseTestCase; 30 | import org.junit.Test; 31 | 32 | /** 33 | * Tests of the main BookKeeper client using networkless comunication 34 | */ 35 | public class LocalBookKeeperTest extends BaseTestCase { 36 | 37 | protected ServerConfiguration newServerConfiguration() throws Exception { 38 | return super 39 | .newServerConfiguration() 40 | .setEnableLocalTransport(true); 41 | } 42 | 43 | DigestType digestType; 44 | 45 | public LocalBookKeeperTest(DigestType digestType) { 46 | super(4); 47 | this.digestType=digestType; 48 | } 49 | 50 | @Test 51 | public void testUseLocalBookie() throws Exception { 52 | ClientConfiguration conf = new ClientConfiguration() 53 | .setZkServers(zkUtil.getZooKeeperConnectString()) 54 | .setZkTimeout(20000); 55 | 56 | CountDownLatch l = new CountDownLatch(1); 57 | zkUtil.sleepServer(5, l); 58 | l.await(); 59 | 60 | BookKeeper bkc = new BookKeeper(conf); 61 | LedgerHandle h = bkc.createLedger(1,1,digestType, "testPasswd".getBytes()); 62 | h.addEntry("test".getBytes()); 63 | h.close(); 64 | bkc.close(); 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/TestBKConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.conf; 23 | 24 | public class TestBKConfiguration { 25 | 26 | public static ServerConfiguration newServerConfiguration() { 27 | ServerConfiguration confReturn = new ServerConfiguration(); 28 | confReturn.setJournalFlushWhenQueueEmpty(true); 29 | // enable journal format version 30 | confReturn.setJournalFormatVersionToWrite(5); 31 | confReturn.setAllowLoopback(true); 32 | confReturn.setGcWaitTime(1000); 33 | return confReturn; 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/LedgerManagerIteratorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.meta; 23 | 24 | import org.apache.bookkeeper.meta.LedgerManager.LedgerRangeIterator; 25 | import org.junit.Test; 26 | 27 | import static org.junit.Assert.*; 28 | 29 | public class LedgerManagerIteratorTest extends LedgerManagerTestCase { 30 | public LedgerManagerIteratorTest(Class lmFactoryCls) { 31 | super(lmFactoryCls); 32 | } 33 | 34 | @Test(timeout = 60000) 35 | public void testIterateNoLedgers() throws Exception { 36 | LedgerManager lm = getLedgerManager(); 37 | LedgerRangeIterator lri = lm.getLedgerRanges(); 38 | assertNotNull(lri); 39 | if (lri.hasNext()) 40 | lri.next(); 41 | 42 | assertEquals(false, lri.hasNext()); 43 | assertEquals(false, lri.hasNext()); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/TestZkVersion.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.meta; 22 | 23 | import org.junit.Test; 24 | import org.junit.Assert; 25 | 26 | import org.apache.bookkeeper.versioning.Version; 27 | import org.apache.bookkeeper.versioning.Version.Occurred; 28 | 29 | public class TestZkVersion { 30 | 31 | @Test(timeout=60000) 32 | public void testNullZkVersion() { 33 | ZkVersion zkVersion = new ZkVersion(99); 34 | try { 35 | zkVersion.compare(null); 36 | Assert.fail("Should fail comparing with null version."); 37 | } catch (NullPointerException npe) { 38 | } 39 | } 40 | 41 | @Test(timeout=60000) 42 | public void testInvalidVersion() { 43 | ZkVersion zkVersion = new ZkVersion(99); 44 | try { 45 | zkVersion.compare(new Version() { 46 | @Override 47 | public Occurred compare(Version v) { 48 | return Occurred.AFTER; 49 | } 50 | }); 51 | Assert.fail("Should not reach here!"); 52 | } catch (IllegalArgumentException iae) { 53 | } 54 | } 55 | 56 | @Test(timeout=60000) 57 | public void testCompare() { 58 | ZkVersion zv = new ZkVersion(99); 59 | Assert.assertEquals(Occurred.AFTER, zv.compare(new ZkVersion(98))); 60 | Assert.assertEquals(Occurred.BEFORE, zv.compare(new ZkVersion(100))); 61 | Assert.assertEquals(Occurred.CONCURRENTLY, zv.compare(new ZkVersion(99))); 62 | Assert.assertEquals(Occurred.CONCURRENTLY, zv.compare(Version.ANY)); 63 | Assert.assertEquals(Occurred.AFTER, zv.compare(Version.NEW)); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /bookkeeper-server/src/test/java/org/apache/bookkeeper/metastore/MetastoreScannableTableAsyncToSyncConverter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.bookkeeper.metastore; 19 | 20 | import java.util.Set; 21 | 22 | import org.apache.bookkeeper.metastore.MetastoreScannableTable.Order; 23 | 24 | public class MetastoreScannableTableAsyncToSyncConverter extends 25 | MetastoreTableAsyncToSyncConverter { 26 | 27 | private MetastoreScannableTable scannableTable; 28 | 29 | public MetastoreScannableTableAsyncToSyncConverter( 30 | MetastoreScannableTable table) { 31 | super(table); 32 | this.scannableTable = table; 33 | } 34 | 35 | public MetastoreCursor openCursor(String firstKey, boolean firstInclusive, 36 | String lastKey, boolean lastInclusive, 37 | Order order) 38 | throws MSException { 39 | HeldValue retValue = new HeldValue(); 40 | // make the actual async call 41 | this.scannableTable.openCursor(firstKey, firstInclusive, lastKey, lastInclusive, 42 | order, retValue, null); 43 | retValue.waitCallback(); 44 | return retValue.getValue(); 45 | } 46 | 47 | public MetastoreCursor openCursor(String firstKey, boolean firstInclusive, 48 | String lastKey, boolean lastInclusive, 49 | Order order, Set fields) 50 | throws MSException { 51 | HeldValue retValue = new HeldValue(); 52 | // make the actual async call 53 | this.scannableTable.openCursor(firstKey, firstInclusive, lastKey, lastInclusive, 54 | order, fields, retValue, null); 55 | retValue.waitCallback(); 56 | return retValue.getValue(); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/TestBKStats.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.proto; 22 | 23 | import static org.junit.Assert.assertEquals; 24 | 25 | import org.apache.bookkeeper.proto.BKStats.OpStats; 26 | import org.junit.Test; 27 | 28 | /** Tests that Statistics updation in Bookie Server */ 29 | public class TestBKStats { 30 | 31 | /** 32 | * Tests that updatLatency should not fail with 33 | * ArrayIndexOutOfBoundException when latency time coming as negative. 34 | */ 35 | @Test(timeout=60000) 36 | public void testUpdateLatencyShouldNotFailWithAIOBEWithNegativeLatency() 37 | throws Exception { 38 | OpStats opStat = new OpStats(); 39 | opStat.updateLatency(-10); 40 | assertEquals("Should not update any latency metrics", 0, 41 | opStat.numSuccessOps); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/TestDeathwatcher.java: -------------------------------------------------------------------------------- 1 | package org.apache.bookkeeper.proto; 2 | 3 | /* 4 | * 5 | * Licensed to the Apache Software Foundation (ASF) under one 6 | * or more contributor license agreements. See the NOTICE file 7 | * distributed with this work for additional information 8 | * regarding copyright ownership. The ASF licenses this file 9 | * to you under the Apache License, Version 2.0 (the 10 | * "License"); you may not use this file except in compliance 11 | * with the License. You may obtain a copy of the License at 12 | * 13 | * http://www.apache.org/licenses/LICENSE-2.0 14 | * 15 | * Unless required by applicable law or agreed to in writing, 16 | * software distributed under the License is distributed on an 17 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 18 | * KIND, either express or implied. See the License for the 19 | * specific language governing permissions and limitations 20 | * under the License. 21 | * 22 | */ 23 | 24 | import org.junit.*; 25 | 26 | import org.apache.bookkeeper.test.BookKeeperClusterTestCase; 27 | import org.apache.bookkeeper.conf.ServerConfiguration; 28 | 29 | import org.slf4j.Logger; 30 | import org.slf4j.LoggerFactory; 31 | 32 | import static org.junit.Assert.*; 33 | 34 | /** 35 | * Tests for the BookieServer death watcher 36 | */ 37 | public class TestDeathwatcher extends BookKeeperClusterTestCase { 38 | private final static Logger LOG = LoggerFactory.getLogger(TestDeathwatcher.class); 39 | 40 | public TestDeathwatcher() { 41 | super(1); 42 | } 43 | 44 | /** 45 | * Ensure that if the autorecovery daemon is running inside the bookie 46 | * then a failure/crash in the autorecovery daemon will not take down the 47 | * bookie also. 48 | */ 49 | @Test(timeout=30000) 50 | public void testAutorecoveryFailureDoesntKillBookie() throws Exception { 51 | ServerConfiguration conf = newServerConfiguration().setAutoRecoveryDaemonEnabled(true); 52 | BookieServer bs = startBookie(conf); 53 | 54 | assertNotNull("Autorecovery daemon should exist", bs.autoRecoveryMain); 55 | assertTrue("Bookie should be running", bs.isBookieRunning()); 56 | bs.autoRecoveryMain.shutdown(); 57 | Thread.sleep(conf.getDeathWatchInterval()*2); // give deathwatcher time to run 58 | assertTrue("Bookie should be running", bs.isBookieRunning()); 59 | bs.shutdown(); 60 | } 61 | } 62 | 63 | -------------------------------------------------------------------------------- /bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/ReplicationTestUtil.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.replication; 22 | 23 | import java.util.List; 24 | import org.apache.zookeeper.KeeperException; 25 | import org.apache.zookeeper.ZooKeeper; 26 | 27 | /** Utility class for replication tests */ 28 | public class ReplicationTestUtil { 29 | 30 | /** Checks whether ledger is in under-replication */ 31 | static boolean isLedgerInUnderReplication(ZooKeeper zkc, long id, 32 | String basePath) throws KeeperException, InterruptedException { 33 | List children; 34 | try { 35 | children = zkc.getChildren(basePath, true); 36 | } catch (KeeperException.NoNodeException nne) { 37 | return false; 38 | } 39 | 40 | boolean isMatched = false; 41 | for (String child : children) { 42 | if (child.startsWith("urL") && child.contains(String.valueOf(id))) { 43 | isMatched = true; 44 | break; 45 | } else { 46 | String path = basePath + '/' + child; 47 | try { 48 | if (zkc.getChildren(path, false).size() > 0) { 49 | isMatched = isLedgerInUnderReplication(zkc, id, path); 50 | } 51 | } catch (KeeperException.NoNodeException nne) { 52 | return false; 53 | } 54 | } 55 | 56 | } 57 | return isMatched; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BaseTestCase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.test; 23 | 24 | import java.util.Arrays; 25 | import java.util.Collection; 26 | 27 | import org.apache.bookkeeper.client.BookKeeper.DigestType; 28 | 29 | import org.slf4j.Logger; 30 | import org.slf4j.LoggerFactory; 31 | 32 | import org.junit.runner.RunWith; 33 | import org.junit.runners.Parameterized; 34 | import org.junit.runners.Parameterized.Parameters; 35 | 36 | @RunWith(Parameterized.class) 37 | public abstract class BaseTestCase extends BookKeeperClusterTestCase { 38 | static final Logger LOG = LoggerFactory.getLogger(BaseTestCase.class); 39 | 40 | public BaseTestCase(int numBookies) { 41 | super(numBookies); 42 | } 43 | 44 | @Parameters 45 | public static Collection configs() { 46 | return Arrays.asList(new Object[][] { {DigestType.MAC }, {DigestType.CRC32}}); 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /bookkeeper-server/src/test/java/org/apache/bookkeeper/test/MultiLedgerManagerMultiDigestTestCase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.test; 23 | 24 | import java.util.ArrayList; 25 | import java.util.Collection; 26 | 27 | import org.apache.bookkeeper.client.BookKeeper.DigestType; 28 | 29 | import org.slf4j.Logger; 30 | import org.slf4j.LoggerFactory; 31 | 32 | import org.junit.runner.RunWith; 33 | import org.junit.runners.Parameterized; 34 | import org.junit.runners.Parameterized.Parameters; 35 | 36 | /** 37 | * Running test case using different ledger managers. 38 | */ 39 | @RunWith(Parameterized.class) 40 | public abstract class MultiLedgerManagerMultiDigestTestCase extends BookKeeperClusterTestCase { 41 | static final Logger LOG = LoggerFactory.getLogger(MultiLedgerManagerMultiDigestTestCase.class); 42 | 43 | public MultiLedgerManagerMultiDigestTestCase(int numBookies) { 44 | super(numBookies); 45 | } 46 | 47 | @Parameters 48 | public static Collection configs() { 49 | String[] ledgerManagers = { 50 | "org.apache.bookkeeper.meta.FlatLedgerManagerFactory", 51 | "org.apache.bookkeeper.meta.HierarchicalLedgerManagerFactory", 52 | "org.apache.bookkeeper.meta.MSLedgerManagerFactory", 53 | }; 54 | ArrayList cfgs = new ArrayList(ledgerManagers.length); 55 | DigestType[] digestTypes = new DigestType[] { DigestType.MAC, DigestType.CRC32 }; 56 | for (String lm : ledgerManagers) { 57 | for (DigestType type : digestTypes) { 58 | cfgs.add(new Object[] { lm, type }); 59 | } 60 | } 61 | return cfgs; 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /bookkeeper-server/src/test/java/org/apache/bookkeeper/test/MultiLedgerManagerTestCase.java: -------------------------------------------------------------------------------- 1 | package org.apache.bookkeeper.test; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Collection; 5 | 6 | import org.junit.runner.RunWith; 7 | import org.junit.runners.Parameterized; 8 | import org.junit.runners.Parameterized.Parameters; 9 | 10 | /* 11 | * 12 | * Licensed to the Apache Software Foundation (ASF) under one 13 | * or more contributor license agreements. See the NOTICE file 14 | * distributed with this work for additional information 15 | * regarding copyright ownership. The ASF licenses this file 16 | * to you under the Apache License, Version 2.0 (the 17 | * "License"); you may not use this file except in compliance 18 | * with the License. You may obtain a copy of the License at 19 | * 20 | * http://www.apache.org/licenses/LICENSE-2.0 21 | * 22 | * Unless required by applicable law or agreed to in writing, 23 | * software distributed under the License is distributed on an 24 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 25 | * KIND, either express or implied. See the License for the 26 | * specific language governing permissions and limitations 27 | * under the License. 28 | * 29 | */ 30 | 31 | import org.apache.bookkeeper.meta.LedgerManagerFactory; 32 | 33 | /** 34 | * Test Case run over different ledger manager. 35 | */ 36 | @RunWith(Parameterized.class) 37 | public abstract class MultiLedgerManagerTestCase extends BookKeeperClusterTestCase { 38 | 39 | public MultiLedgerManagerTestCase(int numBookies) { 40 | super(numBookies); 41 | } 42 | 43 | @Parameters 44 | public static Collection configs() { 45 | String[] ledgerManagers = new String[] { 46 | "org.apache.bookkeeper.meta.FlatLedgerManagerFactory", 47 | "org.apache.bookkeeper.meta.HierarchicalLedgerManagerFactory", 48 | "org.apache.bookkeeper.meta.MSLedgerManagerFactory", 49 | }; 50 | ArrayList cfgs = new ArrayList(ledgerManagers.length); 51 | for (String lm : ledgerManagers) { 52 | cfgs.add(new Object[] { lm }); 53 | } 54 | return cfgs; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /bookkeeper-server/src/test/java/org/apache/bookkeeper/test/PortManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | package org.apache.bookkeeper.test; 22 | 23 | import java.net.ServerSocket; 24 | import java.io.IOException; 25 | /** 26 | * Port manager allows a base port to be specified on the commandline. 27 | * Tests will then use ports, counting up from this base port. 28 | * This allows multiple instances of the bookkeeper tests to run at once. 29 | */ 30 | public class PortManager { 31 | private static int nextPort = getBasePort(); 32 | 33 | public synchronized static int nextFreePort() { 34 | while (true) { 35 | ServerSocket ss = null; 36 | try { 37 | int port = nextPort++; 38 | ss = new ServerSocket(port); 39 | ss.setReuseAddress(true); 40 | return port; 41 | } catch (IOException ioe) { 42 | } finally { 43 | if (ss != null) { 44 | try { 45 | ss.close(); 46 | } catch (IOException ioe) {} 47 | } 48 | } 49 | } 50 | } 51 | 52 | private static int getBasePort() { 53 | return Integer.valueOf(System.getProperty("test.basePort", "15000")); 54 | } 55 | } -------------------------------------------------------------------------------- /bookkeeper-server/src/test/java/org/apache/bookkeeper/util/TestUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | package org.apache.bookkeeper.util; 23 | 24 | import java.io.File; 25 | import java.util.HashSet; 26 | import java.util.Set; 27 | 28 | import org.apache.bookkeeper.bookie.Bookie; 29 | 30 | public class TestUtils { 31 | public static boolean hasLogFiles(File ledgerDirectory, boolean partial, Integer... logsId) { 32 | boolean result = partial ? false : true; 33 | Set logs = new HashSet(); 34 | for (File file : Bookie.getCurrentDirectory(ledgerDirectory).listFiles()) { 35 | if (file.isFile()) { 36 | String name = file.getName(); 37 | if (!name.endsWith(".log")) { 38 | continue; 39 | } 40 | logs.add(Integer.parseInt(name.split("\\.")[0], 16)); 41 | } 42 | } 43 | for (Integer logId : logsId) { 44 | boolean exist = logs.contains(logId); 45 | if ((partial && exist) 46 | || (!partial && !exist)) { 47 | return !result; 48 | } 49 | } 50 | return result; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /bookkeeper-server/src/test/proto/TestDataFormats.proto: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | option java_package = "org.apache.bookkeeper.proto"; 19 | option optimize_for = SPEED; 20 | 21 | import "src/main/proto/BookkeeperProtocol.proto"; 22 | 23 | enum AuthMessageType { 24 | SUCCESS_RESPONSE = 1; 25 | FAILURE_RESPONSE = 2; 26 | PAYLOAD_MESSAGE = 3; 27 | } 28 | 29 | /** 30 | * 31 | */ 32 | extend AuthMessage { 33 | required AuthMessageType messageType = 1000; 34 | } 35 | -------------------------------------------------------------------------------- /bookkeeper-stats-providers/codahale-metrics-provider/src/test/java/org/apache/bookkeeper/stats/CodahaleOpStatsTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with this 4 | * work for additional information regarding copyright ownership. The ASF 5 | * licenses this file to you under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | package org.apache.bookkeeper.stats; 18 | 19 | import static org.junit.Assert.*; 20 | 21 | public class CodahaleOpStatsTest { 22 | 23 | @org.junit.Test 24 | public void testToOpStatsData() { 25 | OpStatsLogger logger = new CodahaleMetricsProvider().getStatsLogger("test").getOpStatsLogger("testLogger"); 26 | logger.registerSuccessfulValue(1); 27 | // the following should not throw any exception 28 | OpStatsData statsData = logger.toOpStatsData(); 29 | assertEquals(1, statsData.getNumSuccessfulEvents()); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /bookkeeper-stats-providers/pom.xml: -------------------------------------------------------------------------------- 1 | 17 | 18 | 22 | 23 | bookkeeper 24 | org.apache.bookkeeper 25 | 4.5.0-SNAPSHOT 26 | 27 | 4.0.0 28 | org.apache.bookkeeper 29 | 4.5.0-SNAPSHOT 30 | bookkeeper-stats-providers 31 | pom 32 | bookkeeper-stats-providers 33 | 34 | twitter-science-provider 35 | twitter-ostrich-provider 36 | codahale-metrics-provider 37 | 38 | 39 | UTF-8 40 | UTF-8 41 | 42 | 43 | -------------------------------------------------------------------------------- /bookkeeper-stats-providers/twitter-ostrich-provider/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 20 | 4.0.0 21 | 22 | bookkeeper 23 | org.apache.bookkeeper 24 | 4.5.0-SNAPSHOT 25 | ../.. 26 | 27 | org.apache.bookkeeper.stats 28 | twitter-ostrich-provider 29 | Stats provider for twitter-ostrich package 30 | http://maven.apache.org 31 | 32 | UTF-8 33 | ${basedir}/lib 34 | 35 | 36 | 37 | org.apache.bookkeeper.stats 38 | bookkeeper-stats-api 39 | ${project.parent.version} 40 | 41 | 42 | com.twitter 43 | ostrich_2.9.2 44 | 9.1.3 45 | 46 | 47 | 48 | 49 | twitter 50 | Twitter repo 51 | default 52 | http://maven.twttr.com 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/CounterImpl.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with this 4 | * work for additional information regarding copyright ownership. The ASF 5 | * licenses this file to you under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | package org.apache.bookkeeper.stats.twitter.ostrich; 18 | 19 | import org.apache.bookkeeper.stats.Counter; 20 | 21 | class CounterImpl implements Counter { 22 | 23 | private final com.twitter.ostrich.stats.Counter ostrichCounter; 24 | 25 | CounterImpl(com.twitter.ostrich.stats.Counter ostrichCounter) { 26 | this.ostrichCounter = ostrichCounter; 27 | } 28 | 29 | @Override 30 | public void clear() { 31 | this.ostrichCounter.reset(); 32 | } 33 | 34 | @Override 35 | public void inc() { 36 | this.ostrichCounter.incr(); 37 | } 38 | 39 | @Override 40 | public void dec() { 41 | this.ostrichCounter.incr(-1); 42 | } 43 | 44 | @Override 45 | public void add(long delta) { 46 | this.ostrichCounter.incr((int)delta); 47 | } 48 | 49 | @Override 50 | public Long get() { 51 | return this.ostrichCounter.apply(); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/OstrichStatsLoggerImpl.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with this 4 | * work for additional information regarding copyright ownership. The ASF 5 | * licenses this file to you under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | package org.apache.bookkeeper.stats.twitter.ostrich; 18 | 19 | import org.apache.bookkeeper.stats.Counter; 20 | import org.apache.bookkeeper.stats.Gauge; 21 | import org.apache.bookkeeper.stats.OpStatsLogger; 22 | import org.apache.bookkeeper.stats.StatsLogger; 23 | import scala.Function0; 24 | import scala.runtime.AbstractFunction0; 25 | 26 | /** 27 | * Implementation of ostrich logger. 28 | */ 29 | class OstrichStatsLoggerImpl implements StatsLogger { 30 | 31 | protected final String scope; 32 | protected final com.twitter.ostrich.stats.StatsProvider ostrichProvider; 33 | 34 | OstrichStatsLoggerImpl(String scope, com.twitter.ostrich.stats.StatsProvider ostrichProvider) { 35 | this.scope = scope; 36 | this.ostrichProvider = ostrichProvider; 37 | } 38 | 39 | @Override 40 | public OpStatsLogger getOpStatsLogger(String statName) { 41 | return new OpStatsLoggerImpl(getStatName(statName), ostrichProvider); 42 | } 43 | 44 | @Override 45 | public Counter getCounter(String statName) { 46 | return new CounterImpl(ostrichProvider.getCounter(getStatName(statName))); 47 | } 48 | 49 | @Override 50 | public void registerGauge(final String statName, final Gauge gauge) { 51 | Function0 gaugeFunc = new AbstractFunction0() { 52 | @Override 53 | public Object apply() { 54 | return gauge.getSample().doubleValue(); 55 | } 56 | }; 57 | ostrichProvider.addGauge(getStatName(statName), gaugeFunc); 58 | } 59 | 60 | private String getStatName(String statName) { 61 | return String.format("%s/%s", scope, statName); 62 | } 63 | 64 | @Override 65 | public StatsLogger scope(String scope) { 66 | return new OstrichStatsLoggerImpl(getStatName(scope), ostrichProvider); 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/CounterImpl.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with this 4 | * work for additional information regarding copyright ownership. The ASF 5 | * licenses this file to you under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | package org.apache.bookkeeper.stats.twitter.science; 18 | 19 | import com.twitter.common.stats.Rate; 20 | import com.twitter.common.stats.Stats; 21 | import org.apache.bookkeeper.stats.Counter; 22 | 23 | import java.util.concurrent.atomic.AtomicLong; 24 | /** 25 | * This will export the value and the rate (per sec) to {@link org.apache.bookkeeper.stats.Stats} 26 | */ 27 | public class CounterImpl implements Counter { 28 | // The name used to export this stat 29 | private String name; 30 | private AtomicLong value; 31 | 32 | public CounterImpl(String name) { 33 | this.name = name; 34 | value = new AtomicLong(0); 35 | setUpStatsExport(); 36 | } 37 | 38 | @Override 39 | public synchronized void clear() { 40 | value.getAndSet(0); 41 | } 42 | 43 | @Override 44 | public Long get() { 45 | return value.get(); 46 | } 47 | 48 | private void setUpStatsExport() { 49 | // Export the value. 50 | Stats.export(name, value); 51 | // Export the rate of this value. 52 | Stats.export(Rate.of(name + "_per_sec", value).build()); 53 | } 54 | 55 | @Override 56 | public void inc() { 57 | value.incrementAndGet(); 58 | } 59 | 60 | @Override 61 | public void dec() { 62 | value.decrementAndGet(); 63 | } 64 | 65 | @Override 66 | public void add(long delta) { 67 | value.addAndGet(delta); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/TwitterStatsLoggerImpl.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with this 4 | * work for additional information regarding copyright ownership. The ASF 5 | * licenses this file to you under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | package org.apache.bookkeeper.stats.twitter.science; 18 | 19 | import com.twitter.common.stats.SampledStat; 20 | import com.twitter.common.stats.Stats; 21 | import org.apache.bookkeeper.stats.Counter; 22 | import org.apache.bookkeeper.stats.Gauge; 23 | import org.apache.bookkeeper.stats.OpStatsLogger; 24 | import org.apache.bookkeeper.stats.StatsLogger; 25 | 26 | /** 27 | * Implementation of twitter-stats logger. 28 | */ 29 | public class TwitterStatsLoggerImpl implements StatsLogger { 30 | 31 | protected final String name; 32 | 33 | public TwitterStatsLoggerImpl(String name) { 34 | this.name = name; 35 | } 36 | 37 | @Override 38 | public OpStatsLogger getOpStatsLogger(String statName) { 39 | return new OpStatsLoggerImpl(getStatName(statName)); 40 | } 41 | 42 | @Override 43 | public Counter getCounter(String statName) { 44 | return new CounterImpl(getStatName(statName)); 45 | } 46 | 47 | @Override 48 | public void registerGauge(final String statName, final Gauge gauge) { 49 | Stats.export(new SampledStat(getStatName(statName), gauge.getDefaultValue()) { 50 | @Override 51 | public T doSample() { 52 | return gauge.getSample(); 53 | } 54 | }); 55 | } 56 | 57 | private String getStatName(String statName) { 58 | return (name + "_" + statName).toLowerCase(); 59 | } 60 | 61 | @Override 62 | public StatsLogger scope(String scope) { 63 | String scopeName; 64 | if (0 == name.length()) { 65 | scopeName = scope; 66 | } else { 67 | scopeName = name + "_" + scope; 68 | } 69 | return new TwitterStatsLoggerImpl(scopeName); 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/TwitterStatsProvider.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with this 4 | * work for additional information regarding copyright ownership. The ASF 5 | * licenses this file to you under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | package org.apache.bookkeeper.stats.twitter.science; 18 | 19 | import org.apache.bookkeeper.stats.StatsLogger; 20 | import org.apache.bookkeeper.stats.StatsProvider; 21 | import org.apache.commons.configuration.Configuration; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | public class TwitterStatsProvider implements StatsProvider { 26 | 27 | static final Logger LOG = LoggerFactory.getLogger(TwitterStatsProvider.class); 28 | 29 | protected final static String STATS_EXPORT = "statsExport"; 30 | protected final static String STATS_HTTP_PORT = "statsHttpPort"; 31 | 32 | private HTTPStatsExporter statsExporter = null; 33 | 34 | @Override 35 | public void start(Configuration conf) { 36 | if (conf.getBoolean(STATS_EXPORT, false)) { 37 | statsExporter = new HTTPStatsExporter(conf.getInt(STATS_HTTP_PORT, 9002)); 38 | } 39 | if (null != statsExporter) { 40 | try { 41 | statsExporter.start(); 42 | } catch (Exception e) { 43 | LOG.error("Fail to start stats exporter : ", e); 44 | } 45 | } 46 | } 47 | 48 | @Override 49 | public void stop() { 50 | if (null != statsExporter) { 51 | try { 52 | statsExporter.stop(); 53 | } catch (Exception e) { 54 | LOG.error("Fail to stop stats exporter : ", e); 55 | } 56 | } 57 | } 58 | 59 | @Override 60 | public StatsLogger getStatsLogger(String name) { 61 | return new TwitterStatsLoggerImpl(name); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /bookkeeper-stats/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 20 | 4.0.0 21 | 22 | bookkeeper 23 | org.apache.bookkeeper 24 | 4.5.0-SNAPSHOT 25 | 26 | org.apache.bookkeeper.stats 27 | bookkeeper-stats-api 28 | Stats API for bookkeeper 29 | http://maven.apache.org 30 | 31 | UTF-8 32 | ${basedir}/lib 33 | 34 | 35 | 36 | commons-configuration 37 | commons-configuration 38 | 1.6 39 | 40 | 41 | junit 42 | junit 43 | 4.8.1 44 | test 45 | 46 | 47 | org.slf4j 48 | slf4j-api 49 | 1.6.4 50 | 51 | 52 | org.slf4j 53 | slf4j-log4j12 54 | 1.6.4 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/Counter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with this 4 | * work for additional information regarding copyright ownership. The ASF 5 | * licenses this file to you under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | package org.apache.bookkeeper.stats; 18 | 19 | /** 20 | * Simple stats that require only increment and decrement 21 | * functions on a Long. Metrics like the number of topics, persist queue size 22 | * etc. should use this. 23 | */ 24 | public interface Counter { 25 | /** 26 | * Clear this stat. 27 | */ 28 | public void clear(); 29 | 30 | /** 31 | * Increment the value associated with this stat. 32 | */ 33 | public void inc(); 34 | 35 | /** 36 | * Decrement the value associated with this stat. 37 | */ 38 | public void dec(); 39 | 40 | /** 41 | * Add delta to the value associated with this stat. 42 | * @param delta 43 | */ 44 | public void add(long delta); 45 | 46 | /** 47 | * Get the value associated with this stat. 48 | */ 49 | public Long get(); 50 | } 51 | -------------------------------------------------------------------------------- /bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/Gauge.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.bookkeeper.stats; 19 | 20 | /** 21 | * A guage is a value that has only one value at a specific point in time. 22 | * An example is the number of elements in a queue. The value of T must be 23 | * some numeric type. 24 | */ 25 | public interface Gauge { 26 | public T getDefaultValue(); 27 | public T getSample(); 28 | } 29 | -------------------------------------------------------------------------------- /bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/NullStatsProvider.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with this 4 | * work for additional information regarding copyright ownership. The ASF 5 | * licenses this file to you under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | package org.apache.bookkeeper.stats; 18 | 19 | import org.apache.commons.configuration.Configuration; 20 | 21 | public class NullStatsProvider implements StatsProvider { 22 | 23 | final StatsLogger nullStatsLogger = new NullStatsLogger(); 24 | 25 | @Override 26 | public void start(Configuration conf) { 27 | // nop 28 | } 29 | 30 | @Override 31 | public void stop() { 32 | // nop 33 | } 34 | 35 | @Override 36 | public StatsLogger getStatsLogger(String scope) { 37 | return nullStatsLogger; 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/OpStatsLogger.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with this 4 | * work for additional information regarding copyright ownership. The ASF 5 | * licenses this file to you under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | package org.apache.bookkeeper.stats; 18 | 19 | import java.util.concurrent.TimeUnit; 20 | 21 | /** 22 | * This interface handles logging of statistics related to each operation (PUBLISH, 23 | * CONSUME etc.) 24 | */ 25 | public interface OpStatsLogger { 26 | 27 | /** 28 | * Increment the failed op counter with the given eventLatency. 29 | * @param eventLatencyMillis The event latency 30 | * @param unit 31 | */ 32 | public void registerFailedEvent(long eventLatency, TimeUnit unit); 33 | 34 | /** 35 | * An operation succeeded with the given eventLatency. Update 36 | * stats to reflect the same 37 | * @param eventLatencyMillis The event latency 38 | * @param unit 39 | */ 40 | public void registerSuccessfulEvent(long eventLatency, TimeUnit unit); 41 | 42 | /** 43 | * An operation with the given value succeeded 44 | * @param value 45 | */ 46 | public void registerSuccessfulValue(long value); 47 | 48 | /** 49 | * An operation with the given value failed 50 | */ 51 | public void registerFailedValue(long value); 52 | 53 | /** 54 | * @return Returns an OpStatsData object with necessary values. We need this function 55 | * to support JMX exports. This should be deprecated sometime in the near future. 56 | * populated. 57 | */ 58 | public OpStatsData toOpStatsData(); 59 | 60 | /** 61 | * Clear stats for this operation. 62 | */ 63 | public void clear(); 64 | } 65 | -------------------------------------------------------------------------------- /bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/StatsLogger.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with this 4 | * work for additional information regarding copyright ownership. The ASF 5 | * licenses this file to you under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | package org.apache.bookkeeper.stats; 18 | 19 | /** 20 | * A simple interface that exposes just 2 useful methods. One to get the logger for an Op stat 21 | * and another to get the logger for a simple stat 22 | */ 23 | public interface StatsLogger { 24 | /** 25 | * @param name 26 | * Stats Name 27 | * @return Get the logger for an OpStat described by the name. 28 | */ 29 | public OpStatsLogger getOpStatsLogger(String name); 30 | 31 | /** 32 | * @param name 33 | * Stats Name 34 | * @return Get the logger for a simple stat described by the name 35 | */ 36 | public Counter getCounter(String name); 37 | 38 | /** 39 | * Register given guage as name name. 40 | * 41 | * @param name 42 | * gauge name 43 | */ 44 | public void registerGauge(String name, Gauge gauge); 45 | 46 | /** 47 | * Provide the stats logger under scope name. 48 | * 49 | * @param name 50 | * scope name. 51 | * @return stats logger under scope name. 52 | */ 53 | public StatsLogger scope(String name); 54 | 55 | } 56 | -------------------------------------------------------------------------------- /bookkeeper-stats/src/main/java/org/apache/bookkeeper/stats/StatsProvider.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with this 4 | * work for additional information regarding copyright ownership. The ASF 5 | * licenses this file to you under the Apache License, Version 2.0 (the 6 | * "License"); you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | * License for the specific language governing permissions and limitations under 15 | * the License. 16 | */ 17 | package org.apache.bookkeeper.stats; 18 | 19 | import org.apache.commons.configuration.Configuration; 20 | 21 | /** 22 | * Provider to provide stats logger for different scopes. 23 | */ 24 | public interface StatsProvider { 25 | /** 26 | * Intialize the stats provider by loading the given configuration conf. 27 | * 28 | * @param conf 29 | * Configuration to configure the stats provider. 30 | */ 31 | public void start(Configuration conf); 32 | 33 | /** 34 | * Close the stats provider 35 | */ 36 | public void stop(); 37 | 38 | /** 39 | * Return the stats logger to a given scope 40 | * @param scope 41 | * Scope for the given stats 42 | * @return stats logger for the given scope 43 | */ 44 | public StatsLogger getStatsLogger(String scope); 45 | } 46 | -------------------------------------------------------------------------------- /compat-deps/pom.xml: -------------------------------------------------------------------------------- 1 | 17 | 18 | 22 | 23 | bookkeeper 24 | org.apache.bookkeeper 25 | 4.5.0-SNAPSHOT 26 | 27 | 4.0.0 28 | org.apache.bookkeeper 29 | 4.5.0-SNAPSHOT 30 | compat-deps 31 | pom 32 | compability dependencies 33 | 34 | bookkeeper-server-compat-4.0.0 35 | bookkeeper-server-compat-4.1.0 36 | bookkeeper-server-compat-4.2.0 37 | 38 | 39 | UTF-8 40 | UTF-8 41 | 42 | 43 | -------------------------------------------------------------------------------- /doc/bookkeeperJMX.textile: -------------------------------------------------------------------------------- 1 | Title: BookKeeper JMX 2 | Notice: Licensed under the Apache License, Version 2.0 (the "License"); 3 | you may not use this file except in compliance with the License. You may 4 | obtain a copy of the License at "http://www.apache.org/licenses/LICENSE-2.0":http://www.apache.org/licenses/LICENSE-2.0. 5 | . 6 | . 7 | Unless required by applicable law or agreed to in writing, 8 | software distributed under the License is distributed on an "AS IS" 9 | BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 10 | implied. See the License for the specific language governing permissions 11 | and limitations under the License. 12 | . 13 | . 14 | 15 | h1. JMX 16 | 17 | Apache BookKeeper has extensive support for JMX, which allows viewing and managing a BookKeeper cluster. 18 | 19 | This document assumes that you have basic knowledge of JMX. See "Sun JMX Technology":http://java.sun.com/javase/technologies/core/mntr-mgmt/javamanagement/ page to get started with JMX. 20 | 21 | See the "JMX Management Guide":http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html for details on setting up local and remote management of VM instances. By default the included __bookkeeper__ script supports only local management - review the linked document to enable support for remote management (beyond the scope of this document). 22 | 23 | __Bookie Server__ is a JMX manageable server, which registers the proper MBeans during initialization to support JMX monitoring and management of the instance. 24 | 25 | h1. Bookie Server MBean Reference 26 | 27 | This table details JMX for a bookie server. 28 | 29 | | _.MBean | _.MBean Object Name | _.Description | 30 | | BookieServer | BookieServer_ | Represents a bookie server. Note that the object name includes bookie port that the server listens on. It is the root MBean for bookie server, which includes statistics for a bookie server. E.g. number packets sent/received, and statistics for add/read operations. | 31 | | Bookie | Bookie | Provide bookie statistics. Currently it just returns current journal queue length waiting to be committed. | 32 | | LedgerCache | LedgerCache | Provide ledger cache statistics. E.g. number of page cached in page cache, number of files opened for ledger index files. | 33 | -------------------------------------------------------------------------------- /doc/doc.textile: -------------------------------------------------------------------------------- 1 | Notice: Licensed under the Apache License, Version 2.0 (the "License"); 2 | you may not use this file except in compliance with the License. You may 3 | obtain a copy of the License at "http://www.apache.org/licenses/LICENSE-2.0":http://www.apache.org/licenses/LICENSE-2.0. 4 | . 5 | Unless required by applicable law or agreed to in writing, 6 | software distributed under the License is distributed on an "AS IS" 7 | BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 8 | implied. See the License for the specific language governing permissions 9 | and limitations under the License. 10 | . 11 | 12 | In the documentation directory, you'll find: 13 | 14 | * @build.txt@: Building Hedwig, or how to set up Hedwig 15 | * @user.txt@: User's Guide, or how to program against the Hedwig API and how to run it 16 | * @dev.txt@: Developer's Guide, or Hedwig internals and hacking details 17 | 18 | These documents are all written in the "Pandoc":http://johnmacfarlane.net/pandoc/ dialect of "Markdown":http://daringfireball.net/projects/markdown/. This makes them readable as plain text files, but also capable of generating HTML or LaTeX documentation. 19 | 20 | Documents are wrapped at 80 chars and use 2-space indentation. 21 | 22 | -------------------------------------------------------------------------------- /src/assemble/src.xml: -------------------------------------------------------------------------------- 1 | 17 | 20 | src 21 | 22 | tar.gz 23 | 24 | true 25 | 26 | 27 | . 28 | true 29 | 30 | **/README 31 | **/LICENSE 32 | **/NOTICE 33 | **/pom.xml 34 | **/src/** 35 | **/conf/** 36 | **/bin/** 37 | **/*.txt 38 | doc/* 39 | 40 | 41 | .git/** 42 | **/.gitignore 43 | **/.svn 44 | **/*.iws 45 | **/*.ipr 46 | **/*.iml 47 | **/.classpath 48 | **/.project 49 | **/.settings 50 | **/target/** 51 | 52 | **/*.log 53 | **/build/** 54 | **/file:/** 55 | **/SecurityAuth.audit* 56 | 57 | 58 | 59 | target/site/apidocs 60 | doc/apidocs 61 | 62 | 63 | 64 | --------------------------------------------------------------------------------