├── CMakeLists.txt ├── LICENSE.txt ├── README.txt ├── build.xml ├── cmake ├── FindLog4cpp.cmake ├── Findlibedit.cmake └── Findxfs.cmake ├── conf ├── ChunkServer.prp ├── KfsClient.prp ├── MetaServer.prp ├── forcefield.cfg ├── machines_distributed.cfg.sample └── machines_local.cfg.sample ├── doc ├── APPS_INT.txt ├── COMPILING.txt ├── DEPLOYING.txt ├── INTRO.txt ├── USING.txt └── WHAT_IS_NEW.txt ├── examples ├── KfsSample.java └── KfsSample_main.cc ├── how-to-launch-on-mac ├── kfs-hadoop ├── 0.15x │ └── src │ │ └── java │ │ └── org │ │ └── apache │ │ └── hadoop │ │ └── fs │ │ └── kfs │ │ ├── IFSImpl.java │ │ ├── KFSImpl.java │ │ ├── KFSInputStream.java │ │ ├── KFSOutputStream.java │ │ ├── KosmosFileSystem.java │ │ └── package.html ├── 0.16x │ └── src │ │ └── java │ │ └── org │ │ └── apache │ │ └── hadoop │ │ └── fs │ │ └── kfs │ │ ├── IFSImpl.java │ │ ├── KFSImpl.java │ │ ├── KFSInputStream.java │ │ ├── KFSOutputStream.java │ │ ├── KosmosFileSystem.java │ │ └── package.html ├── 0.17x │ └── src │ │ └── java │ │ └── org │ │ └── apache │ │ └── hadoop │ │ └── fs │ │ └── kfs │ │ ├── IFSImpl.java │ │ ├── KFSImpl.java │ │ ├── KFSInputStream.java │ │ ├── KFSOutputStream.java │ │ ├── KosmosFileSystem.java │ │ └── package.html └── 0.18x │ └── src │ └── core │ └── org │ └── apache │ └── hadoop │ └── fs │ └── kfs │ ├── IFSImpl.java │ ├── KFSImpl.java │ ├── KFSInputStream.java │ ├── KFSOutputStream.java │ ├── KosmosFileSystem.java │ └── package.html ├── scripts.mac_solaris ├── kfsclean.sh ├── kfsfsck.py ├── kfsinstall.sh ├── kfslaunch.py ├── kfsprune.py ├── kfsrun.sh ├── kfssetup.py ├── kfsshell.py ├── machines.cfg ├── machines.cfg.local ├── metabkup.sh ├── metalogprune.py └── slaves.py ├── scripts ├── kfs.spec ├── kfsclean.sh ├── kfsfsck.py ├── kfsinstall.sh ├── kfslaunch.py ├── kfsprune.py ├── kfsrun.sh ├── kfssetup.py ├── kfsshell.py ├── machines.cfg ├── memory_pin.sh ├── metabkup.sh └── slaves.py ├── src ├── cc │ ├── access │ │ ├── CMakeLists.txt │ │ ├── KfsAccessJni.cc │ │ ├── KfsModulePy.cc │ │ └── kfs_setup.py │ ├── chunk │ │ ├── AtomicRecordAppender.cc │ │ ├── AtomicRecordAppender.h │ │ ├── BufferManager.cc │ │ ├── BufferManager.h │ │ ├── CMakeLists.txt │ │ ├── Chunk.h │ │ ├── ChunkManager.cc │ │ ├── ChunkManager.h │ │ ├── ChunkServer.cc │ │ ├── ChunkServer.h │ │ ├── ChunkServer_main.cc │ │ ├── ChunkSortHelper.cc │ │ ├── ChunkSortHelper.h │ │ ├── ClientManager.cc │ │ ├── ClientManager.h │ │ ├── ClientSM.cc │ │ ├── ClientSM.h │ │ ├── DiskIo.cc │ │ ├── DiskIo.h │ │ ├── KfsOps.cc │ │ ├── KfsOps.h │ │ ├── LeaseClerk.cc │ │ ├── LeaseClerk.h │ │ ├── Logger.cc │ │ ├── Logger.h │ │ ├── MetaServerSM.cc │ │ ├── MetaServerSM.h │ │ ├── RemoteSyncSM.cc │ │ ├── RemoteSyncSM.h │ │ ├── Replicator.cc │ │ ├── Replicator.h │ │ ├── Utils.cc │ │ ├── Utils.h │ │ ├── chunkscrubber_main.cc │ │ ├── chunktrimmer_main.cc │ │ └── chunkupgrade_main.cc │ ├── common │ │ ├── BufferedLogWriter.cc │ │ ├── BufferedLogWriter.h │ │ ├── CMakeLists.txt │ │ ├── DailyRollingFileAppender.cc │ │ ├── DailyRollingFileAppender.h │ │ ├── Version.h │ │ ├── buildVers.py │ │ ├── config.h │ │ ├── cxxutil.h │ │ ├── hsieh_hash.cc │ │ ├── hsieh_hash.h │ │ ├── kfsdecls.h │ │ ├── kfstypes.h │ │ ├── log.cc │ │ ├── log.h │ │ ├── properties.cc │ │ └── properties.h │ ├── doxkfs │ ├── emulator │ │ ├── CMakeLists.txt │ │ ├── ChunkServerEmulator.cc │ │ ├── ChunkServerEmulator.h │ │ ├── EmulatorSetup.cc │ │ ├── EmulatorSetup.h │ │ ├── LayoutEmulator.cc │ │ ├── LayoutEmulator.h │ │ ├── rebalanceexecutor_main.cc │ │ ├── rebalanceplanner_main.cc │ │ ├── replicachecker_main.cc │ │ └── rereplicator_main.cc │ ├── fuse │ │ ├── CMakeLists.txt │ │ └── kfs_fuse_main.cc │ ├── libkfsClient │ │ ├── CMakeLists.txt │ │ ├── KfsAsyncRW.cc │ │ ├── KfsAsyncRW.h │ │ ├── KfsAttr.h │ │ ├── KfsClient.cc │ │ ├── KfsClient.h │ │ ├── KfsClientInt.h │ │ ├── KfsNetClient.cc │ │ ├── KfsNetClient.h │ │ ├── KfsOps.cc │ │ ├── KfsOps.h │ │ ├── KfsPendingOp.cc │ │ ├── KfsPendingOp.h │ │ ├── KfsProtocolWorker.cc │ │ ├── KfsProtocolWorker.h │ │ ├── KfsRead.cc │ │ ├── KfsWrite.cc │ │ ├── LeaseClerk.cc │ │ ├── LeaseClerk.h │ │ ├── Utils.cc │ │ ├── Utils.h │ │ ├── WriteAppender.cc │ │ ├── WriteAppender.h │ │ └── concurrency.h │ ├── libkfsIO │ │ ├── Acceptor.cc │ │ ├── Acceptor.h │ │ ├── BufferedSocket.cc │ │ ├── BufferedSocket.h │ │ ├── CMakeLists.txt │ │ ├── Checksum.cc │ │ ├── Checksum.h │ │ ├── Counter.cc │ │ ├── Counter.h │ │ ├── DiskConnection.h │ │ ├── DiskEvent.h │ │ ├── DiskManager.h │ │ ├── Event.h │ │ ├── EventManager.cc │ │ ├── EventManager.h │ │ ├── FileHandle.h │ │ ├── Globals.cc │ │ ├── Globals.h │ │ ├── IOBuffer.cc │ │ ├── IOBuffer.h │ │ ├── ITimeout.h │ │ ├── KfsCallbackObj.h │ │ ├── NetConnection.cc │ │ ├── NetConnection.h │ │ ├── NetErrorSimulator.cc │ │ ├── NetErrorSimulator.h │ │ ├── NetKicker.cc │ │ ├── NetKicker.h │ │ ├── NetManager.cc │ │ ├── NetManager.h │ │ ├── TcpSocket.cc │ │ ├── TcpSocket.h │ │ ├── TelemetryClient.cc │ │ └── TelemetryClient.h │ ├── meta │ │ ├── CMakeLists.txt │ │ ├── ChildProcessTracker.cc │ │ ├── ChildProcessTracker.h │ │ ├── ChunkReplicator.cc │ │ ├── ChunkReplicator.h │ │ ├── ChunkServer.cc │ │ ├── ChunkServer.h │ │ ├── ChunkServerFactory.cc │ │ ├── ChunkServerFactory.h │ │ ├── ChunkServerHeartbeater.h │ │ ├── ClientManager.h │ │ ├── ClientSM.cc │ │ ├── ClientSM.h │ │ ├── LayoutManager.cc │ │ ├── LayoutManager.h │ │ ├── LeaseCleaner.cc │ │ ├── LeaseCleaner.h │ │ ├── NetDispatch.cc │ │ ├── NetDispatch.h │ │ ├── base.h │ │ ├── checkpoint.cc │ │ ├── checkpoint.h │ │ ├── entry.cc │ │ ├── entry.h │ │ ├── filelister_main.cc │ │ ├── kfsfsck_main.cc │ │ ├── kfsops.cc │ │ ├── kfstree.cc │ │ ├── kfstree.h │ │ ├── kfstypes.h │ │ ├── logcompactor_main.cc │ │ ├── logger.cc │ │ ├── logger.h │ │ ├── meta.cc │ │ ├── meta.h │ │ ├── metaserver_main.cc │ │ ├── queue.h │ │ ├── replay.cc │ │ ├── replay.h │ │ ├── request.cc │ │ ├── request.h │ │ ├── restore.cc │ │ ├── restore.h │ │ ├── restoremtime_main.cc │ │ ├── startup.cc │ │ ├── startup.h │ │ ├── thread.h │ │ ├── util.cc │ │ └── util.h │ ├── qcdio │ │ ├── CMakeLists.txt │ │ ├── Makefile │ │ ├── iovperf.c │ │ ├── qcdebug.h │ │ ├── qcdiskqueue.cpp │ │ ├── qcdiskqueue.h │ │ ├── qcdllist.h │ │ ├── qcfdpoll.cpp │ │ ├── qcfdpoll.h │ │ ├── qciobufferpool.cpp │ │ ├── qciobufferpool.h │ │ ├── qcmutex.cpp │ │ ├── qcmutex.h │ │ ├── qcstutils.h │ │ ├── qcthread.cpp │ │ ├── qcthread.h │ │ ├── qcunittest.cpp │ │ ├── qcutils.cpp │ │ └── qcutils.h │ ├── rebalancer │ │ ├── CMakeLists.txt │ │ ├── rebalancer.h │ │ └── rebalancer_main.cc │ ├── sailfish │ │ ├── mrtools │ │ │ ├── CMakeLists.txt │ │ │ ├── appender.cc │ │ │ ├── icomparer_main.cc │ │ │ ├── ifile_base.h │ │ │ ├── ifilereader_main.cc │ │ │ ├── imerger.cc │ │ │ ├── imerger.h │ │ │ ├── imerger_main.cc │ │ │ ├── input_handler.cc │ │ │ ├── kappender.h │ │ │ ├── kappender_main.cc │ │ │ ├── kdatagen_main.cc │ │ │ ├── merge_validator_main.cc │ │ │ ├── settest_main.cc │ │ │ ├── util.cc │ │ │ ├── util.h │ │ │ ├── workgetter.cc │ │ │ └── workgetter.h │ │ ├── sorter │ │ │ ├── CMakeLists.txt │ │ │ ├── chunksort_test_main.cc │ │ │ ├── chunksorter.cc │ │ │ ├── chunksorter.h │ │ │ ├── chunksorter_main.cc │ │ │ ├── sortertest_main.cc │ │ │ ├── util.cc │ │ │ └── util.h │ │ └── workbuilder │ │ │ ├── CMakeLists.txt │ │ │ ├── clientmgr.h │ │ │ ├── clientsm.cc │ │ │ ├── clientsm.h │ │ │ ├── indexreader.cc │ │ │ ├── job.cc │ │ │ ├── setnumreducer.cc │ │ │ ├── setnumreducer.h │ │ │ ├── statuschecker.cc │ │ │ ├── statuschecker.h │ │ │ ├── util.cc │ │ │ ├── util.h │ │ │ ├── workbuilder.cc │ │ │ ├── workbuilder.h │ │ │ └── workbuilder_main.cc │ ├── telemetry │ │ ├── CMakeLists.txt │ │ ├── packet.h │ │ ├── telemetry_repeater_main.cc │ │ ├── telemetry_server.h │ │ └── telemetry_server_main.cc │ ├── tests │ │ ├── CMakeLists.txt │ │ ├── KfsDataGen_main.cc │ │ ├── KfsDirFileTester_main.cc │ │ ├── KfsDirScanTest_main.cc │ │ ├── KfsIndexReader_main.cc │ │ ├── KfsLogTest_main.cc │ │ ├── KfsPerfReader_main.cc │ │ ├── KfsPerfWriter_main.cc │ │ ├── KfsRW_main.cc │ │ ├── KfsReader_main.cc │ │ ├── KfsRecordAppend_main.cc │ │ ├── KfsSeekWrite_main.cc │ │ ├── KfsTester.properties.sample │ │ ├── KfsTrunc_main.cc │ │ ├── KfsWriter_main.cc │ │ ├── memory_pinner_main.cc │ │ └── mkfstree_main.cc │ └── tools │ │ ├── CMakeLists.txt │ │ ├── KfsAppend.cc │ │ ├── KfsCat_main.cc │ │ ├── KfsCd.cc │ │ ├── KfsChangeReplication.cc │ │ ├── KfsCp.cc │ │ ├── KfsFstat.cc │ │ ├── KfsLs.cc │ │ ├── KfsMkdirs.cc │ │ ├── KfsMv.cc │ │ ├── KfsPwd.cc │ │ ├── KfsRm.cc │ │ ├── KfsRmdir.cc │ │ ├── KfsShell.h │ │ ├── KfsToolsCommon.cc │ │ ├── KfsToolsCommon.h │ │ ├── MonUtils.cc │ │ ├── MonUtils.h │ │ ├── cpfromkfs_main.cc │ │ ├── cptokfs_main.cc │ │ ├── kfscp_main.cc │ │ ├── kfsdataverify_main.cc │ │ ├── kfsfileenum_main.cc │ │ ├── kfsls_main.cc │ │ ├── kfsping_main.cc │ │ ├── kfsput_main.cc │ │ ├── kfsrebalance_main.cc │ │ ├── kfsretire_main.cc │ │ ├── kfsshell_main.cc │ │ ├── kfsstats_main.cc │ │ ├── kfstoggleworm_main.cc │ │ └── utils.cc ├── java │ └── org │ │ └── kosmix │ │ └── kosmosfs │ │ └── access │ │ ├── KfsAccess.java │ │ ├── KfsFileAttr.java │ │ ├── KfsInputChannel.java │ │ ├── KfsOutputChannel.java │ │ ├── KfsTest.java │ │ └── Positionable.java ├── python │ ├── flogger │ │ ├── flogger.cfg │ │ ├── flogger.py │ │ └── readdirplus.py │ ├── kfscp │ │ └── kfscp.py │ ├── rmr │ │ └── rmr.py │ ├── rw │ │ └── rw.py │ └── unittests │ │ ├── KfsClient.prp │ │ ├── dirops.py │ │ ├── worm.py │ │ ├── write-flush.py │ │ └── write.py └── set_svnprops.sh └── webui ├── files ├── kfsstyle.css └── sorttable │ ├── arrow-down.gif │ ├── arrow-none.gif │ ├── arrow-up.gif │ └── sorttable.js └── kfsstatus.py /README.txt: -------------------------------------------------------------------------------- 1 | 2 | # 3 | # $Id: README.txt 386 2010-05-27 16:01:24Z sriramsrao $ 4 | # 5 | # Created on 2007/08/23 6 | # 7 | # Copyright 2008 Quantcast Corp. 8 | # Copyright 2007 Kosmix Corp. 9 | # 10 | # This file is part of Kosmos File System (KFS). 11 | # 12 | # Licensed under the Apache License, Version 2.0 13 | # (the "License"); you may not use this file except in compliance with 14 | # the License. You may obtain a copy of the License at 15 | # 16 | # http://www.apache.org/licenses/LICENSE-2.0 17 | # 18 | # Unless required by applicable law or agreed to in writing, software 19 | # distributed under the License is distributed on an "AS IS" BASIS, 20 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | # implied. See the License for the specific language governing 22 | # permissions and limitations under the License. 23 | # 24 | # Sriram Rao 25 | # Quantcast Corp. 26 | 27 | ================= 28 | 29 | Welcome to the Kosmos File System (KFS)! The documentation is now on the 30 | project Wiki: 31 | 32 | http://kosmosfs.wiki.sourceforge.net/ 33 | 34 | KFS is being released under the Apache 2.0 license. A copy of the license 35 | is included in the file LICENSE.txt. 36 | 37 | 38 | DIRECTORY ORGANIZATION 39 | ====================== 40 | - kfs (top-level directory) 41 | | 42 | |---> conf (sample config files) 43 | |---> examples (Example client code for accessing KFS) 44 | | 45 | |---> src 46 | | 47 | |----> cc 48 | | 49 | |---> access (Java/Python glue code) 50 | |---> meta (meta server code) 51 | |---> chunk (chunk server code) 52 | |---> libkfsClient (client library code) 53 | |---> libkfsIO (IO library used by KFS) 54 | |---> common (common declarations) 55 | |---> fuse (FUSE module for Linux) 56 | |---> tools (KFS tools) 57 | | 58 | |----> java 59 | |---> org/kosmix/kosmosfs/access: Java wrappers to call KFS-JNI code 60 | | 61 | |----> python 62 | |---> tests (Python test scripts) 63 | 64 | -------------------------------------------------------------------------------- /cmake/FindLog4cpp.cmake: -------------------------------------------------------------------------------- 1 | # - Find Log4cpp 2 | # Find the native Log4cpp includes and library 3 | # 4 | # Log4cpp_INCLUDE_DIR - where to find Log4cpp.h, etc. 5 | # Log4cpp_LIBRARIES - List of libraries when using Log4cpp. 6 | # Log4cpp_FOUND - True if Log4cpp found. 7 | 8 | 9 | IF (Log4cpp_INCLUDE_DIR) 10 | # Already in cache, be silent 11 | SET(Log4cpp_FIND_QUIETLY TRUE) 12 | ENDIF (Log4cpp_INCLUDE_DIR) 13 | 14 | FIND_PATH(Log4cpp_INCLUDE_DIR log4cpp/Category.hh 15 | /opt/local/include 16 | /usr/local/include 17 | /usr/include 18 | ) 19 | 20 | SET(Log4cpp_NAMES log4cpp) 21 | FIND_LIBRARY(Log4cpp_LIBRARY 22 | NAMES ${Log4cpp_NAMES} 23 | PATHS /usr/local/lib64/lib /usr/lib /usr/local/lib /opt/local/lib 24 | ) 25 | 26 | IF (Log4cpp_INCLUDE_DIR AND Log4cpp_LIBRARY) 27 | SET(Log4cpp_FOUND TRUE) 28 | SET( Log4cpp_LIBRARIES ${Log4cpp_LIBRARY} ) 29 | ELSE (Log4cpp_INCLUDE_DIR AND Log4cpp_LIBRARY) 30 | SET(Log4cpp_FOUND FALSE) 31 | SET( Log4cpp_LIBRARIES ) 32 | ENDIF (Log4cpp_INCLUDE_DIR AND Log4cpp_LIBRARY) 33 | 34 | IF (Log4cpp_FOUND) 35 | IF (NOT Log4cpp_FIND_QUIETLY) 36 | MESSAGE(STATUS "Found Log4cpp: ${Log4cpp_LIBRARY}") 37 | ENDIF (NOT Log4cpp_FIND_QUIETLY) 38 | ELSE (Log4cpp_FOUND) 39 | IF (Log4cpp_FIND_REQUIRED) 40 | MESSAGE(STATUS "Looked for Log4cpp libraries named ${Log4cppS_NAMES}.") 41 | MESSAGE(FATAL_ERROR "Could NOT find Log4cpp library") 42 | ENDIF (Log4cpp_FIND_REQUIRED) 43 | ENDIF (Log4cpp_FOUND) 44 | 45 | MARK_AS_ADVANCED( 46 | Log4cpp_LIBRARY 47 | Log4cpp_INCLUDE_DIR 48 | ) 49 | -------------------------------------------------------------------------------- /cmake/Findlibedit.cmake: -------------------------------------------------------------------------------- 1 | FIND_PATH(LIBEDIT_INCLUDE_DIR readline.h /usr/include/editline /usr/local/include/editline) 2 | 3 | FIND_LIBRARY(LIBEDIT_LIBRARY NAMES edit PATH /usr/lib /usr/local/lib /usr/lib64 /usr/local/lib64) 4 | 5 | IF (LIBEDIT_INCLUDE_DIR AND LIBEDIT_LIBRARY) 6 | SET(LIBEDIT_FOUND TRUE) 7 | ENDIF (LIBEDIT_INCLUDE_DIR AND LIBEDIT_LIBRARY) 8 | 9 | IF (LIBEDIT_FOUND) 10 | IF (NOT libedit_FIND_QUIETLY) 11 | MESSAGE(STATUS "Found libedit: ${LIBEDIT_LIBRARY}") 12 | ENDIF (NOT libedit_FIND_QUIETLY) 13 | ADD_DEFINITIONS(-DHAVE_LIBEDIT=1) 14 | ELSE (LIBEDIT_FOUND) 15 | IF (libedit_FIND_REQUIRED) 16 | MESSAGE(FATAL_ERROR "Could not find libedit") 17 | ENDIF (libedit_FIND_REQUIRED) 18 | ENDIF (LIBEDIT_FOUND) 19 | -------------------------------------------------------------------------------- /cmake/Findxfs.cmake: -------------------------------------------------------------------------------- 1 | # - Find XFS includes 2 | # 3 | # XFS_INCLUDE_DIR - where to find XFS.h, etc. 4 | # XFS_FOUND - True if XFS found. 5 | 6 | 7 | IF (XFS_INCLUDE_DIR) 8 | # Already in cache, be silent 9 | SET(XFS_FIND_QUIETLY TRUE) 10 | ENDIF (XFS_INCLUDE_DIR) 11 | 12 | FIND_PATH(XFS_INCLUDE_DIR xfs/xfs.h 13 | /opt/local/include 14 | /usr/local/include 15 | /usr/include 16 | ) 17 | 18 | IF (XFS_INCLUDE_DIR) 19 | SET(XFS_FOUND TRUE) 20 | ELSE (XFS_INCLUDE_DIR) 21 | SET(XFS_FOUND FALSE) 22 | ENDIF (XFS_INCLUDE_DIR) 23 | 24 | IF (XFS_FOUND) 25 | IF (NOT XFS_FIND_QUIETLY) 26 | MESSAGE(STATUS "Found XFS:") 27 | ENDIF (NOT XFS_FIND_QUIETLY) 28 | ADD_DEFINITIONS(-DKFS_USE_XFS=1) 29 | ELSE (XFS_FOUND) 30 | IF (XFS_FIND_REQUIRED) 31 | MESSAGE(FATAL_ERROR "Could NOT find XFS") 32 | ENDIF (XFS_FIND_REQUIRED) 33 | MESSAGE(STATUS "Could NOT find XFS") 34 | ENDIF (XFS_FOUND) 35 | 36 | MARK_AS_ADVANCED( 37 | XFS_INCLUDE_DIR 38 | ) 39 | -------------------------------------------------------------------------------- /conf/ChunkServer.prp: -------------------------------------------------------------------------------- 1 | # Configuration for chunk server: 2 | # meta server location 3 | chunkServer.metaServer.hostname = somehost.com 4 | chunkServer.metaServer.port = 30000 5 | # port to open for client connections 6 | chunkServer.clientPort = 22000 7 | # Directory for storing the chunks 8 | chunkServer.chunkDir = ./chunks 9 | chunkServer.logDir = ./logs 10 | # provide 300G of storage space 11 | chunkServer.totalSpace = 300000000000 12 | 13 | -------------------------------------------------------------------------------- /conf/KfsClient.prp: -------------------------------------------------------------------------------- 1 | # Where is the metaserver 2 | metaServer.name = somehost.com 3 | metaServer.port = 20000 4 | -------------------------------------------------------------------------------- /conf/MetaServer.prp: -------------------------------------------------------------------------------- 1 | # Port at which clients can connect to us 2 | metaServer.clientPort = 20000 3 | # Port at which chunkservers can connect to us 4 | metaServer.chunkServerPort = 30000 5 | metaServer.logDir = ./kfslog 6 | metaServer.cpDir = ./kfscp 7 | -------------------------------------------------------------------------------- /conf/forcefield.cfg: -------------------------------------------------------------------------------- 1 | [metaserver] 2 | node: localhost 3 | rundir: /Users/sriramr/kfs/run/meta 4 | baseport: 20000 5 | clusterkey: test-cluster 6 | [chunkserver1] 7 | node: localhost 8 | rundir: /Users/sriramr/kfs/run/chunk 9 | baseport: 30000 10 | space: 30 G 11 | 12 | -------------------------------------------------------------------------------- /conf/machines_distributed.cfg.sample: -------------------------------------------------------------------------------- 1 | # KFS Machine configuration file 2 | # 3 | # The following configuration is for a multi-node KFS setup. There is 4 | # one meta server and 3 chunk servers. The chunk servers are exporting 5 | # 30,000,000 bytes or 30G apiece. Note that, though the baseport 6 | # for the chunkservers is the same on all nodes, because the nodes are 7 | # different, the values are unique on each node; similary for the 8 | # rundir of the chunk servers. 9 | # 10 | [metaserver] 11 | node: hostA 12 | rundir: /Users/kosmix/kfsRun/meta 13 | baseport: 20000 14 | clusterkey: test-cluster 15 | [chunkserver1] 16 | node: host1 17 | rundir: /Users/kosmix/kfsRun/chunk 18 | baseport: 30000 19 | space: 30 G 20 | [chunkserver2] 21 | node: host2 22 | rundir: /Users/kosmix/kfsRun/chunk 23 | baseport: 30000 24 | space: 30 G 25 | [chunkserver3] 26 | node: host3 27 | rundir: /Users/kosmix/kfsRun/chunk 28 | baseport: 30000 29 | space: 30000 M 30 | -------------------------------------------------------------------------------- /conf/machines_local.cfg.sample: -------------------------------------------------------------------------------- 1 | # KFS Machine configuration file 2 | # 3 | # The following configuration is a single node KFS setup. There is 4 | # one meta server and 2 chunk servers. The chunk servers are exporting 5 | # 30,000,000 bytes or 30 G apiece. Note the unique values for the baseport, rundir. 6 | # 7 | [metaserver] 8 | node: localhost 9 | rundir: /Users/kosmix/kfsRun/meta 10 | baseport: 20000 11 | clusterkey: test-cluster 12 | [chunkserver1] 13 | node: localhost 14 | rundir: /Users/kosmix/kfsRun/chunk1 15 | baseport: 30000 16 | space: 30 G 17 | [chunkserver1] 18 | node: localhost 19 | rundir: /Users/kosmix/kfsRun/chunk2 20 | baseport: 40000 21 | space: 30000 M 22 | -------------------------------------------------------------------------------- /doc/APPS_INT.txt: -------------------------------------------------------------------------------- 1 | 2 | # $Id: APPS_INT.txt 386 2010-05-27 16:01:24Z sriramsrao $ 3 | # 4 | # Created on 2007/08/23 5 | # 6 | # Copyright 2007 Kosmix Corp. 7 | # 8 | # This file is part of Kosmos File System (KFS). 9 | # 10 | # Licensed under the Apache License, Version 2.0 11 | # (the "License"); you may not use this file except in compliance with 12 | # the License. You may obtain a copy of the License at 13 | # 14 | # http://www.apache.org/licenses/LICENSE-2.0 15 | # 16 | # Unless required by applicable law or agreed to in writing, software 17 | # distributed under the License is distributed on an "AS IS" BASIS, 18 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | # implied. See the License for the specific language governing 20 | # permissions and limitations under the License. 21 | # 22 | # Sriram Rao 23 | # Kosmix Corp. 24 | 25 | TABLE OF CONTENTS 26 | ================= 27 | * INTRODUCTION 28 | * CLIENT-SIDE API 29 | * LINKING 30 | * HOW DOES IT WORK? 31 | 32 | INTRODUCTION 33 | ============ 34 | 35 | This document describes how to integrate the KFS client library with your applications. 36 | 37 | CLIENT-SIDE API 38 | =============== 39 | 40 | To use KFS with your applications, the application will need to be 41 | modified. Two sample programs are included in the package. 42 | - ~/code/kfs/examples/KfsTest_main.cc 43 | - ~/code/kfs/examples/KfsTest.java 44 | 45 | Each program shows how to use the client-side API. The KFS client 46 | library API is defined in: 47 | - C++: ~/code/kfs/src/cc/libkfsClient/KfsClient.h 48 | - Java: ~/code/kfs/src/java/org/kosmix/kosmosfs/access/KfsAccess.java 49 | 50 | NOTE: For Java applications, we use the JNI interface to get at the 51 | C++ KFS client library code from Java applications. 52 | 53 | LINKING 54 | ======== 55 | 56 | After KFS client library are integrated with your applications, you 57 | will need to perform the following steps to link the application: 58 | - C++: link with ~/code/kfs/build/lib-static/libkfsClient.a 59 | - Java: Add ~/code/kfs/build/kfs-0.1.jar to your CLASSPATH; also 60 | ~/code/kfs/build/lib/ should be in your LD_LIBRARY_PATH 61 | 62 | NOTE: ~/code/kfs/build/lib will, by default, contain libraries built 63 | with "Debug" option. This library code in this mode is "very 64 | chatty". Hence, it is recommended that you use the libraries built 65 | with "Release" option with your applications (such as, ~/code/kfs/build/release/...). 66 | 67 | HOW DOES IT WORK 68 | ================ 69 | 70 | In either case,the mode of execution is as follows: 71 | - the application interacts with the KFS client library 72 | - the KFS client library interfaces with the 73 | metaserver/chunkserver(s) 74 | - If data is replicated, whenever there is a failure in a 75 | chunkserver, the client library will fail-over to another chunkserver 76 | that has the data; this fail-over is transparent to the application. 77 | -------------------------------------------------------------------------------- /doc/USING.txt: -------------------------------------------------------------------------------- 1 | 2 | # $Id: USING.txt 386 2010-05-27 16:01:24Z sriramsrao $ 3 | # 4 | # Created on 2007/08/23 5 | # 6 | # Copyright 2007 Kosmix Corp. 7 | # 8 | # This file is part of Kosmos File System (KFS). 9 | # 10 | # Licensed under the Apache License, Version 2.0 11 | # (the "License"); you may not use this file except in compliance with 12 | # the License. You may obtain a copy of the License at 13 | # 14 | # http://www.apache.org/licenses/LICENSE-2.0 15 | # 16 | # Unless required by applicable law or agreed to in writing, software 17 | # distributed under the License is distributed on an "AS IS" BASIS, 18 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | # implied. See the License for the specific language governing 20 | # permissions and limitations under the License. 21 | # 22 | # 23 | # Sriram Rao 24 | # Kosmix Corp. 25 | 26 | TABLE OF CONTENTS 27 | ================= 28 | * INTRODUCTION 29 | * TOOLS SOURCE/BINARIES 30 | * LOADING/UNLOADING DATA 31 | * FILE SYSTEM TOOLS 32 | * SCRIPTING SUPPORT 33 | 34 | INTRODUCTION 35 | ============ 36 | 37 | This document describes how to use KFS. In particular, how to 38 | load/unload data from KFS servers; the set of tools that are available 39 | to view the filesystem tree. 40 | 41 | TOOLS SOURCE/BINARIES 42 | ===================== 43 | 44 | The source code for all the tools included in the package is at: 45 | ~/code/kfs/src/cc/tools 46 | 47 | When the package is compiled and installed, the tools are installed 48 | in: ~/code/kfs/build/bin/tools 49 | 50 | LOADING/UNLOADING DATA 51 | ====================== 52 | 53 | To load pre-existing data files into KFS, the data needs to be copied 54 | in to a KFS deployment. 55 | - cpFs2Kfs is a tool that copies a directory hierarchy from a local 56 | filesystem to a KFS; this tool creates a corresponding directory 57 | hierarchy in KFS 58 | 59 | When loading data into KFS, you can run multiple instances of cpFs2Kfs 60 | to speed up the copy; each instance copies in a portion of the 61 | directory tree. 62 | 63 | For debugging purposes, data can also copied out from KFS to a local 64 | filesystem: 65 | - cpKfs2Fs is a tool that copies a directory hierarchy from KFS 66 | to a local filesystem; this tool creates a corresponding directory 67 | hierarchy in the local filesystem 68 | 69 | These tools take arguments from the command line. 70 | Type: -h to see the list of options for that tool. 71 | 72 | FILE SYSTEM TOOLS 73 | ================= 74 | 75 | To manipulate the KFS directory tree, KfsShell tool is provided. 76 | This tool provides a basic set of filesystem commands such as 77 | - mkdir, rmdir, rm, cd, cp, mv, ls 78 | 79 | SCRIPTING SUPPORT 80 | ================= 81 | 82 | To manipulate the KFS directory tree from a script, commands can be 83 | "piped" to the KfsShell tool. For instance, to create a directory: 84 | mkdir foo | KfsShell -q -m -p 85 | 86 | 87 | -------------------------------------------------------------------------------- /doc/WHAT_IS_NEW.txt: -------------------------------------------------------------------------------- 1 | 2 | # $Id: WHAT_IS_NEW.txt 386 2010-05-27 16:01:24Z sriramsrao $ 3 | # 4 | # Created on 2007/11/10 5 | # 6 | # Copyright 2007 Kosmix Corp. 7 | # 8 | # This file is part of Kosmos File System (KFS). 9 | # 10 | # Licensed under the Apache License, Version 2.0 11 | # (the "License"); you may not use this file except in compliance with 12 | # the License. You may obtain a copy of the License at 13 | # 14 | # http://www.apache.org/licenses/LICENSE-2.0 15 | # 16 | # Unless required by applicable law or agreed to in writing, software 17 | # distributed under the License is distributed on an "AS IS" BASIS, 18 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | # implied. See the License for the specific language governing 20 | # permissions and limitations under the License. 21 | # 22 | # 23 | # Sriram Rao 24 | # Kosmix Corp. 25 | 26 | INTRODUCTION 27 | ============ 28 | 29 | This document describes the changes from a previous release. 30 | 31 | Changes from version 0.1 32 | ========================= 33 | - Support for multiple directories for storing chunks at the chunkserver. 34 | - Support for periodic remote backup of metaserver checkpoint files 35 | - Use log4cpp to write out metaserver/chunkserver log files. log4cpp 36 | provides log file rotation support. 37 | - Bug fixes 38 | 39 | Changes from version 0.1.1 40 | ========================== 41 | - Bug fixes 42 | 43 | Changes from version 0.1.2 44 | ========================= 45 | - Support for compiling on 32-bit Linux platforms 46 | - Support for compiling with gcc 4.3 47 | - Bug fixes 48 | 49 | -------------------------------------------------------------------------------- /how-to-launch-on-mac: -------------------------------------------------------------------------------- 1 | 2 | 1. Deploy: 3 | forcefield-lm:scripts.mac_solaris sriramr$ ./kfssetup.py -f ../conf/forcefield.cfg -r tar -w ../webui -b ../build -u -s 4 | 5 | 2. Run: 6 | forcefield-lm:scripts.mac_solaris sriramr$ ./kfslaunch.py -f ../conf/forcefield.cfg -s 7 | 8 | to stop: 9 | forcefield-lm:scripts.mac_solaris sriramr$ ./kfslaunch.py -f ../conf/forcefield.cfg -S 10 | -------------------------------------------------------------------------------- /kfs-hadoop/0.15x/src/java/org/apache/hadoop/fs/kfs/IFSImpl.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed under the Apache License, Version 2.0 4 | * (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | * implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | * 15 | * @author: Sriram Rao (Kosmix Corp.) 16 | * 17 | * We need to provide the ability to the code in fs/kfs without really 18 | * having a KFS deployment. In particular, the glue code that wraps 19 | * around calls to KfsAccess object. This is accomplished by defining a 20 | * filesystem implementation interface: 21 | * -- for testing purposes, a dummy implementation of this interface 22 | * will suffice; as long as the dummy implementation is close enough 23 | * to doing what KFS does, we are good. 24 | * -- for deployment purposes with KFS, this interface is 25 | * implemented by the KfsImpl object. 26 | */ 27 | 28 | package org.apache.hadoop.fs.kfs; 29 | 30 | import java.io.*; 31 | 32 | import org.apache.hadoop.fs.FSDataInputStream; 33 | import org.apache.hadoop.fs.FSDataOutputStream; 34 | import org.apache.hadoop.fs.FileStatus; 35 | import org.apache.hadoop.fs.Path; 36 | 37 | interface IFSImpl { 38 | public boolean exists(String path) throws IOException; 39 | public boolean isDirectory(String path) throws IOException; 40 | public boolean isFile(String path) throws IOException; 41 | public String[] readdir(String path) throws IOException; 42 | public FileStatus[] readdirplus(Path path) throws IOException; 43 | 44 | public int mkdirs(String path) throws IOException; 45 | public int rename(String source, String dest) throws IOException; 46 | 47 | public int rmdir(String path) throws IOException; 48 | public int remove(String path) throws IOException; 49 | public long filesize(String path) throws IOException; 50 | public short getReplication(String path) throws IOException; 51 | public short setReplication(String path, short replication) throws IOException; 52 | public String[][] getDataLocation(String path, long start, long len) throws IOException; 53 | 54 | public long getModificationTime(String path) throws IOException; 55 | public FSDataOutputStream create(String path, short replication, int bufferSize) throws IOException; 56 | public FSDataInputStream open(String path, int bufferSize) throws IOException; 57 | 58 | }; 59 | -------------------------------------------------------------------------------- /kfs-hadoop/0.15x/src/java/org/apache/hadoop/fs/kfs/KFSOutputStream.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed under the Apache License, Version 2.0 4 | * (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | * implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | * 15 | * @author: Sriram Rao (Kosmix Corp.) 16 | * 17 | * Implements the Hadoop FSOutputStream interfaces to allow applications to write to 18 | * files in Kosmos File System (KFS). 19 | */ 20 | 21 | package org.apache.hadoop.fs.kfs; 22 | 23 | import java.io.*; 24 | import java.net.*; 25 | import java.util.*; 26 | import java.nio.ByteBuffer; 27 | 28 | import org.apache.hadoop.conf.Configuration; 29 | import org.apache.hadoop.fs.Path; 30 | import org.apache.hadoop.fs.FSDataOutputStream; 31 | import org.apache.hadoop.util.Progressable; 32 | 33 | import org.kosmix.kosmosfs.access.KfsAccess; 34 | import org.kosmix.kosmosfs.access.KfsOutputChannel; 35 | 36 | class KFSOutputStream extends OutputStream { 37 | 38 | private String path; 39 | private KfsOutputChannel kfsChannel; 40 | 41 | public KFSOutputStream(KfsAccess kfsAccess, String path, short replication) { 42 | this.path = path; 43 | 44 | this.kfsChannel = kfsAccess.kfs_create(path, replication); 45 | } 46 | 47 | public long getPos() throws IOException { 48 | if (kfsChannel == null) { 49 | throw new IOException("File closed"); 50 | } 51 | return kfsChannel.tell(); 52 | } 53 | 54 | public void write(int v) throws IOException { 55 | if (kfsChannel == null) { 56 | throw new IOException("File closed"); 57 | } 58 | byte[] b = new byte[1]; 59 | 60 | b[0] = (byte) v; 61 | write(b, 0, 1); 62 | } 63 | 64 | public void write(byte b[], int off, int len) throws IOException { 65 | if (kfsChannel == null) { 66 | throw new IOException("File closed"); 67 | } 68 | 69 | kfsChannel.write(ByteBuffer.wrap(b, off, len)); 70 | } 71 | 72 | public void flush() throws IOException { 73 | if (kfsChannel == null) { 74 | throw new IOException("File closed"); 75 | } 76 | kfsChannel.sync(); 77 | } 78 | 79 | public synchronized void close() throws IOException { 80 | if (kfsChannel == null) { 81 | return; 82 | } 83 | flush(); 84 | kfsChannel.close(); 85 | kfsChannel = null; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /kfs-hadoop/0.16x/src/java/org/apache/hadoop/fs/kfs/IFSImpl.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed under the Apache License, Version 2.0 4 | * (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | * implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | * 15 | * @author: Sriram Rao (Kosmix Corp.) 16 | * 17 | * We need to provide the ability to the code in fs/kfs without really 18 | * having a KFS deployment. In particular, the glue code that wraps 19 | * around calls to KfsAccess object. This is accomplished by defining a 20 | * filesystem implementation interface: 21 | * -- for testing purposes, a dummy implementation of this interface 22 | * will suffice; as long as the dummy implementation is close enough 23 | * to doing what KFS does, we are good. 24 | * -- for deployment purposes with KFS, this interface is 25 | * implemented by the KfsImpl object. 26 | */ 27 | 28 | package org.apache.hadoop.fs.kfs; 29 | 30 | import java.io.*; 31 | 32 | import org.apache.hadoop.fs.FSDataInputStream; 33 | import org.apache.hadoop.fs.FSDataOutputStream; 34 | import org.apache.hadoop.fs.FileStatus; 35 | import org.apache.hadoop.fs.Path; 36 | 37 | interface IFSImpl { 38 | public boolean exists(String path) throws IOException; 39 | public boolean isDirectory(String path) throws IOException; 40 | public boolean isFile(String path) throws IOException; 41 | public String[] readdir(String path) throws IOException; 42 | public FileStatus[] readdirplus(Path path) throws IOException; 43 | 44 | public int mkdirs(String path) throws IOException; 45 | public int rename(String source, String dest) throws IOException; 46 | 47 | public int rmdir(String path) throws IOException; 48 | public int remove(String path) throws IOException; 49 | public long filesize(String path) throws IOException; 50 | public short getReplication(String path) throws IOException; 51 | public short setReplication(String path, short replication) throws IOException; 52 | public String[][] getDataLocation(String path, long start, long len) throws IOException; 53 | 54 | public long getModificationTime(String path) throws IOException; 55 | public FSDataOutputStream create(String path, short replication, int bufferSize) throws IOException; 56 | public FSDataInputStream open(String path, int bufferSize) throws IOException; 57 | 58 | }; 59 | -------------------------------------------------------------------------------- /kfs-hadoop/0.16x/src/java/org/apache/hadoop/fs/kfs/KFSOutputStream.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed under the Apache License, Version 2.0 4 | * (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | * implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | * 15 | * @author: Sriram Rao (Kosmix Corp.) 16 | * 17 | * Implements the Hadoop FSOutputStream interfaces to allow applications to write to 18 | * files in Kosmos File System (KFS). 19 | */ 20 | 21 | package org.apache.hadoop.fs.kfs; 22 | 23 | import java.io.*; 24 | import java.net.*; 25 | import java.util.*; 26 | import java.nio.ByteBuffer; 27 | 28 | import org.apache.hadoop.conf.Configuration; 29 | import org.apache.hadoop.fs.Path; 30 | import org.apache.hadoop.fs.FSDataOutputStream; 31 | import org.apache.hadoop.util.Progressable; 32 | 33 | import org.kosmix.kosmosfs.access.KfsAccess; 34 | import org.kosmix.kosmosfs.access.KfsOutputChannel; 35 | 36 | class KFSOutputStream extends OutputStream { 37 | 38 | private String path; 39 | private KfsOutputChannel kfsChannel; 40 | 41 | public KFSOutputStream(KfsAccess kfsAccess, String path, short replication) { 42 | this.path = path; 43 | 44 | this.kfsChannel = kfsAccess.kfs_create(path, replication); 45 | } 46 | 47 | public long getPos() throws IOException { 48 | if (kfsChannel == null) { 49 | throw new IOException("File closed"); 50 | } 51 | return kfsChannel.tell(); 52 | } 53 | 54 | public void write(int v) throws IOException { 55 | if (kfsChannel == null) { 56 | throw new IOException("File closed"); 57 | } 58 | byte[] b = new byte[1]; 59 | 60 | b[0] = (byte) v; 61 | write(b, 0, 1); 62 | } 63 | 64 | public void write(byte b[], int off, int len) throws IOException { 65 | if (kfsChannel == null) { 66 | throw new IOException("File closed"); 67 | } 68 | 69 | kfsChannel.write(ByteBuffer.wrap(b, off, len)); 70 | } 71 | 72 | public void flush() throws IOException { 73 | if (kfsChannel == null) { 74 | throw new IOException("File closed"); 75 | } 76 | kfsChannel.sync(); 77 | } 78 | 79 | public synchronized void close() throws IOException { 80 | if (kfsChannel == null) { 81 | return; 82 | } 83 | flush(); 84 | kfsChannel.close(); 85 | kfsChannel = null; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /kfs-hadoop/0.17x/src/java/org/apache/hadoop/fs/kfs/IFSImpl.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed under the Apache License, Version 2.0 4 | * (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | * implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | * 15 | * @author: Sriram Rao (Kosmix Corp.) 16 | * 17 | * We need to provide the ability to the code in fs/kfs without really 18 | * having a KFS deployment. In particular, the glue code that wraps 19 | * around calls to KfsAccess object. This is accomplished by defining a 20 | * filesystem implementation interface: 21 | * -- for testing purposes, a dummy implementation of this interface 22 | * will suffice; as long as the dummy implementation is close enough 23 | * to doing what KFS does, we are good. 24 | * -- for deployment purposes with KFS, this interface is 25 | * implemented by the KfsImpl object. 26 | */ 27 | 28 | package org.apache.hadoop.fs.kfs; 29 | 30 | import java.io.*; 31 | 32 | import org.apache.hadoop.fs.FSDataInputStream; 33 | import org.apache.hadoop.fs.FSDataOutputStream; 34 | import org.apache.hadoop.fs.FileStatus; 35 | import org.apache.hadoop.fs.Path; 36 | 37 | interface IFSImpl { 38 | public boolean exists(String path) throws IOException; 39 | public boolean isDirectory(String path) throws IOException; 40 | public boolean isFile(String path) throws IOException; 41 | public String[] readdir(String path) throws IOException; 42 | public FileStatus[] readdirplus(Path path) throws IOException; 43 | 44 | public int mkdirs(String path) throws IOException; 45 | public int rename(String source, String dest) throws IOException; 46 | 47 | public int rmdir(String path) throws IOException; 48 | public int remove(String path) throws IOException; 49 | public long filesize(String path) throws IOException; 50 | public short getReplication(String path) throws IOException; 51 | public short setReplication(String path, short replication) throws IOException; 52 | public String[][] getDataLocation(String path, long start, long len) throws IOException; 53 | 54 | public long getModificationTime(String path) throws IOException; 55 | public FSDataOutputStream create(String path, short replication, int bufferSize) throws IOException; 56 | public FSDataInputStream open(String path, int bufferSize) throws IOException; 57 | 58 | }; 59 | -------------------------------------------------------------------------------- /kfs-hadoop/0.17x/src/java/org/apache/hadoop/fs/kfs/KFSOutputStream.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed under the Apache License, Version 2.0 4 | * (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | * implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | * 15 | * @author: Sriram Rao (Kosmix Corp.) 16 | * 17 | * Implements the Hadoop FSOutputStream interfaces to allow applications to write to 18 | * files in Kosmos File System (KFS). 19 | */ 20 | 21 | package org.apache.hadoop.fs.kfs; 22 | 23 | import java.io.*; 24 | import java.net.*; 25 | import java.util.*; 26 | import java.nio.ByteBuffer; 27 | 28 | import org.apache.hadoop.conf.Configuration; 29 | import org.apache.hadoop.fs.Path; 30 | import org.apache.hadoop.fs.FSDataOutputStream; 31 | import org.apache.hadoop.util.Progressable; 32 | 33 | import org.kosmix.kosmosfs.access.KfsAccess; 34 | import org.kosmix.kosmosfs.access.KfsOutputChannel; 35 | 36 | class KFSOutputStream extends OutputStream { 37 | 38 | private String path; 39 | private KfsOutputChannel kfsChannel; 40 | 41 | public KFSOutputStream(KfsAccess kfsAccess, String path, short replication) { 42 | this.path = path; 43 | 44 | this.kfsChannel = kfsAccess.kfs_create(path, replication); 45 | } 46 | 47 | public long getPos() throws IOException { 48 | if (kfsChannel == null) { 49 | throw new IOException("File closed"); 50 | } 51 | return kfsChannel.tell(); 52 | } 53 | 54 | public void write(int v) throws IOException { 55 | if (kfsChannel == null) { 56 | throw new IOException("File closed"); 57 | } 58 | byte[] b = new byte[1]; 59 | 60 | b[0] = (byte) v; 61 | write(b, 0, 1); 62 | } 63 | 64 | public void write(byte b[], int off, int len) throws IOException { 65 | if (kfsChannel == null) { 66 | throw new IOException("File closed"); 67 | } 68 | 69 | kfsChannel.write(ByteBuffer.wrap(b, off, len)); 70 | } 71 | 72 | public void flush() throws IOException { 73 | if (kfsChannel == null) { 74 | throw new IOException("File closed"); 75 | } 76 | kfsChannel.sync(); 77 | } 78 | 79 | public synchronized void close() throws IOException { 80 | if (kfsChannel == null) { 81 | return; 82 | } 83 | flush(); 84 | kfsChannel.close(); 85 | kfsChannel = null; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /kfs-hadoop/0.18x/src/core/org/apache/hadoop/fs/kfs/IFSImpl.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed under the Apache License, Version 2.0 4 | * (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | * implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | * 15 | * @author: Sriram Rao (Kosmix Corp.) 16 | * 17 | * We need to provide the ability to the code in fs/kfs without really 18 | * having a KFS deployment. In particular, the glue code that wraps 19 | * around calls to KfsAccess object. This is accomplished by defining a 20 | * filesystem implementation interface: 21 | * -- for testing purposes, a dummy implementation of this interface 22 | * will suffice; as long as the dummy implementation is close enough 23 | * to doing what KFS does, we are good. 24 | * -- for deployment purposes with KFS, this interface is 25 | * implemented by the KfsImpl object. 26 | */ 27 | 28 | package org.apache.hadoop.fs.kfs; 29 | 30 | import java.io.*; 31 | 32 | import org.apache.hadoop.fs.FSDataInputStream; 33 | import org.apache.hadoop.fs.FSDataOutputStream; 34 | 35 | interface IFSImpl { 36 | public boolean exists(String path) throws IOException; 37 | public boolean isDirectory(String path) throws IOException; 38 | public boolean isFile(String path) throws IOException; 39 | public String[] readdir(String path) throws IOException; 40 | 41 | public int mkdirs(String path) throws IOException; 42 | public int rename(String source, String dest) throws IOException; 43 | 44 | public int rmdir(String path) throws IOException; 45 | public int remove(String path) throws IOException; 46 | public long filesize(String path) throws IOException; 47 | public short getReplication(String path) throws IOException; 48 | public short setReplication(String path, short replication) throws IOException; 49 | public String[][] getDataLocation(String path, long start, long len) throws IOException; 50 | 51 | public long getModificationTime(String path) throws IOException; 52 | public FSDataOutputStream create(String path, short replication, int bufferSize) throws IOException; 53 | public FSDataInputStream open(String path, int bufferSize) throws IOException; 54 | 55 | }; 56 | -------------------------------------------------------------------------------- /kfs-hadoop/0.18x/src/core/org/apache/hadoop/fs/kfs/KFSOutputStream.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed under the Apache License, Version 2.0 4 | * (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | * implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | * 15 | * @author: Sriram Rao (Kosmix Corp.) 16 | * 17 | * Implements the Hadoop FSOutputStream interfaces to allow applications to write to 18 | * files in Kosmos File System (KFS). 19 | */ 20 | 21 | package org.apache.hadoop.fs.kfs; 22 | 23 | import java.io.*; 24 | import java.net.*; 25 | import java.util.*; 26 | import java.nio.ByteBuffer; 27 | 28 | import org.apache.hadoop.conf.Configuration; 29 | import org.apache.hadoop.fs.Path; 30 | import org.apache.hadoop.fs.FSDataOutputStream; 31 | import org.apache.hadoop.util.Progressable; 32 | 33 | import org.kosmix.kosmosfs.access.KfsAccess; 34 | import org.kosmix.kosmosfs.access.KfsOutputChannel; 35 | 36 | class KFSOutputStream extends OutputStream { 37 | 38 | private String path; 39 | private KfsOutputChannel kfsChannel; 40 | 41 | public KFSOutputStream(KfsAccess kfsAccess, String path, short replication) { 42 | this.path = path; 43 | 44 | this.kfsChannel = kfsAccess.kfs_create(path, replication); 45 | } 46 | 47 | public long getPos() throws IOException { 48 | if (kfsChannel == null) { 49 | throw new IOException("File closed"); 50 | } 51 | return kfsChannel.tell(); 52 | } 53 | 54 | public void write(int v) throws IOException { 55 | if (kfsChannel == null) { 56 | throw new IOException("File closed"); 57 | } 58 | byte[] b = new byte[1]; 59 | 60 | b[0] = (byte) v; 61 | write(b, 0, 1); 62 | } 63 | 64 | public void write(byte b[], int off, int len) throws IOException { 65 | if (kfsChannel == null) { 66 | throw new IOException("File closed"); 67 | } 68 | 69 | kfsChannel.write(ByteBuffer.wrap(b, off, len)); 70 | } 71 | 72 | public void flush() throws IOException { 73 | if (kfsChannel == null) { 74 | throw new IOException("File closed"); 75 | } 76 | kfsChannel.sync(); 77 | } 78 | 79 | public synchronized void close() throws IOException { 80 | if (kfsChannel == null) { 81 | return; 82 | } 83 | flush(); 84 | kfsChannel.close(); 85 | kfsChannel = null; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /scripts.mac_solaris/kfsshell.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # $Id: kfsshell.py 24 2007-09-27 07:17:06Z sriramsrao $ 4 | # 5 | # Copyright 2007 Kosmix Corp. 6 | # 7 | # This file is part of Kosmos File System (KFS). 8 | # 9 | # Licensed under the Apache License, Version 2.0 10 | # (the "License"); you may not use this file except in compliance with 11 | # the License. You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 18 | # implied. See the License for the specific language governing 19 | # permissions and limitations under the License. 20 | # 21 | # Script that launches KfsShell: get the location of the metaserver 22 | # from the machines.cfg file and launch KfsShell 23 | # 24 | # Look for /tools/KfsShell 25 | # 26 | # Use machines.cfg 27 | # 28 | 29 | import os,os.path,sys,getopt 30 | from ConfigParser import ConfigParser 31 | 32 | def usage(): 33 | print "%s [-f, --file ] [ -b, --bin ]\n" % sys.argv[0] 34 | 35 | if __name__ == '__main__': 36 | (opts, args) = getopt.getopt(sys.argv[1:], "b:f:h", 37 | ["bin=", "file=", "help"]) 38 | op = "" 39 | filename = "" 40 | bindir = "" 41 | for (o, a) in opts: 42 | if o in ("-h", "--help"): 43 | usage() 44 | sys.exit(2) 45 | if o in ("-f", "--file"): 46 | filename = a 47 | elif o in ("-b", "--bin"): 48 | bindir = a 49 | 50 | if not os.path.exists(filename): 51 | print "%s : config file doesn't exist\n" % filename 52 | sys.exit(-1) 53 | 54 | if not os.path.exists(bindir): 55 | print "%s : bindir doesn't exist\n" % bindir 56 | sys.exit(-1) 57 | 58 | config = ConfigParser() 59 | config.readfp(open(filename, 'r')) 60 | if not config.has_section('metaserver'): 61 | raise config.NoSectionError, "No metaserver section" 62 | 63 | node = config.get('metaserver', 'node') 64 | port = config.getint('metaserver', 'baseport') 65 | cmd = "%s/tools/KfsShell -s %s -p %d" % (bindir, node, port) 66 | os.system(cmd) 67 | 68 | -------------------------------------------------------------------------------- /scripts.mac_solaris/machines.cfg: -------------------------------------------------------------------------------- 1 | # KFS Machine configuration file 2 | [metaserver] 3 | node: dev104 4 | rundir: /home/sriram/kfs/meta 5 | baseport: 20000 6 | backup_path: dev105:/home/sriram/kfs/meta_bkup 7 | [chunkserver1] 8 | node: dev100 9 | rundir: /home/sriram/kfs/chunk 10 | chunkDir: /home/sriram/kfs/chunk/bin/kfschunk1 /home/sriram/kfs/chunk/bin/kfschunk2 11 | baseport: 30000 12 | space: 200 G 13 | [chunkserver2] 14 | node: dev101 15 | rundir: /home/sriram/kfs/chunk 16 | chunkDir: /home/sriram/kfs/chunk/bin/kfschunk1 /home/sriram/kfs/chunk/bin/kfschunk2 17 | baseport: 30000 18 | space: 200 G 19 | [chunkserver3] 20 | node: dev102 21 | rundir: /home/sriram/kfs/chunk 22 | chunkDir: /home/sriram/kfs/chunk/bin/kfschunk1 /home/sriram/kfs/chunk/bin/kfschunk2 23 | baseport: 30000 24 | space: 200 G 25 | [chunkserver4] 26 | node: dev103 27 | rundir: /home/sriram/kfs/chunk 28 | chunkDir: /home/sriram/kfs/chunk/bin/kfschunk1 /home/sriram/kfs/chunk/bin/kfschunk2 29 | baseport: 30000 30 | space: 200 G 31 | [chunkserver5] 32 | node: dev104 33 | rundir: /home/sriram/kfs/chunk 34 | baseport: 30000 35 | space: 200 G 36 | [chunkserver6] 37 | node: dev105 38 | rundir: /home/sriram/kfs/chunk 39 | chunkDir: /home/sriram/kfs/chunk/bin/kfschunk1 /home/sriram/kfs/chunk/bin/kfschunk2 40 | baseport: 30000 41 | space: 200 G 42 | [chunkserver7] 43 | node: dev107 44 | rundir: /home/sriram/kfs/chunk 45 | baseport: 30000 46 | space: 200 G 47 | -------------------------------------------------------------------------------- /scripts.mac_solaris/machines.cfg.local: -------------------------------------------------------------------------------- 1 | # KFS Machine configuration file 2 | [metaserver] 3 | node: localhost 4 | rundir: /export/home/sriram/kfs/meta 5 | baseport: 20000 6 | [chunkserver1] 7 | node: localhost 8 | rundir: /export/home/sriram/kfs/chunk1 9 | chunkDir: /export/home/sriram/kfs/chunk1/bin/kfschunk1 /export/home/sriram/kfs/chunk1/bin/kfschunk2 10 | baseport: 30000 11 | space: 200 G 12 | -------------------------------------------------------------------------------- /scripts.mac_solaris/metabkup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | # 3 | # $Id: metabkup.sh 36 2007-11-12 02:43:36Z sriramsrao $ 4 | # 5 | # 6 | # This file is part of Kosmos File System (KFS). 7 | # 8 | # Licensed under the Apache License, Version 2.0 9 | # (the "License"); you may not use this file except in compliance with 10 | # the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | # implied. See the License for the specific language governing 18 | # permissions and limitations under the License. 19 | # 20 | # Script to copy the metaserver checkpoint files to a remote node. 21 | # The same script can also be used to restore the checkpoint files 22 | # from remote path to local. 23 | # 24 | 25 | # process any command-line arguments 26 | # TEMP=`getopt -o d:b:R:h -l dir:,backup:,recover:help -n metabkup.sh -- "$@"` 27 | # eval set -- "$TEMP" 28 | 29 | set -- `getopt d:b:p:R:h $*` 30 | 31 | recover=0 32 | backup_node= 33 | backup_path= 34 | # while true 35 | for i in $* 36 | do 37 | case "$i" in 38 | -d|--dir) kfs_dir=$2;; 39 | -b|--backup_node) backup_node=$2;; 40 | -p|--backup_path) backup_path=$2;; 41 | -R|--recover) recover=1;; 42 | -h|--help) echo "usage: $0 [-d kfsdir] [-b backup_node] [-p backup_path] {-recover}"; exit ;; 43 | --) break ;; 44 | esac 45 | shift 46 | done 47 | 48 | cpdir="$kfs_dir/bin/kfscp" 49 | logdir="$kfs_dir/bin/kfslog" 50 | if [ ! -d $cpdir ]; 51 | then 52 | echo "$cpdir is non-existent" 53 | exit -1 54 | fi 55 | 56 | if [ $recover -eq 0 ]; 57 | then 58 | rsync -avz --delete $cpdir $backup_node:$backup_path 59 | rsync -avz $logdir $backup_node:$backup_path 60 | else 61 | # Restore the checkpoint files from remote node 62 | rsync -avz $backup_node:$backup_path/"kfscp" . 63 | rsync -avz $backup_node:$backup_path/"kfslog" . 64 | fi 65 | -------------------------------------------------------------------------------- /scripts.mac_solaris/metalogprune.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # $Id: metalogprune.py 117 2008-08-05 20:51:44Z sriramsrao $ 4 | # 5 | # Copyright 2008 Quantcast Corp. 6 | # 7 | # This file is part of Kosmos File System (KFS). 8 | # 9 | # Licensed under the Apache License, Version 2.0 10 | # (the "License"); you may not use this file except in compliance with 11 | # the License. You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 18 | # implied. See the License for the specific language governing 19 | # permissions and limitations under the License. 20 | # 21 | # \file metalogprune.py 22 | # \brief KFS log and checkpoint housekeeping 23 | # 24 | # We gzip and keep all the old log files around. This is a bit of an 25 | # overkill---we can prune away files that are no longer referenced by 26 | # any checkpoint. We find the oldest checkpoint file and find that 27 | # the log it references; files older than that log file are deleted. 28 | # 29 | import os 30 | import sys 31 | import glob 32 | import stat 33 | import time 34 | import getopt 35 | import gzip 36 | 37 | def age(file): 38 | """return age of file (last mtime) in seconds""" 39 | 40 | now = time.time() 41 | return now - os.stat(file)[stat.ST_MTIME] 42 | 43 | def orderByAge(this, that): 44 | if age(this) > age(that): 45 | return this 46 | return that 47 | 48 | def olderThanLog(logfile, lognum): 49 | """Return True if logfile which is of the form log.# has a 50 | sequence number less than lognum""" 51 | (base, extn) = os.path.splitext(logfile) 52 | extn = extn[1:] 53 | if extn == 'gz': 54 | val = int(os.path.splitext(base)[1][1:]) 55 | else: 56 | val = int(extn) 57 | return val < lognum 58 | 59 | def prunefiles(cpdir, logdir): 60 | """Find the log file that is referenced by the oldest CP file. 61 | Log files that are older than that one can be deleted.""" 62 | oldest = reduce(orderByAge, glob.glob(cpdir + '/chkpt.*')) 63 | if oldest is None: 64 | return 65 | print "Oldest cp: %s" % oldest 66 | # get the log file 67 | for l in open(oldest).xreadlines(): 68 | if l.startswith('log/'): 69 | lognum = int(os.path.splitext(l[4:])[1][1:]) 70 | print lognum 71 | alllogfiles = glob.glob(logdir + '/log.*') 72 | oldones = [f for f in alllogfiles if olderThanLog(f, lognum)] 73 | for f in oldones: 74 | os.remove(f) 75 | break 76 | 77 | if (__name__ == "__main__"): 78 | if len(sys.argv) != 3: 79 | raise getopt.GetoptError, "missing arguments" 80 | 81 | # kfscpdir, kfslogdir 82 | prunefiles(sys.argv[1], sys.argv[2]) 83 | -------------------------------------------------------------------------------- /scripts.mac_solaris/slaves.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # $Id: slaves.py 181 2008-10-03 23:59:12Z sriramsrao $ 4 | # 5 | # Copyright 2008 Quantcast Corp. 6 | # Author: Sriram Rao 7 | # 8 | # This file is part of Kosmos File System (KFS). 9 | # 10 | # Licensed under the Apache License, Version 2.0 11 | # (the "License"); you may not use this file except in compliance with 12 | # the License. You may obtain a copy of the License at 13 | # 14 | # http://www.apache.org/licenses/LICENSE-2.0 15 | # 16 | # Unless required by applicable law or agreed to in writing, software 17 | # distributed under the License is distributed on an "AS IS" BASIS, 18 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | # implied. See the License for the specific language governing 20 | # permissions and limitations under the License. 21 | # 22 | # Script that logs into a node and runs a command. 23 | # Use for checking liveness of a given node 24 | # 25 | 26 | import os,sys,popen2 27 | import threading 28 | 29 | class Pinger(threading.Thread): 30 | def __init__(self, node, cmd): 31 | threading.Thread.__init__(self) 32 | self.node = node 33 | self.cmd = cmd 34 | 35 | def run(self): 36 | cmd = "ssh -o StrictHostKeyChecking=no %s %s " % (self.node, self.cmd) 37 | p = popen2.Popen3(cmd, True) 38 | for out in p.fromchild: 39 | if len(out) > 1: 40 | print out.strip() 41 | 42 | if __name__ == '__main__': 43 | if len(sys.argv) != 3: 44 | print "Usage: slaves.py " 45 | sys.exit(0) 46 | 47 | pingers = [] 48 | for l in open(sys.argv[1], 'r').readlines(): 49 | w = Pinger(l.strip(), sys.argv[2]) 50 | pingers.append(w) 51 | 52 | numPerRound = 100 53 | for i in xrange(0, len(pingers), numPerRound): 54 | for j in xrange(0, numPerRound): 55 | if i + j >= len(pingers): 56 | break 57 | pingers[i + j].start() 58 | for j in xrange(0, numPerRound): 59 | if i + j >= len(pingers): 60 | break 61 | pingers[i + j].join(15) 62 | print "Done with ", i + numPerRound, " workers" 63 | -------------------------------------------------------------------------------- /scripts/kfs.spec: -------------------------------------------------------------------------------- 1 | %define debug_package %{nil} 2 | %define debug_packages %{nil} 3 | 4 | Summary: KFS Binary Package 5 | Name: kfs 6 | Version: 0.2.3 7 | Release: 1 8 | License: Apache 9 | Group: System Environment 10 | URL: http://kosmosfs.sourceforge.net 11 | Source0: %{name}-%{version}.tar.gz 12 | #BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root 13 | BuildRoot: %{_builddir} 14 | 15 | %description 16 | This package contains C++ binary distribution of KFS 17 | 18 | %prep 19 | %setup -q 20 | 21 | %build 22 | 23 | %install 24 | #rm -rf $RPM_BUILD_ROOT 25 | 26 | %clean 27 | rm -rf $RPM_BUILD_ROOT 28 | 29 | %post 30 | 31 | 32 | %files 33 | %defattr(-,root,root,-) 34 | %doc 35 | /%{name}-%{version}/bin/* 36 | /%{name}-%{version}/lib/* 37 | 38 | 39 | %changelog 40 | * Thu Nov 27 2008 Sriram Rao - 41 | - Initial build. 42 | 43 | -------------------------------------------------------------------------------- /scripts/kfsshell.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # $Id: kfsshell.py 400 2010-08-22 06:07:05Z sriramsrao $ 4 | # 5 | # Copyright 2007 Kosmix Corp. 6 | # 7 | # This file is part of Kosmos File System (KFS). 8 | # 9 | # Licensed under the Apache License, Version 2.0 10 | # (the "License"); you may not use this file except in compliance with 11 | # the License. You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 18 | # implied. See the License for the specific language governing 19 | # permissions and limitations under the License. 20 | # 21 | # Script that launches KfsShell: get the location of the metaserver 22 | # from the machines.cfg file and launch KfsShell 23 | # 24 | # Look for /tools/KfsShell 25 | # 26 | # Use machines.cfg 27 | # 28 | 29 | import os,os.path,sys,getopt 30 | from ConfigParser import ConfigParser 31 | 32 | def usage(): 33 | print "%s [-f, --file ] [ -b, --bin ]\n" % sys.argv[0] 34 | 35 | if __name__ == '__main__': 36 | (opts, args) = getopt.getopt(sys.argv[1:], "b:f:h", 37 | ["bin=", "file=", "help"]) 38 | op = "" 39 | filename = "" 40 | bindir = "" 41 | for (o, a) in opts: 42 | if o in ("-h", "--help"): 43 | usage() 44 | sys.exit(2) 45 | if o in ("-f", "--file"): 46 | filename = a 47 | elif o in ("-b", "--bin"): 48 | bindir = a 49 | 50 | if not os.path.exists(filename): 51 | print "%s : config file doesn't exist\n" % filename 52 | sys.exit(-1) 53 | 54 | if not os.path.exists(bindir): 55 | print "%s : bindir doesn't exist\n" % bindir 56 | sys.exit(-1) 57 | 58 | config = ConfigParser() 59 | config.readfp(open(filename, 'r')) 60 | if not config.has_section('metaserver'): 61 | raise config.NoSectionError, "No metaserver section" 62 | 63 | node = config.get('metaserver', 'node') 64 | port = config.getint('metaserver', 'baseport') 65 | cmd = "%s/tools/kfsshell -s %s -p %d" % (bindir, node, port) 66 | os.system(cmd) 67 | 68 | -------------------------------------------------------------------------------- /scripts/machines.cfg: -------------------------------------------------------------------------------- 1 | # KFS Machine configuration file 2 | [metaserver] 3 | node: dev104 4 | rundir: /home/sriram/kfs/meta 5 | baseport: 20000 6 | backup_path: dev105:/home/sriram/kfs/meta_bkup 7 | [chunkserver1] 8 | node: dev100 9 | rundir: /home/sriram/kfs/chunk 10 | chunkDir: /home/sriram/kfs/chunk/bin/kfschunk1 /home/sriram/kfs/chunk/bin/kfschunk2 11 | baseport: 30000 12 | space: 200 G 13 | [chunkserver2] 14 | node: dev101 15 | rundir: /home/sriram/kfs/chunk 16 | chunkDir: /home/sriram/kfs/chunk/bin/kfschunk1 /home/sriram/kfs/chunk/bin/kfschunk2 17 | baseport: 30000 18 | space: 200 G 19 | [chunkserver3] 20 | node: dev102 21 | rundir: /home/sriram/kfs/chunk 22 | chunkDir: /home/sriram/kfs/chunk/bin/kfschunk1 /home/sriram/kfs/chunk/bin/kfschunk2 23 | baseport: 30000 24 | space: 200 G 25 | [chunkserver4] 26 | node: dev103 27 | rundir: /home/sriram/kfs/chunk 28 | chunkDir: /home/sriram/kfs/chunk/bin/kfschunk1 /home/sriram/kfs/chunk/bin/kfschunk2 29 | baseport: 30000 30 | space: 200 G 31 | [chunkserver5] 32 | node: dev104 33 | rundir: /home/sriram/kfs/chunk 34 | baseport: 30000 35 | space: 200 G 36 | [chunkserver6] 37 | node: dev105 38 | rundir: /home/sriram/kfs/chunk 39 | chunkDir: /home/sriram/kfs/chunk/bin/kfschunk1 /home/sriram/kfs/chunk/bin/kfschunk2 40 | baseport: 30000 41 | space: 200 G 42 | [chunkserver7] 43 | node: dev107 44 | rundir: /home/sriram/kfs/chunk 45 | baseport: 30000 46 | space: 200 G 47 | -------------------------------------------------------------------------------- /scripts/metabkup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # $Id: metabkup.sh 94 2008-07-22 17:29:02Z sriramsrao $ 4 | # 5 | # 6 | # This file is part of Kosmos File System (KFS). 7 | # 8 | # Licensed under the Apache License, Version 2.0 9 | # (the "License"); you may not use this file except in compliance with 10 | # the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | # implied. See the License for the specific language governing 18 | # permissions and limitations under the License. 19 | # 20 | # Script to copy the metaserver checkpoint files to a remote node. 21 | # The same script can also be used to restore the checkpoint files 22 | # from remote path to local. 23 | # 24 | 25 | # process any command-line arguments 26 | TEMP=`getopt -o d:b:p:R:h -l dir:,backup_node:,backup_path:,recover:help -n metabkup.sh -- "$@"` 27 | eval set -- "$TEMP" 28 | 29 | recover=0 30 | backup_node= 31 | backup_path= 32 | while true 33 | do 34 | case "$1" in 35 | -d|--dir) kfs_dir=$2; shift 2;; 36 | -b|--backup_node) backup_node=$2; shift 2;; 37 | -p|--backup_path) backup_path=$2; shift 2;; 38 | -R|--recover) recover=1; shift;; 39 | -h|--help) echo "usage: $0 [-d cpdir] [-b backup_node] [-p backup_path] {-recover}"; exit ;; 40 | --) shift; break ;; 41 | esac 42 | done 43 | 44 | cpdir="$kfs_dir/bin/kfscp" 45 | logdir="$kfs_dir/bin/kfslog" 46 | if [ ! -d $cpdir ]; 47 | then 48 | echo "$cpdir is non-existent" 49 | exit -1 50 | fi 51 | 52 | if [ $recover -eq 0 ]; 53 | then 54 | rsync -avz --delete $cpdir $backup_node:$backup_path 55 | rsync -avz $logdir $backup_node:$backup_path 56 | else 57 | # Restore the checkpoint files from remote node 58 | rsync -avz $backup_node:$backup_path/"kfscp" . 59 | rsync -avz $backup_node:$backup_path/"kfslog" . 60 | fi 61 | -------------------------------------------------------------------------------- /scripts/slaves.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # $Id: slaves.py 181 2008-10-03 23:59:12Z sriramsrao $ 4 | # 5 | # Copyright 2008 Quantcast Corp. 6 | # Author: Sriram Rao 7 | # 8 | # This file is part of Kosmos File System (KFS). 9 | # 10 | # Licensed under the Apache License, Version 2.0 11 | # (the "License"); you may not use this file except in compliance with 12 | # the License. You may obtain a copy of the License at 13 | # 14 | # http://www.apache.org/licenses/LICENSE-2.0 15 | # 16 | # Unless required by applicable law or agreed to in writing, software 17 | # distributed under the License is distributed on an "AS IS" BASIS, 18 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | # implied. See the License for the specific language governing 20 | # permissions and limitations under the License. 21 | # 22 | # Script that logs into a node and runs a command. 23 | # Use for checking liveness of a given node 24 | # 25 | 26 | import os,sys,popen2 27 | import threading 28 | 29 | class Pinger(threading.Thread): 30 | def __init__(self, node, cmd): 31 | threading.Thread.__init__(self) 32 | self.node = node 33 | self.cmd = cmd 34 | 35 | def run(self): 36 | cmd = "ssh -o StrictHostKeyChecking=no %s %s " % (self.node, self.cmd) 37 | p = popen2.Popen3(cmd, True) 38 | for out in p.fromchild: 39 | if len(out) > 1: 40 | print out.strip() 41 | 42 | if __name__ == '__main__': 43 | if len(sys.argv) != 3: 44 | print "Usage: slaves.py " 45 | sys.exit(0) 46 | 47 | pingers = [] 48 | for l in open(sys.argv[1], 'r').readlines(): 49 | w = Pinger(l.strip(), sys.argv[2]) 50 | pingers.append(w) 51 | 52 | numPerRound = 100 53 | for i in xrange(0, len(pingers), numPerRound): 54 | for j in xrange(0, numPerRound): 55 | if i + j >= len(pingers): 56 | break 57 | pingers[i + j].start() 58 | for j in xrange(0, numPerRound): 59 | if i + j >= len(pingers): 60 | break 61 | pingers[i + j].join(15) 62 | print "Done with ", i + numPerRound, " workers" 63 | -------------------------------------------------------------------------------- /src/cc/access/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | # Take all the .cc files and build a library out of them 3 | add_library (kfs_access SHARED KfsAccessJni.cc) 4 | add_dependencies (kfs_access kfsClient-shared) 5 | target_link_libraries (kfs_access kfsClient-shared) 6 | if (NOT APPLE) 7 | target_link_libraries(kfs_access rt) 8 | endif (NOT APPLE) 9 | 10 | # 11 | install (TARGETS kfs_access 12 | LIBRARY DESTINATION lib 13 | ARCHIVE DESTINATION lib-static) 14 | -------------------------------------------------------------------------------- /src/cc/access/kfs_setup.py: -------------------------------------------------------------------------------- 1 | # 2 | # $Id: kfs_setup.py 1552 2011-01-06 22:21:54Z sriramr $ 3 | # 4 | # Use the distutils setup function to build and install the KFS module. 5 | # Execute this as: 6 | # python kfs_setup.py ~/code/kfs/build/lib/ build 7 | # and this will build kfs.so in ./build/.../kfs.so 8 | # This needs to be installed /usr/lib64/python/site-packages or in an 9 | # alternate location; see COMPILING for instructions 10 | # In addition, ~/code/kfs/build/lib needs to be in the LD_LIBRARY_PATH 11 | # After installation, python apps can access kfs. 12 | # 13 | from distutils.core import setup, Extension 14 | import sys 15 | 16 | kfs_lib_dir = sys.argv[1] 17 | del sys.argv[1] 18 | 19 | kfsext = Extension('kfs', 20 | include_dirs = ['/home/srao/p4trees/kfssort/kosmosfs/src/cc/', '/usr/local/include/boost-1_37/'], 21 | libraries = ['kfsClient'], 22 | library_dirs = [kfs_lib_dir], 23 | runtime_library_dirs = ['/home/qmr_ksort/sortmaster/lib'], 24 | sources = ['KfsModulePy.cc']) 25 | 26 | setup(name = "kfs", version = "0.3", 27 | description="KFS client module", 28 | author="Blake Lewis and Sriram Rao", 29 | ext_modules = [kfsext]) 30 | -------------------------------------------------------------------------------- /src/cc/chunk/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | # build libchunk 3 | set (lib_srcs 4 | AtomicRecordAppender.cc 5 | BufferManager.cc 6 | ChunkSortHelper.cc 7 | ChunkManager.cc 8 | ChunkServer.cc 9 | ClientManager.cc 10 | ClientSM.cc 11 | DiskIo.cc 12 | KfsOps.cc 13 | LeaseClerk.cc 14 | Logger.cc 15 | MetaServerSM.cc 16 | RemoteSyncSM.cc 17 | Replicator.cc 18 | Utils.cc 19 | ) 20 | 21 | add_library (kchunklib STATIC ${lib_srcs}) 22 | add_library (kchunklib-shared SHARED ${lib_srcs}) 23 | set_target_properties (kchunklib-shared PROPERTIES OUTPUT_NAME "kchunklib") 24 | set_target_properties (kchunklib PROPERTIES CLEAN_DIRECT_OUTPUT 1) 25 | set_target_properties (kchunklib-shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) 26 | 27 | add_executable (chunkserver ChunkServer_main.cc) 28 | add_executable (chunkscrubber chunkscrubber_main.cc) 29 | #add_executable (chunktrimmer chunktrimmer_main.cc) 30 | 31 | set (exe_files chunkserver chunkscrubber) #chunktrimmer) 32 | 33 | foreach (exe_file ${exe_files}) 34 | if (USE_STATIC_LIB_LINKAGE) 35 | target_link_libraries (${exe_file} kchunklib kfsIO kfsCommon qcdio pthread crypto lzo2) 36 | add_dependencies (${exe_file} kfsCommon kfsIO qcdio) 37 | else (USE_STATIC_LIB_LINKAGE) 38 | target_link_libraries (${exe_file} kchunklib-shared kfsIO-shared kfsCommon-shared qcdio-shared pthread crypto lzo2) 39 | add_dependencies (${exe_file} kfsCommon-shared kfsIO-shared qcdio-shared) 40 | endif (USE_STATIC_LIB_LINKAGE) 41 | endforeach (exe_file) 42 | 43 | if (APPLE) 44 | target_link_libraries(kchunklib-shared kfsIO-shared kfsCommon-shared qcdio-shared crypto lzo2) 45 | endif(APPLE) 46 | 47 | if (NOT APPLE) 48 | target_link_libraries(chunkserver rt) 49 | endif (NOT APPLE) 50 | 51 | if (CMAKE_SYSTEM_NAME STREQUAL "SunOS") 52 | target_link_libraries(chunkserver umem) 53 | endif (CMAKE_SYSTEM_NAME STREQUAL "SunOS") 54 | 55 | # 56 | # Install them 57 | # 58 | install (TARGETS ${exe_files} kchunklib-shared 59 | RUNTIME DESTINATION bin 60 | LIBRARY DESTINATION lib 61 | ARCHIVE DESTINATION lib/static) 62 | 63 | -------------------------------------------------------------------------------- /src/cc/chunk/ClientManager.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: ClientManager.cc 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2006/03/28 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2006-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // 24 | //---------------------------------------------------------------------------- 25 | 26 | #include "ClientManager.h" 27 | 28 | using std::list; 29 | using namespace KFS; 30 | 31 | ClientManager KFS::gClientManager; 32 | 33 | void 34 | ClientManager::StartAcceptor(int port) 35 | { 36 | mAcceptor = new Acceptor(port, this); 37 | if (!mAcceptor->IsAcceptorStarted()) { 38 | KFS_LOG_FATAL("Unable to start acceptor!"); 39 | exit(-1); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/cc/chunk/Utils.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: Utils.cc 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2006/09/27 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2006-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // 24 | //---------------------------------------------------------------------------- 25 | 26 | #include "Utils.h" 27 | #include "common/log.h" 28 | 29 | using std::vector; 30 | using std::string; 31 | using namespace KFS; 32 | 33 | /// 34 | /// Return true if there is a sequence of "\r\n\r\n". 35 | /// @param[in] iobuf: Buffer with data sent by the client 36 | /// @param[out] msgLen: string length of the command in the buffer 37 | /// @retval true if a command is present; false otherwise. 38 | /// 39 | bool KFS::IsMsgAvail(IOBuffer *iobuf, int *msgLen) 40 | { 41 | const int idx = iobuf->IndexOf(0, "\r\n\r\n"); 42 | if (idx < 0) { 43 | return false; 44 | } 45 | *msgLen = idx + 4; // including terminating seq. length. 46 | return true; 47 | } 48 | 49 | void KFS::die(const string &msg) 50 | { 51 | KFS_LOG_VA_FATAL("Panic'ing: %s", msg.c_str()); 52 | abort(); 53 | } 54 | 55 | void KFS::split(std::vector &component, const string &path, char separator) 56 | { 57 | string::size_type curr = 0, nextsep = 0; 58 | string v; 59 | 60 | while (nextsep != string::npos) { 61 | nextsep = path.find(separator, curr); 62 | v = path.substr(curr, nextsep - curr); 63 | curr = nextsep + 1; 64 | component.push_back(v); 65 | } 66 | } 67 | 68 | float KFS::ComputeTimeDiff(const struct timeval &startTime, const struct timeval &endTime) 69 | { 70 | float timeSpent; 71 | 72 | timeSpent = (endTime.tv_sec * 1e6 + endTime.tv_usec) - 73 | (startTime.tv_sec * 1e6 + startTime.tv_usec); 74 | return timeSpent / 1e6; 75 | } 76 | -------------------------------------------------------------------------------- /src/cc/chunk/Utils.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: Utils.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2006/09/27 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2006-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // 24 | //---------------------------------------------------------------------------- 25 | 26 | #ifndef CHUNKSERVER_UTILS_H 27 | #define CHUNKSERVER_UTILS_H 28 | 29 | #include "libkfsIO/IOBuffer.h" 30 | #include 31 | #include 32 | 33 | namespace KFS 34 | { 35 | 36 | /// 37 | /// Given some data in a buffer, determine if we have a received a 38 | /// valid op---one that ends with "\r\n\r\n". 39 | /// @param[in] iobuf : buffer containing data 40 | /// @param[out] msgLen : if we do have a valid command, return the length of 41 | /// the command 42 | /// @retval True if we have a valid command; False otherwise. 43 | /// 44 | extern bool IsMsgAvail(IOBuffer *iobuf, int *msgLen); 45 | 46 | /// 47 | /// \brief bomb out on "impossible" error 48 | /// \param[in] msg panic text 49 | /// 50 | extern void die(const std::string &msg); 51 | 52 | /// 53 | /// Split a path into components as defined by a separator. For instance, 54 | /// "a.b.c" when split on "." will result into a vector of the form ["a", "b", "c"] 55 | /// 56 | extern void split(std::vector &component, const std::string &path, char separator); 57 | 58 | /// 59 | /// \brief compute the time difference in seconds between start and 60 | /// end times 61 | /// \param[in] startTime, endTime require: endTime >= startTime 62 | /// \retval The time difference in seconds. 63 | /// 64 | extern float ComputeTimeDiff(const struct timeval &startTime, const struct timeval &endTime); 65 | } 66 | 67 | #endif // CHUNKSERVER_UTILS_H 68 | -------------------------------------------------------------------------------- /src/cc/common/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | # Take all the .cc files and build a library out of them 3 | set (lib_src 4 | BufferedLogWriter.cc 5 | hsieh_hash.cc 6 | log.cc 7 | properties.cc 8 | ) 9 | 10 | # for the version file 11 | SET (VERSION_CC ${CMAKE_SOURCE_DIR}/src/cc/common/Version.cc) 12 | set_source_files_properties(${VERSION_CC} PROPERTIES GENERATED 1) 13 | 14 | # 15 | # Build both static/dynamic libraries. Force the linking of all apps 16 | # with a statically linked library. Since kfsCommon is the symbol 17 | # used everywhere, associate that symbol with the STATIC library. 18 | # 19 | 20 | add_custom_target (Version.cc ALL 21 | COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/buildVers.py ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/Version.cc 22 | VERBATIM) 23 | 24 | add_library (kfsCommon-shared SHARED ${lib_src} ${VERSION_CC}) 25 | add_library (kfsCommon STATIC ${lib_src} ${VERSION_CC}) 26 | set_target_properties (kfsCommon-shared PROPERTIES OUTPUT_NAME "kfsCommon") 27 | set_target_properties (kfsCommon PROPERTIES CLEAN_DIRECT_OUTPUT 1) 28 | set_target_properties (kfsCommon-shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) 29 | 30 | target_link_libraries (kfsCommon qcdio ${Log4cpp_LIBRARY} pthread) 31 | target_link_libraries (kfsCommon-shared qcdio-shared ${Log4cpp_LIBRARY} pthread) 32 | 33 | # 34 | install (TARGETS kfsCommon-shared kfsCommon 35 | LIBRARY DESTINATION lib 36 | ARCHIVE DESTINATION lib/static) 37 | 38 | install (FILES kfstypes.h kfsdecls.h DESTINATION include/kfs/common) 39 | -------------------------------------------------------------------------------- /src/cc/common/DailyRollingFileAppender.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: DailyRollingFileAppender.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2009/02/05 5 | // 6 | // Copyright 2009 Quantcast Corporation. 7 | // 8 | // This file is part of Kosmos File System (KFS). 9 | // 10 | // Licensed under the Apache License, Version 2.0 11 | // (the "License"); you may not use this file except in compliance with 12 | // the License. You may obtain a copy of the License at 13 | // 14 | // http://www.apache.org/licenses/LICENSE-2.0 15 | // 16 | // Unless required by applicable law or agreed to in writing, software 17 | // distributed under the License is distributed on an "AS IS" BASIS, 18 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | // implied. See the License for the specific language governing 20 | // permissions and limitations under the License. 21 | // 22 | // \brief Enhance the rolling file appender to rotate logs on a daily basis. 23 | //---------------------------------------------------------------------------- 24 | 25 | #ifndef COMMON_DAILYROLLINGFILEAPPENDER_H 26 | #define COMMON_DAILYROLLINGFILEAPPENDER_H 27 | 28 | #include "log.h" 29 | #include 30 | #include 31 | 32 | namespace log4cpp 33 | { 34 | /** 35 | * Enhance the file appender and roll the log file on each day. 36 | */ 37 | class LOG4CPP_EXPORT DailyRollingFileAppender : public LayoutAppender { 38 | public: 39 | DailyRollingFileAppender(const std::string &name, const std::string &fileName, 40 | unsigned int maxDaysToKeep = 30, 41 | bool append = true, 42 | mode_t mode = 00644, 43 | LayoutAppender* appender = 0); 44 | virtual ~DailyRollingFileAppender(); 45 | void setMaxDaysToKeep(unsigned int maxDaysToKeep); 46 | unsigned int getMaxDaysToKeep() const; 47 | void rollOver(); 48 | virtual void close(); 49 | protected: 50 | virtual void _append(const log4cpp::LoggingEvent &event); 51 | unsigned int _maxDaysToKeep; 52 | // record the time at which the file that we are currently 53 | // logging to was created. In case of a restart, this 54 | // variable records the last modified time of the file we are 55 | // currently logging to 56 | struct tm _logsTime; 57 | std::string _fileName; 58 | LayoutAppender& _appender; 59 | }; 60 | } 61 | 62 | #endif // COMMON_DAILYROLLINGFILEAPPENDER_H 63 | -------------------------------------------------------------------------------- /src/cc/common/Version.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: Version.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // \brief Header file for getting KFS version #'s related to builds. 5 | // 6 | // Created 2008/10/20 7 | // 8 | // Copyright 2008 Quantcast Corp. 9 | // 10 | // This file is part of Kosmos File System (KFS). 11 | // 12 | // Licensed under the Apache License, Version 2.0 13 | // (the "License"); you may not use this file except in compliance with 14 | // the License. You may obtain a copy of the License at 15 | // 16 | // http://www.apache.org/licenses/LICENSE-2.0 17 | // 18 | // Unless required by applicable law or agreed to in writing, software 19 | // distributed under the License is distributed on an "AS IS" BASIS, 20 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | // implied. See the License for the specific language governing 22 | // permissions and limitations under the License. 23 | // 24 | //---------------------------------------------------------------------------- 25 | 26 | #ifndef COMMON_KFSVERSION_H 27 | #define COMMON_KFSVERSION_H 28 | 29 | #include 30 | 31 | namespace KFS { 32 | extern const std::string KFS_BUILD_VERSION_STRING; 33 | extern const std::string KFS_SOURCE_REVISION_STRING; 34 | } 35 | 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /src/cc/common/buildVers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # $Id: buildVers.py 3488 2011-12-15 00:34:05Z sriramr $ 4 | # 5 | # Copyright 2008 Quantcast Corp. 6 | # 7 | # This file is part of Kosmos File System (KFS). 8 | # 9 | # Licensed under the Apache License, Version 2.0 10 | # (the "License"); you may not use this file except in compliance with 11 | # the License. You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 18 | # implied. See the License for the specific language governing 19 | # permissions and limitations under the License. 20 | # 21 | # Script to generate Version.cc. What we are interested in is the 22 | # revision # from the repository for the entire tree as we built the 23 | # source. This script is invoked by the makefile. 24 | # 25 | 26 | import os,sys 27 | 28 | buildstr = "" 29 | srcRevision = "" 30 | 31 | fh = open(sys.argv[2], "w") 32 | print >> fh, "//" 33 | print >> fh, "// This file is generated during compilation. DO NOT EDIT!" 34 | print >> fh, "//" 35 | print >> fh, "#include \"Version.h\" " 36 | print >> fh, "const std::string KFS::KFS_BUILD_VERSION_STRING=\"%s\";" % buildstr 37 | print >> fh, "const std::string KFS::KFS_SOURCE_REVISION_STRING=\"%s\";" % srcRevision 38 | 39 | -------------------------------------------------------------------------------- /src/cc/common/config.h: -------------------------------------------------------------------------------- 1 | 2 | //---------------------------------------------------------- -*- Mode: C++ -*- 3 | // $Id: config.h 1552 2011-01-06 22:21:54Z sriramr $ 4 | // 5 | // Created 2006/08/23 6 | // 7 | // Copyright 2008 Quantcast Corp. 8 | // Copyright 2006-2008 Kosmix Corp. 9 | // 10 | // This file is part of Kosmos File System (KFS). 11 | // 12 | // Licensed under the Apache License, Version 2.0 13 | // (the "License"); you may not use this file except in compliance with 14 | // the License. You may obtain a copy of the License at 15 | // 16 | // http://www.apache.org/licenses/LICENSE-2.0 17 | // 18 | // Unless required by applicable law or agreed to in writing, software 19 | // distributed under the License is distributed on an "AS IS" BASIS, 20 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | // implied. See the License for the specific language governing 22 | // permissions and limitations under the License. 23 | // 24 | //---------------------------------------------------------------------------- 25 | 26 | #ifndef CC_CONFIG_H 27 | #define CC_CONFIG_H 28 | 29 | //---------------------------------------------------------------------------- 30 | // Attribute to disable "unused variable" warnings 31 | // 32 | // Example: 33 | // int UNUSED_ATTR r = aFunctionThatAlwaysAlwaysAlwaysReturnsZero(); 34 | // assert(r == 0); 35 | // 36 | // Note, this doesn't break the variable when it actually *is* used, 37 | // as in a debug build. It just makes the compiler keep quiet about 38 | // not using it in release builds. 39 | //---------------------------------------------------------------------------- 40 | #if !defined(UNUSED_ATTR) 41 | #if defined(__GNUC__) 42 | #define UNUSED_ATTR __attribute__((unused)) 43 | #else 44 | #define UNUSED_ATTR 45 | #endif 46 | #endif 47 | 48 | 49 | #endif // CC_CONFIG_H 50 | -------------------------------------------------------------------------------- /src/cc/common/cxxutil.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: cxxutil.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // \brief Hash declarations to get the code to compile on Mac 5 | // 6 | // Created 2007/10/01 7 | // 8 | // Copyright 2008 Quantcast Corp. 9 | // Copyright 2007-2008 Kosmix Corp. 10 | // 11 | // This file is part of Kosmos File System (KFS). 12 | // 13 | // Licensed under the Apache License, Version 2.0 14 | // (the "License"); you may not use this file except in compliance with 15 | // the License. You may obtain a copy of the License at 16 | // 17 | // http://www.apache.org/licenses/LICENSE-2.0 18 | // 19 | // Unless required by applicable law or agreed to in writing, software 20 | // distributed under the License is distributed on an "AS IS" BASIS, 21 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 22 | // implied. See the License for the specific language governing 23 | // permissions and limitations under the License. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef COMMON_CXXUTIL_H 28 | #define COMMON_CXXUTIL_H 29 | 30 | #if defined (__APPLE__) || defined (__i386__) 31 | #include 32 | #if ((__GNUC__ == 4) && (__GNUC_MINOR__ < 2)) 33 | namespace std 34 | { 35 | namespace tr1 36 | { 37 | template <> struct hash { 38 | std::size_t operator()(long long val) const { return static_cast(val); } 39 | }; 40 | } 41 | } 42 | #endif 43 | #endif 44 | 45 | #endif 46 | -------------------------------------------------------------------------------- /src/cc/common/hsieh_hash.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: hsieh_hash.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2008/07/17 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // 8 | // This file is part of Kosmos File System (KFS). 9 | // 10 | // Licensed under the Apache License, Version 2.0 11 | // (the "License"); you may not use this file except in compliance with 12 | // the License. You may obtain a copy of the License at 13 | // 14 | // http://www.apache.org/licenses/LICENSE-2.0 15 | // 16 | // Unless required by applicable law or agreed to in writing, software 17 | // distributed under the License is distributed on an "AS IS" BASIS, 18 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | // implied. See the License for the specific language governing 20 | // permissions and limitations under the License. 21 | // 22 | // \brief String hash using Hsieh hash function. 23 | // 24 | //---------------------------------------------------------------------------- 25 | 26 | #ifndef COMMON_HSIEH_HASH_H 27 | #define COMMON_HSIEH_HASH_H 28 | 29 | #include 30 | #include 31 | #include 32 | namespace KFS 33 | { 34 | struct Hsieh_hash_fcn { 35 | std::size_t operator()(const char *data, int len) const; 36 | std::size_t operator()(const std::string &data) const; 37 | }; 38 | } 39 | 40 | #endif // COMMON_HSIEH_HASH_H 41 | -------------------------------------------------------------------------------- /src/cc/emulator/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | # 3 | # For the library take everything except the *_main.cc files 4 | # 5 | set (lib_srcs 6 | ChunkServerEmulator.cc 7 | LayoutEmulator.cc 8 | EmulatorSetup.cc 9 | ) 10 | 11 | add_library (kfsEmulator STATIC ${lib_srcs}) 12 | add_library (kfsEmulator-shared SHARED ${lib_srcs}) 13 | set_target_properties (kfsEmulator-shared PROPERTIES OUTPUT_NAME "kfsEmulator") 14 | set_target_properties (kfsEmulator PROPERTIES CLEAN_DIRECT_OUTPUT 1) 15 | set_target_properties (kfsEmulator-shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) 16 | 17 | set (exe_files rebalanceplanner rebalanceexecutor replicachecker rereplicator) 18 | foreach (exe_file ${exe_files}) 19 | add_executable (${exe_file} ${exe_file}_main.cc) 20 | if (USE_STATIC_LIB_LINKAGE) 21 | target_link_libraries (${exe_file} kfsEmulator kfsMeta kfsIO kfsCommon qcdio pthread crypto) 22 | add_dependencies (${exe_file} kfsEmulator kfsCommon kfsIO qcdio kfsMeta) 23 | else (USE_STATIC_LIB_LINKAGE) 24 | target_link_libraries (${exe_file} kfsEmulator-shared kfsMeta-shared kfsIO-shared kfsCommon-shared qcdio-shared pthread crypto) 25 | add_dependencies (${exe_file} kfsEmulator-shared kfsCommon-shared kfsIO-shared kfsMeta-shared qcdio-shared) 26 | endif (USE_STATIC_LIB_LINKAGE) 27 | endforeach (exe_file) 28 | 29 | if (APPLE) 30 | target_link_libraries(kfsEmulator-shared kfsMeta-shared kfsCommon-shared kfsIO-shared) 31 | endif (APPLE) 32 | 33 | if (NOT APPLE) 34 | target_link_libraries(kfsEmulator rt) 35 | endif (NOT APPLE) 36 | 37 | if (CMAKE_SYSTEM_NAME STREQUAL "SunOS") 38 | target_link_libraries(kfsEmulator mtmalloc) 39 | endif (CMAKE_SYSTEM_NAME STREQUAL "SunOS") 40 | 41 | # 42 | # Install them 43 | # 44 | install (TARGETS ${exe_files} kfsEmulator kfsEmulator-shared 45 | RUNTIME DESTINATION bin/emulator 46 | LIBRARY DESTINATION lib 47 | ARCHIVE DESTINATION lib/static) 48 | 49 | -------------------------------------------------------------------------------- /src/cc/emulator/ChunkServerEmulator.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: ChunkServerEmulator.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2008/08/27 5 | // 6 | // 7 | // Copyright 2008 Quantcast Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief An emulator for a chunkserver. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef EMULATOR_CHUNKSERVEREMULATOR_H 28 | #define EMULATOR_CHUNKSERVEREMULATOR_H 29 | 30 | #include 31 | #include 32 | #include 33 | 34 | #include "meta/ChunkServer.h" 35 | 36 | namespace KFS 37 | { 38 | 39 | class ChunkServerEmulator : public ChunkServer { 40 | public: 41 | ChunkServerEmulator(const ServerLocation &loc,int rack); 42 | void Enqueue(MetaChunkRequest *r); 43 | void Dispatch(); 44 | 45 | // when this emulated server goes down, fail the pending ops 46 | // that were destined to this node 47 | void FailPendingOps(); 48 | 49 | void HostingChunk(kfsChunkId_t cid, size_t chunksize) { 50 | mChunks.insert(cid); 51 | mNumChunks++; 52 | mUsedSpace += chunksize; 53 | mAllocSpace += chunksize; 54 | } 55 | void SetRebalancePlanOutFd(int fd) { 56 | mOutFd = fd; 57 | } 58 | private: 59 | std::set mChunks; 60 | int mOutFd; 61 | 62 | }; 63 | 64 | typedef boost::shared_ptr ChunkServerEmulatorPtr; 65 | 66 | } 67 | 68 | #endif // EMULATOR_CHUNKSERVEREMULATOR_H 69 | -------------------------------------------------------------------------------- /src/cc/emulator/EmulatorSetup.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: EmulatorSetup.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2008/08/29 5 | // 6 | // 7 | // Copyright 2008 Quantcast Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief Setup code to get an emulator up. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef EMULATOR_EMULATORSETUP_H 28 | #define EMULATOR_EMULATORSETUP_H 29 | 30 | #include 31 | namespace KFS 32 | { 33 | // pass an optional argument that enables changing the degree of replication for a file. 34 | void EmulatorSetup(std::string &logdir, std::string &cpdir, std::string &networkFn, 35 | std::string &chunkmapFn, int16_t minReplicasPerFile = 1, 36 | bool addChunksToReplicationChecker = false); 37 | } 38 | 39 | #endif // EMULATOR_EMULATORSETUP_H 40 | -------------------------------------------------------------------------------- /src/cc/fuse/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | # 3 | # Build the fuse binary; build it only on demand 4 | # 5 | add_executable (kfs_fuse kfs_fuse_main.cc) 6 | link_directories(${Fuse_LIBRARY_DIR}) 7 | target_link_libraries (kfs_fuse kfsClient fuse pthread rt) 8 | 9 | # 10 | install (TARGETS kfs_fuse 11 | RUNTIME DESTINATION bin 12 | LIBRARY DESTINATION lib 13 | ARCHIVE DESTINATION lib-static) 14 | -------------------------------------------------------------------------------- /src/cc/libkfsClient/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | # Take all the .cc files and build a library out of them 3 | file (GLOB sources *.cc) 4 | 5 | # 6 | # Build a static and a dynamically linked libraries. Both libraries 7 | # should have the same root name, but installed in different places 8 | # 9 | add_library (kfsClient STATIC ${sources}) 10 | add_library (kfsClient-shared SHARED ${sources}) 11 | set_target_properties (kfsClient-shared PROPERTIES OUTPUT_NAME "kfsClient") 12 | 13 | # 14 | # Since the objects have to be built twice, set this up so they don't 15 | # clobber each other. 16 | 17 | set_target_properties (kfsClient PROPERTIES CLEAN_DIRECT_OUTPUT 1) 18 | set_target_properties (kfsClient-shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) 19 | 20 | add_dependencies (kfsClient kfsCommon kfsIO qcdio) 21 | # get everthing into one 22 | target_link_libraries (kfsClient kfsCommon kfsIO qcdio crypto lzo2) 23 | target_link_libraries (kfsClient-shared kfsCommon-shared kfsIO-shared qcdio-shared crypto lzo2) 24 | # 25 | install (TARGETS kfsClient kfsClient-shared 26 | LIBRARY DESTINATION lib 27 | ARCHIVE DESTINATION lib/static) 28 | 29 | install (FILES KfsAttr.h KfsClient.h DESTINATION include/kfs) 30 | -------------------------------------------------------------------------------- /src/cc/libkfsClient/KfsPendingOp.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: KfsPendingOp.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2008/12/28 5 | // 6 | // Copyright 2008,2009 Quantcast Corp. 7 | // 8 | // This file is part of Kosmos File System (KFS). 9 | // 10 | // Licensed under the Apache License, Version 2.0 11 | // (the "License"); you may not use this file except in compliance with 12 | // the License. You may obtain a copy of the License at 13 | // 14 | // http://www.apache.org/licenses/LICENSE-2.0 15 | // 16 | // Unless required by applicable law or agreed to in writing, software 17 | // distributed under the License is distributed on an "AS IS" BASIS, 18 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | // implied. See the License for the specific language governing 20 | // permissions and limitations under the License. 21 | // 22 | // 23 | //---------------------------------------------------------------------------- 24 | 25 | #ifndef KFS_PENDING_OP_H 26 | #define KFS_PENDING_OP_H 27 | 28 | #include "common/kfstypes.h" 29 | #include "qcdio/qcthread.h" 30 | #include "qcdio/qcmutex.h" 31 | 32 | namespace KFS { 33 | 34 | class KfsClientImpl; 35 | 36 | class KfsPendingOp : public QCRunnable 37 | { 38 | public: 39 | KfsPendingOp( 40 | KfsClientImpl& inImpl); 41 | ~KfsPendingOp(); 42 | bool Start( 43 | int inFd, 44 | bool inReadFlag); 45 | bool IsPending() const 46 | { return (mFd >= 0); } 47 | void Stop(); 48 | virtual void Run(); 49 | 50 | private: 51 | KfsClientImpl& mImpl; 52 | volatile int mFd; 53 | bool mReadFlag; 54 | bool mStopFlag; 55 | ssize_t mResult; 56 | QCThread mThread; 57 | QCMutex mMutex; 58 | QCCondVar mWakeUpCond; 59 | }; 60 | 61 | } /* namespace KFS */ 62 | 63 | #endif /* KFS_PENDING_OP_H */ 64 | -------------------------------------------------------------------------------- /src/cc/libkfsClient/Utils.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: Utils.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2006/08/31 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2006-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief Utils.h: Utilities for manipulating paths and other misc. support. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef LIBKFSCLIENT_UTILS_H 28 | #define LIBKFSCLIENT_UTILS_H 29 | 30 | #include 31 | extern "C" { 32 | #include 33 | } 34 | 35 | namespace KFS { 36 | 37 | // we call this function by creating temporaries on the stack. to let 38 | // that thru, dont' stick in "&" 39 | extern std::string strip_dots(std::string path); 40 | extern std::string build_path(std::string &cwd, const char *input); 41 | 42 | extern void GetTimeval(std::string &s, struct timeval &tv); 43 | // Introduce a delay for nsecs...i.e., sleep 44 | extern void Sleep(int nsecs); 45 | 46 | } 47 | 48 | #endif // LIBKFSCLIENT_UTILS_H 49 | -------------------------------------------------------------------------------- /src/cc/libkfsClient/concurrency.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: concurrency.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // \brief Provide class definitions relating to concurrency. 5 | // 6 | // Created 2007/08/21 7 | // 8 | // Copyright 2007 Kosmix Corp. 9 | // 10 | // This file is part of Kosmos File System (KFS). 11 | // 12 | // Licensed under the Apache License, Version 2.0 13 | // (the "License"); you may not use this file except in compliance with 14 | // the License. You may obtain a copy of the License at 15 | // 16 | // http://www.apache.org/licenses/LICENSE-2.0 17 | // 18 | // Unless required by applicable law or agreed to in writing, software 19 | // distributed under the License is distributed on an "AS IS" BASIS, 20 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | // implied. See the License for the specific language governing 22 | // permissions and limitations under the License. 23 | // 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef LIBKFSCLIENT_CONCURRENCY_H 28 | #define LIBKFSCLIENT_CONCURRENCY_H 29 | 30 | #include 31 | #include 32 | extern "C" { 33 | #include 34 | } 35 | 36 | namespace KFS 37 | { 38 | class MutexLock; 39 | } 40 | 41 | // 42 | // The MutexLock class is a convenience class that facilitates 43 | // locking a mutex on function entry and automatically unlocking 44 | // it on function exit. 45 | // 46 | class KFS::MutexLock { 47 | public: 48 | /** 49 | * Acquire the mutex 50 | */ 51 | MutexLock( pthread_mutex_t *mutex ) : mMutex(mutex) 52 | { pthread_mutex_lock(mMutex); } 53 | 54 | /** 55 | * Release the mutex automatically 56 | */ 57 | ~MutexLock() 58 | { Release(); }; 59 | void Release() 60 | { 61 | if (mMutex) { 62 | int rval = pthread_mutex_unlock(mMutex); 63 | assert(!rval); 64 | (void)rval; 65 | mMutex = 0; 66 | } 67 | } 68 | 69 | private: 70 | pthread_mutex_t *mMutex; 71 | }; 72 | 73 | #endif // LIBKFSCLIENT_CONCURRENCY_H 74 | -------------------------------------------------------------------------------- /src/cc/libkfsIO/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | # Take all the .cc files and build a library out of them 3 | set (sources 4 | Acceptor.cc 5 | BufferedSocket.cc 6 | Checksum.cc 7 | Counter.cc 8 | EventManager.cc 9 | Globals.cc 10 | IOBuffer.cc 11 | NetConnection.cc 12 | NetErrorSimulator.cc 13 | NetManager.cc 14 | TcpSocket.cc 15 | TelemetryClient.cc 16 | ) 17 | 18 | add_library (kfsIO STATIC ${sources}) 19 | add_library (kfsIO-shared SHARED ${sources}) 20 | set_target_properties (kfsIO-shared PROPERTIES OUTPUT_NAME "kfsIO") 21 | 22 | set_target_properties (kfsIO PROPERTIES CLEAN_DIRECT_OUTPUT 1) 23 | set_target_properties (kfsIO-shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) 24 | 25 | add_dependencies (kfsIO qcdio kfsCommon) 26 | add_dependencies (kfsIO-shared qcdio-shared kfsCommon-shared) 27 | target_link_libraries (kfsIO qcdio kfsCommon pthread ${Boost_LIBRARIES}) 28 | target_link_libraries (kfsIO-shared qcdio-shared kfsCommon-shared pthread) 29 | if (NOT APPLE) 30 | target_link_libraries (kfsIO rt) 31 | target_link_libraries (kfsIO-shared rt) 32 | endif(NOT APPLE) 33 | 34 | if (CMAKE_KFS_USE_INTEL_IPP) 35 | target_link_libraries(kfsIO ${CMAKE_IPP_LIBS}) 36 | target_link_libraries(kfsIO-shared ${CMAKE_IPP_LIBS}) 37 | else (CMAKE_KFS_USE_INTEL_IPP) 38 | target_link_libraries(kfsIO z) 39 | target_link_libraries(kfsIO-shared z) 40 | endif (CMAKE_KFS_USE_INTEL_IPP) 41 | 42 | if (CMAKE_SYSTEM_NAME STREQUAL "SunOS") 43 | target_link_libraries (kfsIO nsl socket resolv) 44 | target_link_libraries (kfsIO-shared nsl socket resolv) 45 | endif (CMAKE_SYSTEM_NAME STREQUAL "SunOS") 46 | 47 | install (TARGETS kfsIO kfsIO-shared 48 | LIBRARY DESTINATION lib 49 | ARCHIVE DESTINATION lib/static) 50 | -------------------------------------------------------------------------------- /src/cc/libkfsIO/Checksum.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: Checksum.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2006/09/12 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2006-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // Code for computing 32-bit Adler checksums 24 | //---------------------------------------------------------------------------- 25 | 26 | #ifndef CHUNKSERVER_CHECKSUM_H 27 | #define CHUNKSERVER_CHECKSUM_H 28 | 29 | #include 30 | #include 31 | #include "libkfsIO/IOBuffer.h" 32 | 33 | namespace KFS 34 | { 35 | /// Checksums are computed on 64KB block boundaries. We use the 36 | /// "rolling" 32-bit Adler checksum algorithm 37 | const uint32_t CHECKSUM_BLOCKSIZE = 65536; 38 | 39 | const uint32_t MOD_ADLER = 65521; 40 | 41 | extern uint32_t OffsetToChecksumBlockNum(off_t offset); 42 | 43 | extern uint32_t OffsetToChecksumBlockStart(off_t offset); 44 | 45 | extern uint32_t OffsetToChecksumBlockEnd(off_t offset); 46 | 47 | /// Call this function if you want checksum computed over CHECKSUM_BLOCKSIZE bytes 48 | extern uint32_t ComputeBlockChecksum(const IOBuffer *data, size_t len); 49 | extern uint32_t ComputeBlockChecksum(const IOBuffer *d1, size_t l1, 50 | const IOBuffer *d2, size_t l2); 51 | extern uint32_t ComputeBlockChecksum(const char *data, size_t len); 52 | 53 | /// Call this function if you want a checksums for a sequence of CHECKSUM_BLOCKSIZE bytes 54 | extern std::vector ComputeChecksums(const IOBuffer *data, size_t len); 55 | extern std::vector ComputeChecksums(const char *data, size_t len); 56 | 57 | } 58 | 59 | #endif // CHUNKSERVER_CHECKSUM_H 60 | -------------------------------------------------------------------------------- /src/cc/libkfsIO/Counter.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: Counter.cc 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2006/07/20 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2006-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief Implementation of the CounterManager and declarations for a 24 | //few counters. 25 | //---------------------------------------------------------------------------- 26 | 27 | #include "Counter.h" 28 | 29 | using namespace KFS; 30 | 31 | /// A few commonly needed counters 32 | Counter gOpenNetFds("Open network fds"); 33 | Counter gOpenDiskFds("Open disk fds"); 34 | Counter gNetBytesRead("Bytes read from network"); 35 | Counter gNetBytesWritten("Bytes written to network"); 36 | Counter gDiskBytesRead("Bytes read from disk"); 37 | Counter gDiskBytesWritten("Bytes written to disk"); 38 | // track the # of failed read/writes 39 | Counter gDiskIOErrors("Disk I/O errors"); 40 | 41 | CounterManager gCounterManager; 42 | 43 | void InitCounterManager() 44 | { 45 | gCounterManager.AddCounter(&gOpenNetFds); 46 | gCounterManager.AddCounter(&gOpenDiskFds); 47 | 48 | gCounterManager.AddCounter(&gNetBytesRead); 49 | gCounterManager.AddCounter(&gNetBytesWritten); 50 | 51 | gCounterManager.AddCounter(&gDiskBytesRead); 52 | gCounterManager.AddCounter(&gDiskBytesWritten); 53 | 54 | gCounterManager.AddCounter(&gDiskIOErrors); 55 | 56 | } 57 | 58 | 59 | -------------------------------------------------------------------------------- /src/cc/libkfsIO/FileHandle.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: FileHandle.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2008/05/13 5 | // 6 | // 7 | // Copyright 2008 Quantcast Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief A ref-counted file-id object. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef _LIBKFSIO_FILEHANDLE_H 28 | #define _LIBKFSIO_FILEHANDLE_H 29 | 30 | #include 31 | 32 | namespace KFS 33 | { 34 | struct FileHandle_t { 35 | FileHandle_t() : mFd(-1) { } 36 | FileHandle_t(int fd) : mFd(fd) { } 37 | ~FileHandle_t() { 38 | if (mFd < 0) 39 | return; 40 | close(mFd); 41 | mFd = -1; 42 | } 43 | void Close() { 44 | if (mFd < 0) 45 | return; 46 | close(mFd); 47 | mFd = -1; 48 | } 49 | int mFd; // the underlying file pointer 50 | }; 51 | 52 | typedef boost::shared_ptr FileHandlePtr; 53 | } 54 | 55 | #endif // _LIBKFSIO_FILEHANDLE_H 56 | -------------------------------------------------------------------------------- /src/cc/libkfsIO/Globals.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: Globals.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2006/10/09 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2006-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief Define the globals needed by the KFS IO library. These 24 | //globals are also available to any app that uses the KFS IO library. 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef LIBKFSIO_GLOBALS_H 28 | #define LIBKFSIO_GLOBALS_H 29 | 30 | #include "NetManager.h" 31 | #include "Counter.h" 32 | 33 | namespace KFS 34 | { 35 | namespace libkfsio 36 | { 37 | 38 | struct Globals_t { 39 | CounterManager counterManager; 40 | // Commonly needed counters 41 | Counter ctrOpenNetFds; 42 | Counter ctrOpenDiskFds; 43 | Counter ctrNetBytesRead; 44 | Counter ctrNetBytesWritten; 45 | Counter ctrDiskBytesRead; 46 | Counter ctrDiskBytesWritten; 47 | // track the # of failed read/writes 48 | Counter ctrDiskIOErrors; 49 | void Init(); 50 | static NetManager& getNetManager(); 51 | static void Destroy(); 52 | static Globals_t& Instance(); 53 | private: 54 | ~Globals_t(); 55 | Globals_t(); 56 | bool mInitedFlag; 57 | bool mDestructedFlag; 58 | NetManager* mForGdbToFindNetManager; 59 | static Globals_t* sForGdbToFindInstance; 60 | }; 61 | 62 | inline static void InitGlobals() 63 | { Globals_t::Instance().Init(); } 64 | inline static void DestroyGlobals() 65 | { Globals_t::Destroy(); } 66 | inline static NetManager& globalNetManager() 67 | { return Globals_t::getNetManager(); } 68 | inline static Globals_t & globals() 69 | { return Globals_t::Instance(); } 70 | } 71 | } 72 | 73 | #endif // LIBKFSIO_GLOBALS_H 74 | -------------------------------------------------------------------------------- /src/cc/libkfsIO/NetErrorSimulator.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: NetErrorSimulator.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2009/10/03 5 | // 6 | // Copyright 2009 Quantcast Corp. 7 | // 8 | // This file is part of Kosmos File System (KFS). 9 | // 10 | // Licensed under the Apache License, Version 2.0 11 | // (the "License"); you may not use this file except in compliance with 12 | // the License. You may obtain a copy of the License at 13 | // 14 | // http://www.apache.org/licenses/LICENSE-2.0 15 | // 16 | // Unless required by applicable law or agreed to in writing, software 17 | // distributed under the License is distributed on an "AS IS" BASIS, 18 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | // implied. See the License for the specific language governing 20 | // permissions and limitations under the License. 21 | // 22 | // 23 | //---------------------------------------------------------------------------- 24 | 25 | #ifndef NET_ERROR_SIMULATOR_H 26 | #define NET_ERROR_SIMULATOR_H 27 | 28 | namespace KFS 29 | { 30 | // Error simulation spec syntax: 31 | // sn=^[^:]*:30000$,pn=^[^:]*:20000$,a=rn+rd+wd+err+rst+rand+log,int=1,sleep|rsleep=2;;... 32 | // sn -- perl regex to match : socket name returned by getsockname() 33 | // pn -- perl regex to match : peer name returned by getpeername() 34 | // a -- action: 35 | // rn -- read none, disable read 36 | // rd -- read discard, discard input buffer data, doesn't disable read 37 | // wd -- write discard, disable write, discard output buffer data 38 | // err -- set poll error flag, reset all other flags 39 | // erd -- when read flag set, set poll error flag, reset all other flags 40 | // ewr -- when write flag set, set poll error flag, reset all other flags 41 | // rst -- close connection, call NetConnection::Close() 42 | // rand -- use random inteval from 0 to "int=" 43 | // exit -- call _exit(1) 44 | // abort -- call abort() 45 | // log -- emit log message when action performed 46 | // int=x -- action interval 47 | // sleep=x -- sleep for x seconds 48 | // rsleep=x -- sleep for random # of seconds in the range [0:x] 49 | // 50 | class NetManager; 51 | bool NetErrorSimulatorConfigure(NetManager& inNetManager, const char* inConfigPtr = 0); 52 | } 53 | 54 | #endif /* NET_ERROR_SIMULATOR_H */ 55 | -------------------------------------------------------------------------------- /src/cc/libkfsIO/NetKicker.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: NetKicker.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2008/05/04 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // 8 | // This file is part of Kosmos File System (KFS). 9 | // 10 | // Licensed under the Apache License, Version 2.0 11 | // (the "License"); you may not use this file except in compliance with 12 | // the License. You may obtain a copy of the License at 13 | // 14 | // http://www.apache.org/licenses/LICENSE-2.0 15 | // 16 | // Unless required by applicable law or agreed to in writing, software 17 | // distributed under the License is distributed on an "AS IS" BASIS, 18 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | // implied. See the License for the specific language governing 20 | // permissions and limitations under the License. 21 | // 22 | // \brief In a multi-threaded setup, the net-manager runs on a 23 | // separate thread. It has a poll loop with a delay of N secs; during 24 | // this time, any messages generated interally maybe held waiting for 25 | // the net-manager thread to wake up. This has the effect of 26 | // introducing N sec delay for sending outbound messages. To force 27 | // the net-manager to wake up, create a "net kicker" object and write 28 | // some data to it. That is, create a pipe and make the pipe fd part 29 | // of the net-manager's poll loop; whenever we write data to the pipe, 30 | // the net-manager will wake up and service out-bound messages. 31 | // 32 | // ---------------------------------------------------------------------------- 33 | 34 | #ifndef LIBKFSIO_NETKICKER_H 35 | #define LIBKFSIO_NETKICKER_H 36 | 37 | namespace KFS 38 | { 39 | class NetKicker { 40 | public: 41 | NetKicker(); 42 | ~NetKicker(); 43 | /// The "write" portion of the pipe writes one byte on the fd. 44 | void Kick(); 45 | private: 46 | class Impl; 47 | Impl& mImpl; 48 | /// This is the callback from the net-manager to drain out the 49 | /// bytes written on the pipe 50 | int Drain(); 51 | int GetFd() const; 52 | friend class NetManager; 53 | }; 54 | } 55 | 56 | #endif // LIBKFSIO_NETKICKER_H 57 | -------------------------------------------------------------------------------- /src/cc/meta/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | # 3 | # For the library take everything except the *_main.cc files 4 | # 5 | set (lib_srcs 6 | checkpoint.cc 7 | ChunkReplicator.cc 8 | ChunkServer.cc 9 | ChunkServerFactory.cc 10 | ChildProcessTracker.cc 11 | ClientSM.cc 12 | entry.cc 13 | kfsops.cc 14 | kfstree.cc 15 | LayoutManager.cc 16 | LeaseCleaner.cc 17 | logger.cc 18 | meta.cc 19 | NetDispatch.cc 20 | replay.cc 21 | request.cc 22 | restore.cc 23 | startup.cc 24 | util.cc 25 | ) 26 | 27 | add_library (kfsMeta STATIC ${lib_srcs}) 28 | add_library (kfsMeta-shared SHARED ${lib_srcs}) 29 | set_target_properties (kfsMeta-shared PROPERTIES OUTPUT_NAME "kfsMeta") 30 | set_target_properties (kfsMeta PROPERTIES CLEAN_DIRECT_OUTPUT 1) 31 | set_target_properties (kfsMeta-shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) 32 | 33 | set (exe_files metaserver logcompactor filelister kfsfsck) 34 | foreach (exe_file ${exe_files}) 35 | add_executable (${exe_file} ${exe_file}_main.cc) 36 | if (USE_STATIC_LIB_LINKAGE) 37 | target_link_libraries (${exe_file} kfsMeta kfsIO kfsCommon qcdio pthread crypto) 38 | add_dependencies (${exe_file} kfsCommon kfsIO kfsMeta qcdio) 39 | else (USE_STATIC_LIB_LINKAGE) 40 | target_link_libraries (${exe_file} kfsMeta-shared kfsIO-shared kfsCommon-shared qcdio-shared pthread crypto) 41 | add_dependencies (${exe_file} kfsCommon-shared kfsIO-shared kfsMeta-shared qcdio-shared) 42 | endif (USE_STATIC_LIB_LINKAGE) 43 | endforeach (exe_file) 44 | 45 | if (APPLE) 46 | target_link_libraries(kfsMeta-shared kfsCommon-shared kfsIO-shared crypto) 47 | endif (APPLE) 48 | 49 | if (NOT APPLE) 50 | target_link_libraries(kfsMeta rt) 51 | target_link_libraries(metaserver rt) 52 | endif (NOT APPLE) 53 | 54 | if (CMAKE_SYSTEM_NAME STREQUAL "SunOS") 55 | # mtmalloc seemed to worsen metaserver startup time; it took 56 | # 4 mins for fsck to load checkpoint from WORM, where as 30 for metaserver. 57 | # So, switch to umem 58 | target_link_libraries(kfsMeta umem) 59 | target_link_libraries(metaserver umem) 60 | endif (CMAKE_SYSTEM_NAME STREQUAL "SunOS") 61 | 62 | # 63 | # Install them 64 | # 65 | install (TARGETS ${exe_files} kfsMeta kfsMeta-shared 66 | RUNTIME DESTINATION bin 67 | LIBRARY DESTINATION lib 68 | ARCHIVE DESTINATION lib/static) 69 | 70 | -------------------------------------------------------------------------------- /src/cc/meta/ChildProcessTracker.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: ChildProcessTracker.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2009/04/30 5 | // 6 | // Copyright 2009 Quantcast Corporation. 7 | // 8 | // This file is part of Kosmos File System (KFS). 9 | // 10 | // Licensed under the Apache License, Version 2.0 11 | // (the "License"); you may not use this file except in compliance with 12 | // the License. You may obtain a copy of the License at 13 | // 14 | // http://www.apache.org/licenses/LICENSE-2.0 15 | // 16 | // Unless required by applicable law or agreed to in writing, software 17 | // distributed under the License is distributed on an "AS IS" BASIS, 18 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | // implied. See the License for the specific language governing 20 | // permissions and limitations under the License. 21 | // 22 | // \brief A timer that periodically tracks child process that have been spawned 23 | // and retrieves their completion status. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef META_CHILDPROCESSTRACKER_H 28 | #define META_CHILDPROCESSTRACKER_H 29 | 30 | #include "libkfsIO/ITimeout.h" 31 | #include 32 | #include 33 | #include 34 | 35 | namespace KFS 36 | { 37 | class MetaRequest; 38 | 39 | class ChildProcessTrackingTimer : public ITimeout { 40 | public: 41 | ChildProcessTrackingTimer(int timeoutMilliSec = 500) { 42 | SetTimeoutInterval(timeoutMilliSec); 43 | }; 44 | // On a timeout check the child processes for exit status 45 | virtual void Timeout(); 46 | // track the process with pid and return the exit status to MetaRequest 47 | void Track(pid_t pid, MetaRequest *r); 48 | private: 49 | typedef std::multimap Pending; 50 | Pending mPending; 51 | }; 52 | 53 | extern ChildProcessTrackingTimer gChildProcessTracker; 54 | } 55 | 56 | #endif // META_CHILDPROCESSTRACKER_H 57 | -------------------------------------------------------------------------------- /src/cc/meta/ChunkReplicator.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: ChunkReplicator.cc 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2007/01/18 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2007-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // 24 | //---------------------------------------------------------------------------- 25 | 26 | #include "ChunkReplicator.h" 27 | #include "libkfsIO/Globals.h" 28 | #include 29 | 30 | using namespace KFS; 31 | using namespace libkfsio; 32 | 33 | ChunkReplicator::ChunkReplicator() : 34 | mInProgress(false), mOp(1, this) 35 | { 36 | SET_HANDLER(this, &ChunkReplicator::HandleEvent); 37 | mTimer = new ChunkReplicatorTimeoutImpl(this); 38 | globalNetManager().RegisterTimeoutHandler(mTimer); 39 | } 40 | 41 | ChunkReplicator::~ChunkReplicator() 42 | { 43 | globalNetManager().UnRegisterTimeoutHandler(mTimer); 44 | delete mTimer; 45 | } 46 | 47 | /// Use the main loop to process the request. 48 | int 49 | ChunkReplicator::HandleEvent(int code, void *data) 50 | { 51 | static seq_t seqNum = 1; 52 | switch (code) { 53 | case EVENT_CMD_DONE: 54 | mInProgress = false; 55 | return 0; 56 | case EVENT_TIMEOUT: 57 | if (mInProgress) 58 | return 0; 59 | mOp.opSeqno = seqNum; 60 | ++seqNum; 61 | mInProgress = true; 62 | submit_request(&mOp); 63 | return 0; 64 | default: 65 | assert(!"Unknown event"); 66 | break; 67 | } 68 | return 0; 69 | } 70 | -------------------------------------------------------------------------------- /src/cc/meta/ChunkReplicator.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: ChunkReplicator.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2007/01/18 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2007-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief A chunk replicator is used to replicate chunks when necessary. For 24 | // instance, if a chunkserver dies, the degree of replication for that 25 | // chunk may be less than the desired amount. In such cases, 26 | // replicate that chunk. This process works by sending a 27 | // "MetaChunkReplicate" op to do the work. 28 | // 29 | //---------------------------------------------------------------------------- 30 | 31 | #ifndef META_CHUNKREPLICATOR_H 32 | #define META_CHUNKREPLICATOR_H 33 | 34 | #include "libkfsIO/KfsCallbackObj.h" 35 | #include "libkfsIO/ITimeout.h" 36 | #include "libkfsIO/Event.h" 37 | #include "request.h" 38 | 39 | namespace KFS 40 | { 41 | 42 | class ChunkReplicatorTimeoutImpl; 43 | 44 | class ChunkReplicator : public KfsCallbackObj { 45 | public: 46 | /// The interval with which we check if chunks are sufficiently replicated 47 | static const int REPLICATION_CHECK_INTERVAL_SECS = 60; 48 | static const int REPLICATION_CHECK_INTERVAL_MSECS = 49 | REPLICATION_CHECK_INTERVAL_SECS * 1000; 50 | 51 | ChunkReplicator(); 52 | ~ChunkReplicator(); 53 | int HandleEvent(int code, void *data); 54 | 55 | private: 56 | /// If a replication op is in progress, skip a send 57 | bool mInProgress; 58 | ChunkReplicatorTimeoutImpl *mTimer; 59 | /// The op for checking 60 | MetaChunkReplicationCheck mOp; 61 | }; 62 | 63 | class ChunkReplicatorTimeoutImpl : public ITimeout { 64 | public: 65 | ChunkReplicatorTimeoutImpl(ChunkReplicator *c) : mOwner(c) { 66 | SetTimeoutInterval(ChunkReplicator::REPLICATION_CHECK_INTERVAL_MSECS); 67 | } 68 | void Timeout() { 69 | mOwner->HandleEvent(EVENT_TIMEOUT, NULL); 70 | } 71 | private: 72 | ChunkReplicator *mOwner; 73 | }; 74 | 75 | 76 | 77 | } 78 | 79 | #endif // META_CHUNKREPLICATOR_H 80 | -------------------------------------------------------------------------------- /src/cc/meta/ChunkServerFactory.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: ChunkServerFactory.cc 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2006/09/29 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2006-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief ChunkServerFactory code to remove a dead server. 24 | //---------------------------------------------------------------------------- 25 | 26 | #include "ChunkServerFactory.h" 27 | #include 28 | using std::find_if; 29 | 30 | using namespace KFS; 31 | 32 | void 33 | ChunkServerFactory::RemoveServer(const ChunkServer *target) 34 | { 35 | list::iterator i; 36 | 37 | i = find_if(mChunkServers.begin(), mChunkServers.end(), 38 | ChunkServerMatcher(target)); 39 | if (i != mChunkServers.end()) { 40 | mChunkServers.erase(i); 41 | } 42 | } 43 | 44 | -------------------------------------------------------------------------------- /src/cc/meta/ChunkServerHeartbeater.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: ChunkServerHeartbeater.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2009/02/24 5 | // 6 | // Copyright 2009 Quantcast Corporation. 7 | // 8 | // This file is part of Kosmos File System (KFS). 9 | // 10 | // Licensed under the Apache License, Version 2.0 11 | // (the "License"); you may not use this file except in compliance with 12 | // the License. You may obtain a copy of the License at 13 | // 14 | // http://www.apache.org/licenses/LICENSE-2.0 15 | // 16 | // Unless required by applicable law or agreed to in writing, software 17 | // distributed under the License is distributed on an "AS IS" BASIS, 18 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | // implied. See the License for the specific language governing 20 | // permissions and limitations under the License. 21 | // 22 | // \brief A timer that periodically sends a heartbeat message to all chunkservers. 23 | // 24 | //---------------------------------------------------------------------------- 25 | 26 | #ifndef META_CHUNKSERVERHEARTBEATER_H 27 | #define META_CHUNKSERVERHEARTBEATER_H 28 | 29 | #include "libkfsIO/ITimeout.h" 30 | #include "LayoutManager.h" 31 | 32 | namespace KFS 33 | { 34 | class ChunkServerHeartbeater : public ITimeout { 35 | public: 36 | ChunkServerHeartbeater(int heartbeatIntervalSec = 60) { 37 | // send heartbeat once every min by default 38 | SetTimeoutInterval(heartbeatIntervalSec * 1000); 39 | }; 40 | // On a timeout send a heartbeat RPC 41 | void Timeout() { 42 | gLayoutManager.HeartbeatChunkServers(); 43 | }; 44 | }; 45 | } 46 | 47 | #endif // META_CHUNKSERVERHEARTBEATER_H 48 | -------------------------------------------------------------------------------- /src/cc/meta/ClientManager.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: ClientManager.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2006/06/02 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2006-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \file ClientManager.h 24 | // \brief Create client state machines whenever clients connect to meta server. 25 | // 26 | //---------------------------------------------------------------------------- 27 | 28 | #ifndef META_CLIENTMANAGER_H 29 | #define META_CLIENTMANAGER_H 30 | 31 | #include "libkfsIO/Acceptor.h" 32 | #include "libkfsIO/KfsCallbackObj.h" 33 | #include "ClientSM.h" 34 | 35 | namespace KFS 36 | { 37 | 38 | class ClientManager : public IAcceptorOwner { 39 | public: 40 | ClientManager() { 41 | mAcceptor = NULL; 42 | }; 43 | virtual ~ClientManager() { 44 | delete mAcceptor; 45 | }; 46 | void StartAcceptor(int port) { 47 | mAcceptor = new Acceptor(port, this); 48 | }; 49 | KfsCallbackObj *CreateKfsCallbackObj(NetConnectionPtr &conn) { 50 | // XXX: Should we keep a list of all client state machines? 51 | return new ClientSM(conn); 52 | } 53 | private: 54 | // The socket object which is setup to accept connections. 55 | Acceptor *mAcceptor; 56 | }; 57 | 58 | 59 | } 60 | 61 | #endif // META_CLIENTMANAGER_H 62 | -------------------------------------------------------------------------------- /src/cc/meta/LeaseCleaner.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: LeaseCleaner.cc 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2006/10/16 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2006-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // 24 | //---------------------------------------------------------------------------- 25 | 26 | 27 | #include "LeaseCleaner.h" 28 | #include "libkfsIO/Globals.h" 29 | #include 30 | 31 | using namespace KFS; 32 | using namespace libkfsio; 33 | 34 | LeaseCleaner::LeaseCleaner() : 35 | mInProgress(false), mOp(1, this) 36 | { 37 | SET_HANDLER(this, &LeaseCleaner::HandleEvent); 38 | /// setup a periodic event to do the cleanup 39 | mTimer = new LeaseCleanerTimeoutImpl(this); 40 | globalNetManager().RegisterTimeoutHandler(mTimer); 41 | } 42 | 43 | LeaseCleaner::~LeaseCleaner() 44 | { 45 | globalNetManager().UnRegisterTimeoutHandler(mTimer); 46 | delete mTimer; 47 | } 48 | 49 | /// Use the main looop to submit the cleanup request. 50 | int 51 | LeaseCleaner::HandleEvent(int code, void *data) 52 | { 53 | static seq_t seqNum = 1; 54 | switch (code) { 55 | case EVENT_CMD_DONE: 56 | mInProgress = false; 57 | return 0; 58 | case EVENT_TIMEOUT: 59 | if (mInProgress) 60 | return 0; 61 | 62 | mOp.opSeqno = seqNum; 63 | ++seqNum; 64 | mInProgress = true; 65 | submit_request(&mOp); 66 | return 0; 67 | default: 68 | assert(!"Unknown event"); 69 | break; 70 | } 71 | return 0; 72 | } 73 | -------------------------------------------------------------------------------- /src/cc/meta/LeaseCleaner.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: LeaseCleaner.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2006/10/16 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2006-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief A lease cleaner to periodically cleanout dead leases. It does so, 24 | // by sending a "MetaLeaseCleanup" op to do the work. 25 | //---------------------------------------------------------------------------- 26 | 27 | #ifndef META_LEASECLEANER_H 28 | #define META_LEASECLEANER_H 29 | 30 | #include "libkfsIO/KfsCallbackObj.h" 31 | #include "libkfsIO/ITimeout.h" 32 | #include "libkfsIO/Event.h" 33 | #include "request.h" 34 | 35 | namespace KFS 36 | { 37 | 38 | class LeaseCleanerTimeoutImpl; 39 | 40 | class LeaseCleaner : public KfsCallbackObj { 41 | public: 42 | // The interval with which we should do the cleanup 43 | static const int CLEANUP_INTERVAL_SECS = 60; 44 | static const int CLEANUP_INTERVAL_MSECS = CLEANUP_INTERVAL_SECS * 1000; 45 | 46 | LeaseCleaner(); 47 | ~LeaseCleaner(); 48 | int HandleEvent(int code, void *data); 49 | 50 | private: 51 | /// If a cleanup op is in progress, skip a send 52 | bool mInProgress; 53 | LeaseCleanerTimeoutImpl *mTimer; 54 | /// The op for doing the cleanup 55 | MetaLeaseCleanup mOp; 56 | }; 57 | 58 | class LeaseCleanerTimeoutImpl : public ITimeout { 59 | public: 60 | LeaseCleanerTimeoutImpl(LeaseCleaner *l) : mOwner(l) { 61 | SetTimeoutInterval(LeaseCleaner::CLEANUP_INTERVAL_MSECS); 62 | } 63 | void Timeout() { 64 | mOwner->HandleEvent(EVENT_TIMEOUT, NULL); 65 | } 66 | private: 67 | LeaseCleaner *mOwner; 68 | }; 69 | 70 | } 71 | 72 | #endif // META_LEASECLEANER_H 73 | -------------------------------------------------------------------------------- /src/cc/meta/NetDispatch.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: NetDispatch.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2006/06/01 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2006-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \file NetDispatch.h 24 | // \brief Meta-server network dispatcher 25 | // 26 | //---------------------------------------------------------------------------- 27 | 28 | #ifndef META_NETDISPATCH_H 29 | #define META_NETDISPATCH_H 30 | 31 | #include "libkfsIO/NetManager.h" 32 | #include "libkfsIO/Acceptor.h" 33 | #include "ChunkServerFactory.h" 34 | #include "ClientManager.h" 35 | 36 | namespace KFS 37 | { 38 | class NetDispatch { 39 | public: 40 | NetDispatch(); 41 | ~NetDispatch(); 42 | void Start(int clientAcceptPort, int chunkServerAcceptPort); 43 | //!< An RPC just finished execution. Send it on its merry way 44 | void Dispatch(MetaRequest *r); 45 | ChunkServerFactory *GetChunkServerFactory() { 46 | return mChunkServerFactory; 47 | } 48 | 49 | private: 50 | ClientManager *mClientManager; //!< tracks the connected clients 51 | ChunkServerFactory *mChunkServerFactory; //!< creates chunk servers when they connect 52 | }; 53 | 54 | extern NetDispatch gNetDispatch; 55 | } 56 | 57 | #endif // META_NETDISPATCH_H 58 | -------------------------------------------------------------------------------- /src/cc/meta/kfstypes.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * $Id: kfstypes.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | * 4 | * \file kfstypes.h 5 | * \brief simple typedefs and enums for the KFS metadata server 6 | * 7 | * Copyright 2008 Quantcast Corp. 8 | * Copyright 2006-2008 Kosmix Corp. 9 | * 10 | * This file is part of Kosmos File System (KFS). 11 | * 12 | * Licensed under the Apache License, Version 2.0 13 | * (the "License"); you may not use this file except in compliance with 14 | * the License. You may obtain a copy of the License at 15 | * 16 | * http://www.apache.org/licenses/LICENSE-2.0 17 | * 18 | * Unless required by applicable law or agreed to in writing, software 19 | * distributed under the License is distributed on an "AS IS" BASIS, 20 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | * implied. See the License for the specific language governing 22 | * permissions and limitations under the License. 23 | * 24 | */ 25 | #if !defined(KFS_TYPES_H) 26 | #define KFS_TYPES_H 27 | 28 | #include 29 | #include 30 | #include "common/kfstypes.h" 31 | 32 | namespace KFS { 33 | 34 | /*! 35 | * \brief KFS metadata types. 36 | */ 37 | enum MetaType { 38 | KFS_UNINIT, //!< uninitialized 39 | KFS_INTERNAL, //!< internal node 40 | KFS_FATTR, //!< file attributes 41 | KFS_DENTRY, //!< directory entry 42 | KFS_CHUNKINFO, //!< chunk information 43 | KFS_SENTINEL = 99999 //!< internal use, must be largest 44 | }; 45 | 46 | /*! 47 | * \brief KFS file types 48 | */ 49 | enum FileType { 50 | KFS_NONE, //!< uninitialized 51 | KFS_FILE, //!< plain file 52 | KFS_DIR //!< directory 53 | }; 54 | 55 | /*! 56 | * \brief KFS lease types 57 | */ 58 | enum LeaseType { 59 | READ_LEASE, 60 | WRITE_LEASE 61 | }; 62 | 63 | } 64 | #endif // !defined(KFS_TYPES_H) 65 | -------------------------------------------------------------------------------- /src/cc/meta/replay.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * $Id: replay.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | * 4 | * \file replay.h 5 | * \brief log replay definitions 6 | * 7 | * Copyright 2008 Quantcast Corp. 8 | * Copyright 2006-2008 Kosmix Corp. 9 | * 10 | * This file is part of Kosmos File System (KFS). 11 | * 12 | * Licensed under the Apache License, Version 2.0 13 | * (the "License"); you may not use this file except in compliance with 14 | * the License. You may obtain a copy of the License at 15 | * 16 | * http://www.apache.org/licenses/LICENSE-2.0 17 | * 18 | * Unless required by applicable law or agreed to in writing, software 19 | * distributed under the License is distributed on an "AS IS" BASIS, 20 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | * implied. See the License for the specific language governing 22 | * permissions and limitations under the License. 23 | */ 24 | #if !defined(KFS_REPLAY_H) 25 | #define KFS_REPLAY_H 26 | 27 | #include 28 | #include 29 | 30 | using std::string; 31 | using std::ifstream; 32 | 33 | namespace KFS { 34 | 35 | class Replay { 36 | ifstream file; //!< the log file being replayed 37 | string path; //!< path name for log file 38 | int number; //!< sequence number for log file 39 | public: 40 | Replay(): number(-1) { }; 41 | ~Replay() { }; 42 | void openlog(const string &p); //!< open the log file for replay 43 | int playlog(); //!< read and apply its contents 44 | int logno() { return number; } 45 | int playAllLogs(); //!< starting from log for logno(), 46 | //!< replay whatever logs we have in the logdir. 47 | }; 48 | 49 | extern Replay replayer; 50 | 51 | } 52 | #endif // !defined(KFS_REPLAY_H) 53 | -------------------------------------------------------------------------------- /src/cc/meta/restore.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * $Id: restore.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | * 4 | * \file restore.h 5 | * \brief rebuild metatree from saved checkpoint 6 | * 7 | * Copyright 2008 Quantcast Corp. 8 | * Copyright 2006-2008 Kosmix Corp. 9 | * 10 | * This file is part of Kosmos File System (KFS). 11 | * 12 | * Licensed under the Apache License, Version 2.0 13 | * (the "License"); you may not use this file except in compliance with 14 | * the License. You may obtain a copy of the License at 15 | * 16 | * http://www.apache.org/licenses/LICENSE-2.0 17 | * 18 | * Unless required by applicable law or agreed to in writing, software 19 | * distributed under the License is distributed on an "AS IS" BASIS, 20 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | * implied. See the License for the specific language governing 22 | * permissions and limitations under the License. 23 | */ 24 | #if !defined(KFS_RESTORE_H) 25 | #define KFS_RESTORE_H 26 | 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include "util.h" 32 | 33 | using std::ifstream; 34 | using std::string; 35 | using std::deque; 36 | using std::map; 37 | 38 | namespace KFS { 39 | 40 | /*! 41 | * \brief state for restoring from a checkpoint file 42 | */ 43 | class Restorer { 44 | ifstream file; //!< the CP file 45 | public: 46 | /* 47 | * process the CP file. also, if the # of replicas of a file is below 48 | * the specified value, bump up replication. this allows us to change 49 | * the filesystem wide degree of replication in a simple manner. 50 | */ 51 | bool rebuild(string cpname, int16_t minNumReplicasPerFile = 1); 52 | }; 53 | 54 | extern bool restore_chunkVersionInc(deque &c); 55 | 56 | /* 57 | * Whenever a checkpoint file is loaded in, it takes up a ton of memory. 58 | * To prevent logcompactor from stomping over a filechecker or filelister or 59 | * vice-versa, all of these tools acquire a lock file before loading the 60 | * checkpoint. 61 | */ 62 | extern void acquire_lockfile(const string &lockfn, int ntries); 63 | 64 | } 65 | #endif // !defined(KFS_RESTORE_H) 66 | -------------------------------------------------------------------------------- /src/cc/meta/startup.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * $Id: startup.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | * 4 | * Copyright 2008 Quantcast Corp. 5 | * Copyright 2006-2008 Kosmix Corp. 6 | * 7 | * This file is part of Kosmos File System (KFS). 8 | * 9 | * Licensed under the Apache License, Version 2.0 10 | * (the "License"); you may not use this file except in compliance with 11 | * the License. You may obtain a copy of the License at 12 | * 13 | * http://www.apache.org/licenses/LICENSE-2.0 14 | * 15 | * Unless required by applicable law or agreed to in writing, software 16 | * distributed under the License is distributed on an "AS IS" BASIS, 17 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 18 | * implied. See the License for the specific language governing 19 | * permissions and limitations under the License. 20 | * 21 | * 22 | * \file startup.h 23 | * \brief code for starting up the metadata server 24 | */ 25 | #if !defined(KFS_STARTUP_H) 26 | #define KFS_STARTUP_H 27 | 28 | extern "C" { 29 | #include 30 | } 31 | 32 | #include 33 | 34 | namespace KFS { 35 | 36 | extern void kfs_startup(const std::string &logdir, const std::string &cpdir, 37 | uint32_t minChunkservers, uint32_t minReplicasPerFile, 38 | bool enablePathToFidCache); 39 | 40 | } 41 | #endif // !defined(KFS_STARTUP_H) 42 | -------------------------------------------------------------------------------- /src/cc/meta/util.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * $Id: util.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | * 4 | * \file util.h 5 | * \brief miscellaneous metadata server code 6 | * 7 | * Copyright 2008 Quantcast Corp. 8 | * Copyright 2006-2008 Kosmix Corp. 9 | * 10 | * This file is part of Kosmos File System (KFS). 11 | * 12 | * Licensed under the Apache License, Version 2.0 13 | * (the "License"); you may not use this file except in compliance with 14 | * the License. You may obtain a copy of the License at 15 | * 16 | * http://www.apache.org/licenses/LICENSE-2.0 17 | * 18 | * Unless required by applicable law or agreed to in writing, software 19 | * distributed under the License is distributed on an "AS IS" BASIS, 20 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | * implied. See the License for the specific language governing 22 | * permissions and limitations under the License. 23 | */ 24 | #if !defined(KFS_UTIL_H) 25 | #define KFS_UTIL_H 26 | 27 | #include 28 | #include 29 | #include 30 | #include "kfstypes.h" 31 | #include "libkfsIO/IOBuffer.h" 32 | 33 | using std::string; 34 | using std::deque; 35 | using std::ostream; 36 | 37 | namespace KFS { 38 | 39 | extern chunkOff_t chunkStartOffset(chunkOff_t offset); 40 | extern int link_latest(const string real, const string alias); 41 | extern const string toString(long long n); 42 | extern long long toNumber(const string s); 43 | extern string makename(const string dir, const string prefix, int number); 44 | extern void split(deque &component, const string path, char sep); 45 | extern bool file_exists(string s); 46 | extern void warn(const string s, bool use_perror); 47 | extern void panic(const string s, bool use_perror); 48 | 49 | extern void sendtime(ostream &os, const string &prefix, 50 | const struct timeval &t, const string &suffix); 51 | 52 | extern string timeToStr(time_t val); 53 | 54 | /// Is a message that ends with "\r\n\r\n" available in the 55 | /// buffer. 56 | /// @param[in] iobuf An IO buffer stream with message 57 | /// received from the chunk server. 58 | /// @param[out] msgLen If a valid message is 59 | /// available, length (in bytes) of the message. 60 | /// @retval true if a message is available; false otherwise 61 | /// 62 | extern bool IsMsgAvail(IOBuffer *iobuf, int *msgLen); 63 | 64 | extern float ComputeTimeDiff(const struct timeval &start, const struct timeval &end); 65 | 66 | } 67 | #endif // !defined(KFS_UTIL_H) 68 | -------------------------------------------------------------------------------- /src/cc/qcdio/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # $Id$ 3 | # 4 | # Created 2008/12/11 5 | # Author: Mike Ovsiannikov 6 | # 7 | # Copyright 2008,2009 Quantcast Corp. 8 | # 9 | # This file is part of Kosmos File System (KFS). 10 | # 11 | # Licensed under the Apache License, Version 2.0 12 | # (the "License"); you may not use this file except in compliance with 13 | # the License. You may obtain a copy of the License at 14 | # 15 | # http://www.apache.org/licenses/LICENSE-2.0 16 | # 17 | # Unless required by applicable law or agreed to in writing, software 18 | # distributed under the License is distributed on an "AS IS" BASIS, 19 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | # implied. See the License for the specific language governing 21 | # permissions and limitations under the License. 22 | # 23 | # 24 | 25 | 26 | # Take all the .cpp files and build a library out of them 27 | file (GLOB sources *.cpp) 28 | file (GLOB includes *.h) 29 | 30 | string(TOUPPER QC_OS_NAME_${CMAKE_SYSTEM_NAME} QC_OS_NAME) 31 | add_definitions (-D_GNU_SOURCE -D${QC_OS_NAME} -DQC_USE_BOOST) 32 | 33 | # 34 | # Build a static and a dynamically linked libraries. Both libraries 35 | # should have the same root name, but installed in different places 36 | # 37 | add_library (qcdio STATIC ${sources}) 38 | add_library (qcdio-shared SHARED ${sources}) 39 | set_target_properties (qcdio-shared PROPERTIES OUTPUT_NAME "qcdio") 40 | 41 | if (NOT APPLE) 42 | target_link_libraries (qcdio rt) 43 | target_link_libraries (qcdio-shared rt) 44 | endif(NOT APPLE) 45 | 46 | # 47 | # Since the objects have to be built twice, set this up so they don't 48 | # clobber each other. 49 | 50 | set_target_properties (qcdio PROPERTIES CLEAN_DIRECT_OUTPUT 1) 51 | set_target_properties (qcdio-shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) 52 | 53 | install (TARGETS qcdio qcdio-shared 54 | LIBRARY DESTINATION lib 55 | ARCHIVE DESTINATION lib/static) 56 | 57 | # install (FILES includes DESTINATION include/qcdio) 58 | -------------------------------------------------------------------------------- /src/cc/qcdio/qcdebug.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: qcdebug.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2008/11/01 5 | // 6 | // Copyright 2008,2009 Quantcast Corp. 7 | // 8 | // This file is part of Kosmos File System (KFS). 9 | // 10 | // Licensed under the Apache License, Version 2.0 11 | // (the "License"); you may not use this file except in compliance with 12 | // the License. You may obtain a copy of the License at 13 | // 14 | // http://www.apache.org/licenses/LICENSE-2.0 15 | // 16 | // Unless required by applicable law or agreed to in writing, software 17 | // distributed under the License is distributed on an "AS IS" BASIS, 18 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | // implied. See the License for the specific language governing 20 | // permissions and limitations under the License. 21 | // 22 | // 23 | //---------------------------------------------------------------------------- 24 | 25 | #ifndef QCDEBUG_H 26 | #define QCDEBUG_H 27 | 28 | #include 29 | 30 | #define QCASSERT(a) assert(a) 31 | 32 | #ifdef NDEBUG 33 | # define QCVERIFY(a) a 34 | #else 35 | # define QCVERIFY(a) QCASSERT(a) 36 | #endif 37 | 38 | #endif /* QCDEBUG_H */ 39 | -------------------------------------------------------------------------------- /src/cc/qcdio/qcfdpoll.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: qcfdpoll.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2009/03/11 5 | // 6 | // Copyright 2008,2009 Quantcast Corp. 7 | // 8 | // This file is part of Kosmos File System (KFS). 9 | // 10 | // Licensed under the Apache License, Version 2.0 11 | // (the "License"); you may not use this file except in compliance with 12 | // the License. You may obtain a copy of the License at 13 | // 14 | // http://www.apache.org/licenses/LICENSE-2.0 15 | // 16 | // Unless required by applicable law or agreed to in writing, software 17 | // distributed under the License is distributed on an "AS IS" BASIS, 18 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | // implied. See the License for the specific language governing 20 | // permissions and limitations under the License. 21 | // 22 | // 23 | //---------------------------------------------------------------------------- 24 | 25 | #ifndef QCFDPOLL_H 26 | #define QCFDPOLL_H 27 | 28 | class QCFdPoll 29 | { 30 | public: 31 | enum OpType 32 | { 33 | kOpTypeNone = 0x00, 34 | kOpTypeIn = 0x01, 35 | kOpTypeOut = 0x02, 36 | kOpTypePri = 0x04, 37 | kOpTypeError = 0x08, 38 | kOpTypeHup = 0x10 39 | }; 40 | typedef int Fd; 41 | 42 | QCFdPoll(); 43 | ~QCFdPoll(); 44 | int Add( 45 | Fd inFd, 46 | int inOpType, 47 | void* inUserDataPtr = 0); 48 | int Set( 49 | Fd inFd, 50 | int inOpType, 51 | void* inUserDataPtr = 0); 52 | int In( 53 | Fd inFd, 54 | void* inUserDataPtr = 0) 55 | { return Set(inFd, kOpTypeIn, inUserDataPtr); } 56 | int Out( 57 | Fd inFd, 58 | void* inUserDataPtr = 0) 59 | { return Set(inFd, kOpTypeOut, inUserDataPtr); } 60 | int Io( 61 | Fd inFd, 62 | void* inUserDataPtr = 0) 63 | { return Set(inFd, kOpTypeIn + kOpTypeOut, inUserDataPtr); } 64 | int Remove( 65 | Fd inFd); 66 | int Poll( 67 | int inMaxEventCountHint, 68 | int inWaitMilliSec); 69 | bool Next( 70 | int& outOpType, 71 | void*& outUserDataPtr); 72 | private: 73 | class Impl; 74 | Impl& mImpl; 75 | 76 | private: 77 | QCFdPoll( 78 | const QCFdPoll& inPoll); 79 | QCFdPoll operator=( 80 | const QCFdPoll& inPoll); 81 | }; 82 | 83 | #endif /* QCFDPOLL_H */ 84 | -------------------------------------------------------------------------------- /src/cc/qcdio/qcutils.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: qcutils.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2008/11/01 5 | // 6 | // Copyright 2008,2009 Quantcast Corp. 7 | // 8 | // This file is part of Kosmos File System (KFS). 9 | // 10 | // Licensed under the Apache License, Version 2.0 11 | // (the "License"); you may not use this file except in compliance with 12 | // the License. You may obtain a copy of the License at 13 | // 14 | // http://www.apache.org/licenses/LICENSE-2.0 15 | // 16 | // Unless required by applicable law or agreed to in writing, software 17 | // distributed under the License is distributed on an "AS IS" BASIS, 18 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | // implied. See the License for the specific language governing 20 | // permissions and limitations under the License. 21 | // 22 | // 23 | //---------------------------------------------------------------------------- 24 | 25 | #ifndef QCUTILS_H 26 | #define QCUTILS_H 27 | 28 | #include 29 | #include 30 | 31 | #define QCRTASSERT(a) \ 32 | if (!(a)) QCUtils::AssertionFailure(#a, __FILE__, __LINE__) 33 | 34 | struct QCUtils 35 | { 36 | static void FatalError( 37 | const char* inMsgPtr, 38 | int inSysError); 39 | static std::string SysError( 40 | int inSysError, 41 | const char* inMsgPtr = 0); 42 | static void AssertionFailure( 43 | const char* inMsgPtr, 44 | const char* inFileNamePtr, 45 | int inLineNum); 46 | static int64_t ReserveFileSpace( 47 | int inFd, 48 | int64_t inSize); 49 | static int64_t UnReserveFileSpace( 50 | int inFd, 51 | int64_t inOffset, 52 | int64_t inLen); 53 | static int AllocateFileSpace( 54 | const char* inFileNamePtr, 55 | int64_t inSize, 56 | int64_t inMinSize = -1, 57 | int64_t* inInitialFileSizePtr = 0); 58 | static int AllocateFileSpace( 59 | int inFd, 60 | int64_t inSize, 61 | int64_t inMinSize = -1, 62 | int64_t* inInitialFileSizePtr = 0); 63 | }; 64 | 65 | #endif /* QCUTILS_H */ 66 | -------------------------------------------------------------------------------- /src/cc/rebalancer/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | # 3 | # For the library take everything except the *_main.cc files 4 | # 5 | 6 | add_executable (rebalancer rebalancer_main.cc) 7 | 8 | set (exe_files rebalancer) 9 | 10 | # 11 | # Install them 12 | # 13 | install (TARGETS ${exe_files} 14 | RUNTIME DESTINATION bin 15 | LIBRARY DESTINATION lib 16 | ARCHIVE DESTINATION lib/static) 17 | 18 | -------------------------------------------------------------------------------- /src/cc/sailfish/mrtools/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | # Build the kappender library 3 | set (lib_srcs 4 | appender.cc 5 | input_handler.cc 6 | workgetter.cc 7 | imerger.cc 8 | util.cc 9 | ) 10 | 11 | add_library (kapplib STATIC ${lib_srcs}) 12 | add_library (kapplib-shared SHARED ${lib_srcs}) 13 | set_target_properties (kapplib-shared PROPERTIES OUTPUT_NAME "kapplib") 14 | set_target_properties (kapplib PROPERTIES CLEAN_DIRECT_OUTPUT 1) 15 | set_target_properties (kapplib-shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) 16 | 17 | add_dependencies (kapplib kfsClient) 18 | add_dependencies (kapplib-shared kfsClient-shared) 19 | 20 | set (exe_files 21 | kappender 22 | imerger 23 | kdatagen 24 | ifilereader 25 | icomparer 26 | merge_validator 27 | ) 28 | 29 | IF (APPLE) 30 | target_link_libraries(kapplib-shared kfsClient-shared kfsCommon-shared) 31 | ENDIF (APPLE) 32 | set (COMPRESS_LIBS crypto lzo2) 33 | add_definitions(-DKFS_COMPRESSION_ENABLED) 34 | 35 | # 36 | # Every executable depends on its namesake source with _main.cc 37 | # 38 | foreach (exe_file ${exe_files}) 39 | add_executable (${exe_file} ${exe_file}_main.cc) 40 | if (USE_STATIC_LIB_LINKAGE) 41 | add_dependencies (${exe_file} kapplib) 42 | target_link_libraries (${exe_file} kapplib kfsClient) 43 | target_link_libraries(${exe_file} ${COMPRESS_LIBS}) 44 | else (USE_STATIC_LIB_LINKAGE) 45 | add_dependencies (${exe_file} kapplib-shared) 46 | target_link_libraries (${exe_file} kapplib-shared kfsClient-shared) 47 | target_link_libraries(${exe_file} ${COMPRESS_LIBS}) 48 | endif (USE_STATIC_LIB_LINKAGE) 49 | endforeach (exe_file) 50 | 51 | # 52 | install (TARGETS ${exe_files} kapplib-shared 53 | RUNTIME DESTINATION bin/sailfish 54 | LIBRARY DESTINATION lib 55 | ARCHIVE DESTINATION lib/static) 56 | -------------------------------------------------------------------------------- /src/cc/sailfish/mrtools/util.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: util.cc $ 3 | // 4 | // Created 2011/02/08 5 | // 6 | // Copyright 2011 Yahoo Corporation. All rights reserved. 7 | // This file is part of the Sailfish project. 8 | // 9 | // Licensed under the Apache License, Version 2.0 10 | // (the "License"); you may not use this file except in compliance with 11 | // the License. You may obtain a copy of the License at 12 | // 13 | // http://www.apache.org/licenses/LICENSE-2.0 14 | // 15 | // Unless required by applicable law or agreed to in writing, software 16 | // distributed under the License is distributed on an "AS IS" BASIS, 17 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 18 | // implied. See the License for the specific language governing 19 | // permissions and limitations under the License. 20 | // 21 | // 22 | //---------------------------------------------------------------------------- 23 | 24 | #include "util.h" 25 | using std::string; 26 | using namespace KFS; 27 | 28 | MsgLogger::LogLevel 29 | KFS::str2LogLevel(string logLevel) 30 | { 31 | if (logLevel == "DEBUG") 32 | return MsgLogger::kLogLevelDEBUG; 33 | if (logLevel == "INFO") 34 | return MsgLogger::kLogLevelINFO; 35 | if (logLevel == "WARN") 36 | return MsgLogger::kLogLevelWARN; 37 | if (logLevel == "FATAL") 38 | return MsgLogger::kLogLevelFATAL; 39 | 40 | return MsgLogger::kLogLevelINFO; 41 | } 42 | 43 | bool KFS::IsMessageAvail(IOBuffer *iobuf, int &msgLen) 44 | { 45 | const int idx = iobuf->IndexOf(0, "\r\n\r\n"); 46 | if (idx < 0) { 47 | return false; 48 | } 49 | msgLen = idx + 4; // including terminating seq. length. 50 | return true; 51 | } 52 | 53 | int KFS::MemAlignedMalloc(void **ptr, size_t alignment, size_t size) 54 | { 55 | if (alignment < sizeof(void *)) { 56 | alignment = sizeof(void *); 57 | } 58 | 59 | // check for power of 2 60 | bool isPowerOf2 = ((alignment != 0) && !(alignment & (alignment - 1))); 61 | 62 | assert(isPowerOf2); 63 | 64 | free(*ptr); 65 | *ptr = NULL; 66 | return posix_memalign(ptr, alignment, size); 67 | } 68 | -------------------------------------------------------------------------------- /src/cc/sailfish/mrtools/util.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: util.h $ 3 | // 4 | // Created 2011/02/08 5 | // 6 | // Copyright 2011 Yahoo Corporation. All rights reserved. 7 | // This file is part of the Sailfish project. 8 | // 9 | // Licensed under the Apache License, Version 2.0 10 | // (the "License"); you may not use this file except in compliance with 11 | // the License. You may obtain a copy of the License at 12 | // 13 | // http://www.apache.org/licenses/LICENSE-2.0 14 | // 15 | // Unless required by applicable law or agreed to in writing, software 16 | // distributed under the License is distributed on an "AS IS" BASIS, 17 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 18 | // implied. See the License for the specific language governing 19 | // permissions and limitations under the License. 20 | // 21 | // 22 | //---------------------------------------------------------------------------- 23 | 24 | #ifndef KAPPENDER_UTIL_H 25 | #define KAPPENDER_UTIL_H 26 | 27 | #include "common/log.h" 28 | #include "libkfsIO/IOBuffer.h" 29 | #include "libkfsClient/KfsOps.h" 30 | #include 31 | #include 32 | 33 | namespace KFS 34 | { 35 | struct HttpGetOp : public KfsOp { 36 | int httpStatusCode; 37 | std::string cmd; 38 | 39 | HttpGetOp(const std::string &c) : 40 | KfsOp(CMD_UNKNOWN, 1), httpStatusCode(-1), cmd(c) { } 41 | // Send a HTTP GET request to the workbuilder 42 | void Request(std::ostream &os) { 43 | os << "GET " << cmd << " HTTP/1.1\r\n"; 44 | os << "Cseq: " << seq << "\r\n"; 45 | if (contentLength == 0) { 46 | os << "\r\n"; 47 | return; 48 | } 49 | os << "Content-Length: " << contentLength << "\r\n\r\n"; 50 | // Mixing << and os.write isn't working. 51 | // os.write(contentBuf, contentLength); 52 | std::string s(contentBuf, contentLength); 53 | os << s; 54 | } 55 | void ParseResponseHeaderSelf(const Properties &prop) { 56 | httpStatusCode = prop.getValue("Http-Status-Code", 200); 57 | } 58 | std::string Show() const { 59 | std::ostringstream os; 60 | 61 | os << " GET " << cmd; 62 | return os.str(); 63 | } 64 | }; 65 | 66 | MsgLogger::LogLevel str2LogLevel(std::string logLevel); 67 | bool IsMessageAvail(IOBuffer *iobuf, int &msgLen); 68 | // Effectively a wrapper around posix_memalign()---this function 69 | // ensures that the args meet the requirements of posix_memalign() 70 | int MemAlignedMalloc(void **ptr, size_t alignment, size_t size); 71 | template 72 | int MemAlignedMalloc(T **ptr, size_t alignment, size_t size) { 73 | return MemAlignedMalloc((void **) ptr, alignment, size); 74 | } 75 | } 76 | 77 | #endif // KAPPENDER_UTIL_H 78 | -------------------------------------------------------------------------------- /src/cc/sailfish/sorter/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | # Build the kappender library 3 | set (lib_srcs 4 | chunksorter.cc 5 | util.cc 6 | ) 7 | 8 | add_library (kcslib STATIC ${lib_srcs}) 9 | add_library (kcslib-shared SHARED ${lib_srcs}) 10 | set_target_properties (kcslib-shared PROPERTIES OUTPUT_NAME "kcslib") 11 | set_target_properties (kcslib PROPERTIES CLEAN_DIRECT_OUTPUT 1) 12 | set_target_properties (kcslib-shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) 13 | 14 | add_dependencies (kcslib kchunklib) 15 | add_dependencies (kcslib-shared kchunklib-shared) 16 | 17 | set (exe_files 18 | chunksorter 19 | chunksort_test 20 | sortertest 21 | ) 22 | 23 | IF (APPLE) 24 | target_link_libraries(kcslib-shared kchunklib-shared kfsCommon-shared) 25 | ENDIF (APPLE) 26 | 27 | set (COMPRESS_LIBS crypto lzo2) 28 | add_definitions(-DKFS_COMPRESSION_ENABLED) 29 | 30 | # 31 | # Every executable depends on its namesake source with _main.cc 32 | # 33 | foreach (exe_file ${exe_files}) 34 | add_executable (${exe_file} ${exe_file}_main.cc) 35 | if (USE_STATIC_LIB_LINKAGE) 36 | add_dependencies (${exe_file} kcslib) 37 | target_link_libraries (${exe_file} kcslib kchunklib kfsIO kfsCommon qcdio crypto) 38 | target_link_libraries(${exe_file} ${COMPRESS_LIBS}) 39 | else (USE_STATIC_LIB_LINKAGE) 40 | add_dependencies (${exe_file} kcslib-shared) 41 | target_link_libraries (${exe_file} kcslib-shared kchunklib-shared kfsIO-shared qcdio-shared) 42 | target_link_libraries(${exe_file} ${COMPRESS_LIBS}) 43 | endif (USE_STATIC_LIB_LINKAGE) 44 | endforeach (exe_file) 45 | 46 | # 47 | install (TARGETS ${exe_files} kcslib-shared 48 | RUNTIME DESTINATION bin/sailfish 49 | LIBRARY DESTINATION lib 50 | ARCHIVE DESTINATION lib/static) 51 | -------------------------------------------------------------------------------- /src/cc/sailfish/sorter/sortertest_main.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: sortertest_main.cc $ 3 | // 4 | // Created 2011/09/21 5 | // 6 | // Copyright 2011 Yahoo Corporation. All rights reserved. 7 | // This file is part of the Sailfish project. 8 | // 9 | // Licensed under the Apache License, Version 2.0 10 | // (the "License"); you may not use this file except in compliance with 11 | // the License. You may obtain a copy of the License at 12 | // 13 | // http://www.apache.org/licenses/LICENSE-2.0 14 | // 15 | // Unless required by applicable law or agreed to in writing, software 16 | // distributed under the License is distributed on an "AS IS" BASIS, 17 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 18 | // implied. See the License for the specific language governing 19 | // permissions and limitations under the License. 20 | // 21 | // \brief Uses the sorter code for testing: if compression is involved, 22 | // that code path is tested too. 23 | //---------------------------------------------------------------------------- 24 | 25 | #include "common/log.h" 26 | #include "chunksorter.h" 27 | 28 | #include 29 | #include 30 | 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | namespace po = boost::program_options; 41 | 42 | using namespace KFS; 43 | using std::string; 44 | 45 | int main(int argc, char **argv) 46 | { 47 | SortFileOp sfo(0); 48 | KFS::SortWorker worker(0, 0, 0); 49 | 50 | // Declare the supported options. 51 | po::options_description desc("Allowed options"); 52 | desc.add_options() 53 | ("help,h", "produce help message") 54 | ("input,i", po::value(&sfo.inputFn)->default_value(""), "Input file to be sorted") 55 | ("output,o", po::value(&sfo.outputFn)->default_value(""), "Output file") 56 | ; 57 | 58 | po::variables_map vm; 59 | po::store(po::parse_command_line(argc, argv, desc), vm); 60 | po::notify(vm); 61 | 62 | if (vm.count("help")) { 63 | std::cout << desc << std::endl; 64 | return 0; 65 | } 66 | 67 | MsgLogger::Init(NULL, MsgLogger::kLogLevelINFO); 68 | 69 | worker.sortFile(&sfo); 70 | 71 | MsgLogger::Stop(); 72 | } 73 | -------------------------------------------------------------------------------- /src/cc/sailfish/sorter/util.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: util.cc $ 3 | // 4 | // Created 2011/02/08 5 | // 6 | // Copyright 2011 Yahoo Corporation. All rights reserved. 7 | // This file is part of the Sailfish project. 8 | // 9 | // Licensed under the Apache License, Version 2.0 10 | // (the "License"); you may not use this file except in compliance with 11 | // the License. You may obtain a copy of the License at 12 | // 13 | // http://www.apache.org/licenses/LICENSE-2.0 14 | // 15 | // Unless required by applicable law or agreed to in writing, software 16 | // distributed under the License is distributed on an "AS IS" BASIS, 17 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 18 | // implied. See the License for the specific language governing 19 | // permissions and limitations under the License. 20 | // 21 | // 22 | //---------------------------------------------------------------------------- 23 | 24 | #include "util.h" 25 | #include 26 | using std::string; 27 | using namespace KFS; 28 | 29 | MsgLogger::LogLevel 30 | KFS::str2LogLevel(string logLevel) 31 | { 32 | if (logLevel == "DEBUG") 33 | return MsgLogger::kLogLevelDEBUG; 34 | if (logLevel == "INFO") 35 | return MsgLogger::kLogLevelINFO; 36 | if (logLevel == "WARN") 37 | return MsgLogger::kLogLevelWARN; 38 | if (logLevel == "FATAL") 39 | return MsgLogger::kLogLevelFATAL; 40 | 41 | return MsgLogger::kLogLevelINFO; 42 | } 43 | 44 | int KFS::MemAlignedMalloc(void **ptr, size_t alignment, size_t size) 45 | { 46 | if (alignment < sizeof(void *)) { 47 | alignment = sizeof(void *); 48 | } 49 | 50 | // check for power of 2 51 | bool isPowerOf2 = ((alignment != 0) && !(alignment & (alignment - 1))); 52 | 53 | assert(isPowerOf2); 54 | 55 | free(*ptr); 56 | *ptr = NULL; 57 | return posix_memalign(ptr, alignment, size); 58 | } 59 | -------------------------------------------------------------------------------- /src/cc/sailfish/sorter/util.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: util.h $ 3 | // 4 | // Created 2011/02/08 5 | // 6 | // Copyright 2011 Yahoo Corporation. All rights reserved. 7 | // This file is part of the Sailfish project. 8 | // 9 | // Licensed under the Apache License, Version 2.0 10 | // (the "License"); you may not use this file except in compliance with 11 | // the License. You may obtain a copy of the License at 12 | // 13 | // http://www.apache.org/licenses/LICENSE-2.0 14 | // 15 | // Unless required by applicable law or agreed to in writing, software 16 | // distributed under the License is distributed on an "AS IS" BASIS, 17 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 18 | // implied. See the License for the specific language governing 19 | // permissions and limitations under the License. 20 | // 21 | // 22 | //---------------------------------------------------------------------------- 23 | 24 | #ifndef KAPPENDER_UTIL_H 25 | #define KAPPENDER_UTIL_H 26 | 27 | #include "common/log.h" 28 | #include 29 | #include 30 | 31 | namespace KFS 32 | { 33 | 34 | MsgLogger::LogLevel str2LogLevel(std::string logLevel); 35 | // Effectively a wrapper around posix_memalign()---this function 36 | // ensures that the args meet the requirements of posix_memalign() 37 | int MemAlignedMalloc(void **ptr, size_t alignment, size_t size); 38 | 39 | template 40 | int MemAlignedMalloc(T **ptr, size_t alignment, size_t size) { 41 | return MemAlignedMalloc((void **) ptr, alignment, size); 42 | } 43 | } 44 | 45 | #endif // KAPPENDER_UTIL_H 46 | -------------------------------------------------------------------------------- /src/cc/sailfish/workbuilder/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | # 3 | # For the library take everything except the *_main.cc files 4 | # 5 | 6 | set (lib_srcs 7 | clientsm.cc 8 | job.cc 9 | indexreader.cc 10 | statuschecker.cc 11 | setnumreducer.cc 12 | workbuilder.cc 13 | util.cc 14 | ) 15 | 16 | # 17 | # build both static and dynamic 18 | # 19 | add_library (kworkb STATIC ${lib_srcs}) 20 | add_library (kworkb-shared SHARED ${lib_srcs}) 21 | set_target_properties (kworkb-shared PROPERTIES OUTPUT_NAME "kworkb") 22 | set_target_properties (kworkb PROPERTIES CLEAN_DIRECT_OUTPUT 1) 23 | set_target_properties (kworkb-shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) 24 | 25 | add_dependencies (kworkb kfsCommon) 26 | add_dependencies (kworkb-shared kfsCommon-shared) 27 | 28 | add_executable (kworkbuilder workbuilder_main.cc) 29 | 30 | set (exe_files kworkbuilder) 31 | 32 | foreach (exe_file ${exe_files}) 33 | if (USE_STATIC_LIB_LINKAGE) 34 | add_dependencies (${exe_file} kworkb kfsClient kfsCommon kfsIO) 35 | target_link_libraries (${exe_file} kworkb kfsClient kfsIO kfsCommon ssl crypto z) 36 | else (USE_STATIC_LIB_LINKAGE) 37 | add_dependencies (${exe_file} kworkb-shared kfsClient-shared kfsCommon-shared kfsIO-shared) 38 | target_link_libraries (${exe_file} kworkb-shared kfsClient-shared kfsIO-shared kfsCommon-shared ssl crypto z) 39 | endif (USE_STATIC_LIB_LINKAGE) 40 | endforeach (exe_file) 41 | 42 | if (APPLE OR CYGWIN) 43 | target_link_libraries(kworkb-shared kfsClient-shared kfsIO-shared kfsCommon-shared ssl crypto z) 44 | endif (APPLE OR CYGWIN) 45 | 46 | # 47 | # Install them 48 | # 49 | install (TARGETS ${exe_files} kworkb kworkb-shared 50 | RUNTIME DESTINATION bin/sailfish 51 | LIBRARY DESTINATION lib 52 | ARCHIVE DESTINATION lib/static) 53 | 54 | -------------------------------------------------------------------------------- /src/cc/sailfish/workbuilder/clientmgr.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: clientmgr.h 3365 2011-11-23 22:40:49Z sriramr $ 3 | // 4 | // Created 2011/01/04 5 | // 6 | // Copyright 2011 Yahoo Corporation. All rights reserved. 7 | // This file is part of the Sailfish project. 8 | // 9 | // Licensed under the Apache License, Version 2.0 10 | // (the "License"); you may not use this file except in compliance with 11 | // the License. You may obtain a copy of the License at 12 | // 13 | // http://www.apache.org/licenses/LICENSE-2.0 14 | // 15 | // Unless required by applicable law or agreed to in writing, software 16 | // distributed under the License is distributed on an "AS IS" BASIS, 17 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 18 | // implied. See the License for the specific language governing 19 | // permissions and limitations under the License. 20 | // 21 | // \brief A client manager that spins up a client sm in response to a connection. 22 | // 23 | //---------------------------------------------------------------------------- 24 | 25 | #ifndef WORKBUILDER_CLIENTMGR_H 26 | #define WORKBUILDER_CLIENTMGR_H 27 | 28 | #include "libkfsIO/Acceptor.h" 29 | #include "libkfsIO/KfsCallbackObj.h" 30 | #include "clientsm.h" 31 | 32 | namespace workbuilder 33 | { 34 | class ClientSM; 35 | 36 | class ClientManager : public KFS::IAcceptorOwner { 37 | public: 38 | ClientManager() { 39 | mAcceptor = NULL; 40 | } 41 | virtual ~ClientManager() { 42 | delete mAcceptor; 43 | } 44 | void StartAcceptor(int port) { 45 | mAcceptor = new KFS::Acceptor(port, this); 46 | } 47 | KFS::KfsCallbackObj *CreateKfsCallbackObj(KFS::NetConnectionPtr &conn) { 48 | return new ClientSM(conn); 49 | } 50 | private: 51 | // The socket object which is setup to accept connections. 52 | KFS::Acceptor *mAcceptor; 53 | }; 54 | extern ClientManager gClientManager; 55 | } 56 | 57 | #endif // WORKBUILDER_CLIENTMGR_H 58 | -------------------------------------------------------------------------------- /src/cc/sailfish/workbuilder/clientsm.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: clientsm.h 3365 2011-11-23 22:40:49Z sriramr $ 3 | // 4 | // Created 2011/01/04 5 | // 6 | // Copyright 2011 Yahoo Corporation. All rights reserved. 7 | // This file is part of the Sailfish project. 8 | // 9 | // Licensed under the Apache License, Version 2.0 10 | // (the "License"); you may not use this file except in compliance with 11 | // the License. You may obtain a copy of the License at 12 | // 13 | // http://www.apache.org/licenses/LICENSE-2.0 14 | // 15 | // Unless required by applicable law or agreed to in writing, software 16 | // distributed under the License is distributed on an "AS IS" BASIS, 17 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 18 | // implied. See the License for the specific language governing 19 | // permissions and limitations under the License. 20 | // 21 | // \brief Declarations for a client SM. This encapsulates all the 22 | // structures associated with a connection/request. 23 | // 24 | //---------------------------------------------------------------------------- 25 | 26 | #ifndef WORKBUILDER_CLIENTSM_H 27 | #define WORKBUILDER_CLIENTSM_H 28 | #include "util.h" 29 | #include "workbuilder.h" 30 | #include "libkfsIO/KfsCallbackObj.h" 31 | #include "libkfsIO/NetConnection.h" 32 | #include 33 | namespace workbuilder 34 | { 35 | class ClientSM : public KFS::KfsCallbackObj { 36 | public: 37 | ClientSM(KFS::NetConnectionPtr &conn); 38 | ~ClientSM(); 39 | // 40 | // Sequence: 41 | // Client connects. 42 | // - A new client sm is born 43 | // - reads a request out of the connection 44 | // - submit the request for execution 45 | // - when the request is done, send a response back. 46 | // - since this is all http, the connection will get closed by client 47 | // and we'll see the close and kill this object. 48 | int HandleRequest(int code, void *data); 49 | private: 50 | KFS::NetConnectionPtr mNetConnection; 51 | 52 | /// Given a (possibly) complete op in a buffer, run it. 53 | bool HandleClientCmd(KFS::IOBuffer *iobuf, int cmdLen); 54 | 55 | /// Op has finished execution. Send a response to the client. 56 | void SendResponse(const WorkbuilderOp &op); 57 | 58 | /// in debug messages print out client IP address so we know 59 | /// where the command is coming from. 60 | std::string mClientIP; 61 | }; 62 | } 63 | 64 | #endif // WORKBUILDER_CLIENTSM_H 65 | -------------------------------------------------------------------------------- /src/cc/sailfish/workbuilder/setnumreducer.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: setnumreducer.h 3809 2012-02-28 22:22:12Z sriramr $ 3 | // 4 | // Created 2011/01/04 5 | // 6 | // Copyright 2011 Yahoo Corporation. All rights reserved. 7 | // This file is part of the Sailfish project. 8 | // 9 | // Licensed under the Apache License, Version 2.0 10 | // (the "License"); you may not use this file except in compliance with 11 | // the License. You may obtain a copy of the License at 12 | // 13 | // http://www.apache.org/licenses/LICENSE-2.0 14 | // 15 | // Unless required by applicable law or agreed to in writing, software 16 | // distributed under the License is distributed on an "AS IS" BASIS, 17 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 18 | // implied. See the License for the specific language governing 19 | // permissions and limitations under the License. 20 | // 21 | // 22 | //---------------------------------------------------------------------------- 23 | 24 | #ifndef WORKBUILDER_SETNUMREDUCER_H 25 | #define WORKBUILDER_SETNUMREDUCER_H 26 | #include "util.h" 27 | #include "statuschecker.h" 28 | #include 29 | 30 | namespace workbuilder 31 | { 32 | class JTNotifier { 33 | public: 34 | JTNotifier(const std::string &jobTracker, const std::string &jobId); 35 | void setState(const char *s) { 36 | mJobState = s; 37 | } 38 | std::string getState() { 39 | return mJobState; 40 | } 41 | JobStatus setNumReducers(int nReducers); 42 | JobStatus reRunTask(int mapTaskId); 43 | int getNumUnfinishedMappers(); 44 | JobStatus getNumReducersToPreempt(int &nReducersToPreempt); 45 | 46 | private: 47 | KFS::ServerLocation mJobTrackerLocation; 48 | std::string mJobId; 49 | std::string mJobState; 50 | }; 51 | } 52 | 53 | #endif // WORKBUILDER_SETNUMREDUCER_H 54 | -------------------------------------------------------------------------------- /src/cc/sailfish/workbuilder/util.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: util.cc 3637 2012-01-25 19:28:06Z sriramr $ 3 | // 4 | // Created 2011/01/04 5 | // 6 | // Copyright 2011 Yahoo Corporation. All rights reserved. 7 | // This file is part of the Sailfish project. 8 | // 9 | // Licensed under the Apache License, Version 2.0 10 | // (the "License"); you may not use this file except in compliance with 11 | // the License. You may obtain a copy of the License at 12 | // 13 | // http://www.apache.org/licenses/LICENSE-2.0 14 | // 15 | // Unless required by applicable law or agreed to in writing, software 16 | // distributed under the License is distributed on an "AS IS" BASIS, 17 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 18 | // implied. See the License for the specific language governing 19 | // permissions and limitations under the License. 20 | // 21 | // 22 | //---------------------------------------------------------------------------- 23 | 24 | #include "util.h" 25 | #include 26 | using namespace workbuilder; 27 | 28 | using std::string; 29 | using std::vector; 30 | 31 | void 32 | workbuilder::split(const string &str, vector &parts, char sep) 33 | { 34 | string::size_type start = 0; 35 | string::size_type slash = 0; 36 | 37 | while (slash != string::npos) { 38 | assert(slash == 0 || str[slash] == sep); 39 | slash = str.find(sep, start); 40 | string nextc = str.substr(start, slash - start); 41 | start = slash + 1; 42 | parts.push_back(nextc); 43 | } 44 | } 45 | 46 | void 47 | workbuilder::splitPair(const string &str, vector &pair, char sep) 48 | { 49 | string::size_type slash = str.rfind(sep); 50 | 51 | if (slash == string::npos) 52 | return; 53 | string s = str.substr(0, slash); 54 | pair.push_back(s); 55 | s = str.substr(slash+1); 56 | pair.push_back(s); 57 | } 58 | 59 | const char * 60 | workbuilder::now(void) 61 | { 62 | static char buf[128]; 63 | static time_t tt = 0; 64 | time_t t = time(0); 65 | if (t != tt) { 66 | tt = t; 67 | struct tm *tm = localtime(&t); 68 | strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", tm); 69 | buf[sizeof(buf)-1]=0; 70 | } 71 | return buf; 72 | } 73 | 74 | int 75 | workbuilder::MemAlignedMalloc(void **ptr, size_t alignment, size_t size) 76 | { 77 | if (alignment < sizeof(void *)) { 78 | alignment = sizeof(void *); 79 | } 80 | 81 | // check for power of 2 82 | bool isPowerOf2 = ((alignment != 0) && !(alignment & (alignment - 1))); 83 | 84 | assert(isPowerOf2); 85 | 86 | free(*ptr); 87 | *ptr = NULL; 88 | return posix_memalign(ptr, alignment, size); 89 | } 90 | -------------------------------------------------------------------------------- /src/cc/sailfish/workbuilder/util.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: util.h 3637 2012-01-25 19:28:06Z sriramr $ 3 | // 4 | // Created 2011/01/04 5 | // 6 | // Copyright 2011 Yahoo Corporation. All rights reserved. 7 | // This file is part of the Sailfish project. 8 | // 9 | // Licensed under the Apache License, Version 2.0 10 | // (the "License"); you may not use this file except in compliance with 11 | // the License. You may obtain a copy of the License at 12 | // 13 | // http://www.apache.org/licenses/LICENSE-2.0 14 | // 15 | // Unless required by applicable law or agreed to in writing, software 16 | // distributed under the License is distributed on an "AS IS" BASIS, 17 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 18 | // implied. See the License for the specific language governing 19 | // permissions and limitations under the License. 20 | // 21 | // 22 | //---------------------------------------------------------------------------- 23 | 24 | #ifndef WORKBUILDER_UTIL_H 25 | #define WORKBUILDER_UTIL_H 26 | #include 27 | #include 28 | #include 29 | namespace workbuilder 30 | { 31 | void split(const std::string &str, std::vector &parts, char sep); 32 | void splitPair(const std::string &str, std::vector &pair, char sep); 33 | const char *now(); 34 | // Effectively a wrapper around posix_memalign()---this function 35 | // ensures that the args meet the requirements of posix_memalign() 36 | int MemAlignedMalloc(void **ptr, size_t alignment, size_t size); 37 | template 38 | int MemAlignedMalloc(T **ptr, size_t alignment, size_t size) { 39 | return MemAlignedMalloc((void **) ptr, alignment, size); 40 | } 41 | } 42 | 43 | #endif // WORKBUILDER_UTIL_H 44 | -------------------------------------------------------------------------------- /src/cc/telemetry/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # $Id: CMakeLists.txt 392 2010-05-27 20:11:26Z sriramsrao $ 3 | # 4 | # Created 2008/09/13 5 | # 6 | # Copyright 2008 Quantcast Corporation. 7 | # 8 | # This file is part of Kosmos File System (KFS). 9 | # 10 | # Licensed under the Apache License, Version 2.0 11 | # (the "License"); you may not use this file except in compliance with 12 | # the License. You may obtain a copy of the License at 13 | # 14 | # http://www.apache.org/licenses/LICENSE-2.0 15 | # 16 | # Unless required by applicable law or agreed to in writing, software 17 | # distributed under the License is distributed on an "AS IS" BASIS, 18 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | # implied. See the License for the specific language governing 20 | # permissions and limitations under the License. 21 | # 22 | # 23 | 24 | set (exe_files 25 | telemetry_server 26 | telemetry_repeater 27 | ) 28 | 29 | # 30 | # Every executable depends on its namesake source with _main.cc 31 | # 32 | foreach (exe_file ${exe_files}) 33 | add_executable (${exe_file} ${exe_file}_main.cc) 34 | if (USE_STATIC_LIB_LINKAGE) 35 | target_link_libraries (${exe_file} kfsCommon qcdio pthread) 36 | if (CMAKE_SYSTEM_NAME STREQUAL "SunOS") 37 | target_link_libraries (${exe_file} socket nsl) 38 | endif (CMAKE_SYSTEM_NAME STREQUAL "SunOS") 39 | add_dependencies (${exe_file} kfsCommon qcdio) 40 | else (USE_STATIC_LIB_LINKAGE) 41 | target_link_libraries (${exe_file} kfsCommon-shared qcdio-shared pthread) 42 | if (CMAKE_SYSTEM_NAME STREQUAL "SunOS") 43 | target_link_libraries (${exe_file} socket nsl) 44 | endif (CMAKE_SYSTEM_NAME STREQUAL "SunOS") 45 | add_dependencies (${exe_file} kfsCommon-shared qcdio-shared) 46 | endif (USE_STATIC_LIB_LINKAGE) 47 | if (NOT APPLE) 48 | target_link_libraries(${exe_file} rt) 49 | endif (NOT APPLE) 50 | 51 | endforeach (exe_file) 52 | 53 | # 54 | install (TARGETS ${exe_files} 55 | RUNTIME DESTINATION bin) 56 | 57 | 58 | -------------------------------------------------------------------------------- /src/cc/telemetry/telemetry_server.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: telemetry_server.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2008/09/13 5 | // 6 | // 7 | // Copyright 2008 Quantcast Corporation. All rights reserved. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \file Telemetry_srv.h 24 | // \brief Telemetry server for KFS. This periodically sends out a 25 | // multicast packet identifying slow nodes in the system. 26 | // 27 | //---------------------------------------------------------------------------- 28 | 29 | 30 | #ifndef TELEMETRY_TELEMETRY_SRV_H 31 | #define TELEMETRY_TELEMETRY_SRV_H 32 | 33 | #include 34 | #include 35 | #include 36 | 37 | #include "common/cxxutil.h" 38 | #include "packet.h" 39 | 40 | namespace KFS 41 | { 42 | struct NodeState_t { 43 | // # of packets we have received for this node in the last 44 | // period. 45 | int numSamplesInPeriod; 46 | double totalTimeInPeriod; 47 | double movingAvg; 48 | NodeState_t() : numSamplesInPeriod(0), totalTimeInPeriod(0.0), movingAvg(0.0) { } 49 | void startPeriod() { 50 | numSamplesInPeriod = 0; 51 | totalTimeInPeriod = 0.0; 52 | } 53 | void sampleData(double timespent) { 54 | numSamplesInPeriod++; 55 | totalTimeInPeriod += timespent; 56 | } 57 | void endPeriod() { 58 | movingAvg /= 2.0; 59 | if (numSamplesInPeriod > 0) 60 | movingAvg += (totalTimeInPeriod / numSamplesInPeriod); 61 | startPeriod(); 62 | } 63 | }; 64 | 65 | typedef std::tr1::unordered_map NodeMap; 66 | typedef std::tr1::unordered_map::iterator NodeMapIter; 67 | 68 | // update state of nodes based on the packets we get from the socket. 69 | void gatherState(int sock); 70 | 71 | // update state of all nodes in system 72 | void updateState(); 73 | // update state of a specific node 74 | void updateNodeState(TelemetryClntPacket_t &tpkt); 75 | 76 | // distribute the list of slow nodes 77 | void distributeState(int sock); 78 | 79 | } 80 | 81 | #endif // TELEMETRY_TELEMETRY_SRV_H 82 | -------------------------------------------------------------------------------- /src/cc/tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # $Id: CMakeLists.txt 392 2010-05-27 20:11:26Z sriramsrao $ 3 | # 4 | # Created 2006/10/20 5 | # 6 | # Copyright 2006 Kosmix Corp. 7 | # 8 | # This file is part of Kosmos File System (KFS). 9 | # 10 | # Licensed under the Apache License, Version 2.0 11 | # (the "License"); you may not use this file except in compliance with 12 | # the License. You may obtain a copy of the License at 13 | # 14 | # http://www.apache.org/licenses/LICENSE-2.0 15 | # 16 | # Unless required by applicable law or agreed to in writing, software 17 | # distributed under the License is distributed on an "AS IS" BASIS, 18 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | # implied. See the License for the specific language governing 20 | # permissions and limitations under the License. 21 | # 22 | # 23 | 24 | set (exe_files 25 | KfsDataGen 26 | KfsDirFileTester 27 | KfsPerfReader 28 | KfsPerfWriter 29 | KfsRecordAppend 30 | KfsReader 31 | KfsSeekWrite 32 | KfsTrunc 33 | KfsWriter 34 | KfsDirScanTest 35 | KfsIndexReader 36 | mkfstree 37 | KfsRW 38 | KfsLogTest 39 | memory_pinner 40 | ) 41 | 42 | # 43 | # Every executable depends on its namesake source with _main.cc 44 | # 45 | foreach (exe_file ${exe_files}) 46 | add_executable (${exe_file} ${exe_file}_main.cc) 47 | if (USE_STATIC_LIB_LINKAGE) 48 | add_dependencies (${exe_file} kfsClient) 49 | target_link_libraries (${exe_file} tools kfsClient) 50 | else (USE_STATIC_LIB_LINKAGE) 51 | add_dependencies (${exe_file} kfsClient-shared) 52 | target_link_libraries (${exe_file} tools-shared kfsClient-shared) 53 | endif (USE_STATIC_LIB_LINKAGE) 54 | 55 | endforeach (exe_file) 56 | 57 | # 58 | install (TARGETS ${exe_files} 59 | RUNTIME DESTINATION bin/tests) 60 | 61 | 62 | -------------------------------------------------------------------------------- /src/cc/tests/KfsDataGen_main.cc: -------------------------------------------------------------------------------- 1 | 2 | // 3 | // Generate out test data to write/read from KFS and validate for correctness. 4 | // 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | using std::cout; 12 | using std::endl; 13 | using std::ofstream; 14 | 15 | void generateData(char *buf, int numBytes); 16 | int 17 | main(int argc, char **argv) 18 | { 19 | int fsize; 20 | 21 | if (argc < 2) { 22 | cout << "Usage: " << argv[0] << " " << endl; 23 | exit(0); 24 | } 25 | fsize = atoi(argv[2]); 26 | generateData(argv[1], fsize); 27 | 28 | } 29 | 30 | void 31 | generateData(char *fname, int numBytes) 32 | { 33 | ofstream ofs(fname); 34 | int i; 35 | 36 | if (!ofs) { 37 | cout << "Unable to open file: " << fname << endl; 38 | return; 39 | } 40 | 41 | for (i = 0; i < numBytes; i++) { 42 | ofs << (char) ('a' + (rand() % 26)); 43 | } 44 | ofs.close(); 45 | } 46 | 47 | 48 | -------------------------------------------------------------------------------- /src/cc/tests/KfsLogTest_main.cc: -------------------------------------------------------------------------------- 1 | #include "common/BufferedLogWriter.h" 2 | #include "qcdio/qcutils.h" 3 | #include "common/properties.h" 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | using namespace KFS; 14 | using namespace std; 15 | 16 | static uint64_t Now() 17 | { 18 | struct timeval theTime; 19 | if (gettimeofday(&theTime, 0)) { 20 | QCUtils::FatalError("gettimeofday", errno); 21 | } 22 | return (int64_t(theTime.tv_sec) * 1000000 + theTime.tv_usec); 23 | } 24 | 25 | int main(int argc, char** argv) 26 | { 27 | BufferedLogWriter theLogWriter(fileno(stderr), 0, 1 << 20); 28 | if (argc > 1) { 29 | Properties theProps; 30 | if (theProps.loadProperties(argv[1], '=', true) != 0) { 31 | return 1; 32 | } 33 | theLogWriter.SetParameters(theProps, "logWriter."); 34 | } 35 | if (argc > 2) { 36 | int theMax = (int)atof(argv[2]); 37 | uint64_t theStart = Now(); 38 | uint64_t thePrev = theStart; 39 | for (int i = 1; i <= theMax; i++) { 40 | theLogWriter.Append(BufferedLogWriter::kLogLevelDEBUG, "%d", i); 41 | if ((i & 0xFFFFF) == 0) { 42 | const uint64_t theNow = Now(); 43 | cout << i * 1e6 / (double(theNow - theStart) + 1e-7) << 44 | " rec/sec avg " << 45 | 1e6 * 0xFFFFF / (double(theNow - thePrev) + 1e-7) << 46 | " rec/sec\n"; 47 | thePrev = theNow; 48 | } 49 | } 50 | } else { 51 | string theLine; 52 | while (getline(cin, theLine)) { 53 | theLogWriter.Append(BufferedLogWriter::kLogLevelDEBUG, 54 | "%s", theLine.c_str()); 55 | } 56 | } 57 | return 0; 58 | } 59 | -------------------------------------------------------------------------------- /src/cc/tests/KfsTester.properties.sample: -------------------------------------------------------------------------------- 1 | # Configuration for kfs client tester: 2 | metaServer.name = sriram.corp.kosmix.com 3 | metaServer.port = 10001 4 | 5 | -------------------------------------------------------------------------------- /src/cc/tests/memory_pinner_main.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: memory_pinner_main.cc $ 3 | // 4 | // Created 2011/03/07 5 | // 6 | // Copyright 2011 Yahoo Corporation. All rights reserved. 7 | // Yahoo PROPRIETARY and CONFIDENTIAL. 8 | // 9 | // \brief Tool that pins down main memory. This provides a simple way 10 | // to lock down the memory and reduce memory available for buffer cache. 11 | //---------------------------------------------------------------------------- 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | using std::cout; 23 | using std::endl; 24 | 25 | int main(int argc, char **argv) 26 | { 27 | off_t memSizeToPin; 28 | void *bufStart; 29 | 30 | if (argc < 2) { 31 | cout << "Usage: " << argv[0] << " " << endl; 32 | exit(0); 33 | } 34 | memSizeToPin = boost::lexical_cast(argv[1]); 35 | 36 | bufStart = mmap(NULL, memSizeToPin, PROT_READ|PROT_WRITE, 37 | MAP_PRIVATE|MAP_ANON|MAP_NORESERVE, -1, 0); 38 | if (bufStart == NULL) { 39 | cout << "Unable to mmap memory: error = " << errno << endl; 40 | exit(-1); 41 | } 42 | if (mlock(bufStart, memSizeToPin) < 0) { 43 | cout << "Unable to lock memory: error = " << errno << " which is: " 44 | << strerror(errno) << endl; 45 | exit(-1); 46 | } 47 | // Set this to some value so that the page is really there 48 | char *dataBuf = (char *) bufStart; 49 | for (off_t i = 0; i < memSizeToPin; i++) { 50 | dataBuf[i] = 'a'; 51 | } 52 | cout << "Got the memory...hit cntrl-c to exit" << endl; 53 | while (1) { 54 | sleep(60); 55 | } 56 | } 57 | 58 | -------------------------------------------------------------------------------- /src/cc/tools/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | # Build the tools library 3 | set (lib_srcs 4 | MonUtils.cc 5 | KfsCd.cc 6 | KfsChangeReplication.cc 7 | KfsCp.cc 8 | KfsFstat.cc 9 | KfsLs.cc 10 | KfsMkdirs.cc 11 | KfsMv.cc 12 | KfsRm.cc 13 | KfsRmdir.cc 14 | KfsPwd.cc 15 | KfsAppend.cc 16 | KfsToolsCommon.cc 17 | utils.cc 18 | ) 19 | 20 | add_library (tools STATIC ${lib_srcs}) 21 | add_library (tools-shared SHARED ${lib_srcs}) 22 | set_target_properties (tools-shared PROPERTIES OUTPUT_NAME "tools") 23 | set_target_properties (tools PROPERTIES CLEAN_DIRECT_OUTPUT 1) 24 | set_target_properties (tools-shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) 25 | 26 | add_dependencies (tools kfsClient) 27 | add_dependencies (tools-shared kfsClient-shared) 28 | 29 | set (exe_files 30 | cptokfs 31 | cpfromkfs 32 | kfsshell 33 | kfsping 34 | kfstoggleworm 35 | kfsput 36 | kfsstats 37 | kfsfileenum 38 | kfsdataverify 39 | kfsretire 40 | kfscp 41 | kfsls 42 | ) 43 | 44 | if (APPLE) 45 | target_link_libraries(tools-shared kfsClient-shared kfsCommon-shared) 46 | endif (APPLE) 47 | 48 | # 49 | # Every executable depends on its namesake source with _main.cc 50 | # 51 | foreach (exe_file ${exe_files}) 52 | add_executable (${exe_file} ${exe_file}_main.cc) 53 | if (USE_STATIC_LIB_LINKAGE) 54 | add_dependencies (${exe_file} tools) 55 | target_link_libraries (${exe_file} tools kfsClient) 56 | else (USE_STATIC_LIB_LINKAGE) 57 | add_dependencies (${exe_file} tools-shared) 58 | target_link_libraries (${exe_file} tools-shared kfsClient-shared) 59 | endif (USE_STATIC_LIB_LINKAGE) 60 | endforeach (exe_file) 61 | 62 | if (LIBEDIT_FOUND) 63 | target_link_libraries(kfsshell ${LIBEDIT_LIBRARY}) 64 | endif (LIBEDIT_FOUND) 65 | 66 | # 67 | install (TARGETS ${exe_files} tools tools-shared 68 | RUNTIME DESTINATION bin/tools 69 | LIBRARY DESTINATION lib 70 | ARCHIVE DESTINATION lib/static) 71 | -------------------------------------------------------------------------------- /src/cc/tools/KfsAppend.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: KfsAppend.cc 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2007/09/20 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2007-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief Tool that prints the cwd. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #include 28 | #include 29 | #include 30 | 31 | extern "C" { 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | } 39 | 40 | #include "libkfsClient/KfsClient.h" 41 | #include "tools/KfsShell.h" 42 | 43 | #include 44 | 45 | using std::cout; 46 | using std::endl; 47 | using std::vector; 48 | using std::string; 49 | 50 | using namespace KFS; 51 | 52 | int 53 | KFS::tools::handleAppend(const vector &args) 54 | { 55 | if (args.size() != 2 || args[0] == "--help") { 56 | cout << "Usage: append src dst " << endl; 57 | return 0; 58 | } 59 | 60 | KfsClientPtr const kfsClient = getKfsClientFactory()->GetClient(); 61 | if (! kfsClient) { 62 | cout << "unable to create kfs client" << endl; 63 | return -EINVAL; 64 | } 65 | off_t offset = -1; 66 | const int ret = kfsClient->CoalesceBlocks(args[0].c_str(), args[1].c_str(), &offset); 67 | cout << "append status: " << ret << " offset: " << offset << endl; 68 | return ret; 69 | } 70 | -------------------------------------------------------------------------------- /src/cc/tools/KfsCd.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: KfsCd.cc 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2007/09/20 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2007-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief Tool that changes the cwd. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #include 28 | #include 29 | #include 30 | 31 | extern "C" { 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | } 39 | 40 | #include "libkfsClient/KfsClient.h" 41 | #include "tools/KfsShell.h" 42 | 43 | #include 44 | 45 | using std::cout; 46 | using std::endl; 47 | using std::vector; 48 | using std::string; 49 | 50 | using namespace KFS; 51 | 52 | int 53 | KFS::tools::handleCd(const vector &args) 54 | { 55 | if ((args.size() < 1) || (args[0] == "--help")) { 56 | cout << "Usage: cd " << endl; 57 | return 0; 58 | } 59 | 60 | KfsClientPtr kfsClient = getKfsClientFactory()->GetClient(); 61 | 62 | int res; 63 | 64 | if ((res = kfsClient->Cd(args[0].c_str())) < 0) { 65 | cout << "cd failed: " << ErrorCodeToStr(res) << endl; 66 | return res; 67 | } 68 | return 0; 69 | } 70 | 71 | -------------------------------------------------------------------------------- /src/cc/tools/KfsChangeReplication.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: KfsChangeReplication.cc 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2007/09/20 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2007-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief Tool that changes the degree of replication for a file. 24 | // This tool asks the metaserver to change the degree of replication; 25 | // the change, presuming resources exist, will be done by the 26 | // metaserver asynchronously. 27 | // 28 | //---------------------------------------------------------------------------- 29 | 30 | #include 31 | #include 32 | #include 33 | 34 | extern "C" { 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | } 42 | 43 | #include "libkfsClient/KfsClient.h" 44 | #include "tools/KfsShell.h" 45 | 46 | #include 47 | 48 | using std::cout; 49 | using std::endl; 50 | using std::vector; 51 | using std::string; 52 | 53 | using namespace KFS; 54 | 55 | int 56 | KFS::tools::handleChangeReplication(const vector &args) 57 | { 58 | if ((args.size() < 2) || (args[0] == "--help")) { 59 | cout << "Usage: changeReplication " << endl; 60 | return 0; 61 | } 62 | 63 | KfsClientPtr kfsClient = getKfsClientFactory()->GetClient(); 64 | 65 | int16_t numReplicas = 1; 66 | int res; 67 | 68 | try { 69 | numReplicas = (int16_t) boost::lexical_cast (args[1]); 70 | } catch (...) { 71 | cout << "Print : " << args[1] << " is not a valid replication factor" << endl; 72 | return -EINVAL; 73 | } 74 | 75 | if ((res = kfsClient->SetReplicationFactor(args[0].c_str(), numReplicas)) < 0) { 76 | cout << "Set replication failed: " << ErrorCodeToStr(res) << endl; 77 | return res; 78 | } 79 | return 0; 80 | } 81 | 82 | -------------------------------------------------------------------------------- /src/cc/tools/KfsFstat.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: KfsFstat.cc 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2008/12/15 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // 8 | // This file is part of Kosmos File System (KFS). 9 | // 10 | // Licensed under the Apache License, Version 2.0 11 | // (the "License"); you may not use this file except in compliance with 12 | // the License. You may obtain a copy of the License at 13 | // 14 | // http://www.apache.org/licenses/LICENSE-2.0 15 | // 16 | // Unless required by applicable law or agreed to in writing, software 17 | // distributed under the License is distributed on an "AS IS" BASIS, 18 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | // implied. See the License for the specific language governing 20 | // permissions and limitations under the License. 21 | // 22 | // \brief Tool for fstat'ing a file. 23 | // 24 | //---------------------------------------------------------------------------- 25 | 26 | #include 27 | #include 28 | #include 29 | 30 | extern "C" { 31 | #include 32 | #include 33 | #include 34 | #include 35 | } 36 | 37 | #include "libkfsClient/KfsClient.h" 38 | #include "tools/KfsShell.h" 39 | 40 | using std::cout; 41 | using std::endl; 42 | using std::ofstream; 43 | using std::vector; 44 | 45 | using namespace KFS; 46 | 47 | int 48 | KFS::tools::handleFstat(const vector &args) 49 | { 50 | if ((args.size() >= 1) && (args[0] == "--help")) { 51 | cout << "Usage: stat {} " << endl; 52 | return 0; 53 | } 54 | 55 | KfsClientPtr kfsClient = getKfsClientFactory()->GetClient(); 56 | 57 | struct stat statInfo; 58 | kfsClient->Stat(args[0].c_str(), statInfo); 59 | 60 | cout << "File: " << args[0] << endl; 61 | cout << "ctime: " << statInfo.st_ctime << endl; 62 | cout << "mtime: " << statInfo.st_mtime << endl; 63 | cout << "Size: " << statInfo.st_size << endl; 64 | 65 | return 0; 66 | } 67 | -------------------------------------------------------------------------------- /src/cc/tools/KfsMkdirs.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: KfsMkdirs.cc 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2006/09/21 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2006-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief Tool that implements mkdir -p 24 | //---------------------------------------------------------------------------- 25 | 26 | #include 27 | #include 28 | #include 29 | 30 | extern "C" { 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | } 38 | 39 | #include "libkfsClient/KfsClient.h" 40 | #include "tools/KfsShell.h" 41 | 42 | using std::cout; 43 | using std::endl; 44 | using std::vector; 45 | using std::string; 46 | 47 | using namespace KFS; 48 | using namespace KFS::tools; 49 | 50 | int 51 | KFS::tools::handleMkdirs(const vector &args) 52 | { 53 | if ((args.size() < 1) || (args[0] == "--help") || (args[0] == "")) { 54 | cout << "Usage: mkdir " << endl; 55 | return 0; 56 | } 57 | 58 | return doMkdirs(args[0].c_str()); 59 | } 60 | -------------------------------------------------------------------------------- /src/cc/tools/KfsMv.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: KfsMv.cc 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2007/09/24 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2007-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief Tool that renames a file/directory from a KFS path to another 24 | // KFS path. This does the analogous of "mv". 25 | // 26 | //---------------------------------------------------------------------------- 27 | 28 | #include 29 | #include 30 | #include 31 | 32 | extern "C" { 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | } 40 | 41 | #include "libkfsClient/KfsClient.h" 42 | #include "tools/KfsShell.h" 43 | 44 | using std::cout; 45 | using std::endl; 46 | using std::ifstream; 47 | 48 | using namespace KFS; 49 | using namespace KFS::tools; 50 | 51 | 52 | int 53 | KFS::tools::handleMv(const vector &args) 54 | { 55 | int res; 56 | 57 | if ((args.size() < 2) || (args[0] == "--help") || (args[0] == "") || (args[1] == "")) { 58 | cout << "Usage: mv " << endl; 59 | return 0; 60 | } 61 | 62 | KfsClientPtr kfsClient = getKfsClientFactory()->GetClient(); 63 | 64 | if (!kfsClient->Exists(args[0].c_str())) { 65 | cout << "Source path: " << args[0] << " is non-existent!" << endl; 66 | return -ENOENT; 67 | } 68 | 69 | string target = args[1].c_str(); 70 | 71 | if (kfsClient->IsFile(args[0].c_str()) && 72 | kfsClient->IsDirectory(args[1].c_str())) { 73 | string parent, filename; 74 | 75 | GetPathComponents(args[0], parent, filename); 76 | target += "/"; 77 | target += filename; 78 | } 79 | 80 | if ((res = kfsClient->Rename(args[0].c_str(), target.c_str())) < 0) { 81 | cout << "Rename failed: " << ErrorCodeToStr(res) << endl; 82 | return res; 83 | } 84 | return 0; 85 | } 86 | -------------------------------------------------------------------------------- /src/cc/tools/KfsPwd.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: KfsPwd.cc 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2007/09/20 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2007-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief Tool that prints the cwd. 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #include 28 | #include 29 | #include 30 | 31 | extern "C" { 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | } 39 | 40 | #include "libkfsClient/KfsClient.h" 41 | #include "tools/KfsShell.h" 42 | 43 | #include 44 | 45 | using std::cout; 46 | using std::endl; 47 | using std::vector; 48 | using std::string; 49 | 50 | using namespace KFS; 51 | 52 | int 53 | KFS::tools::handlePwd(const vector &args) 54 | { 55 | if ((args.size() >= 1) && (args[0] == "--help")) { 56 | cout << "Usage: pwd " << endl; 57 | return 0; 58 | } 59 | 60 | KfsClientPtr kfsClient = getKfsClientFactory()->GetClient(); 61 | 62 | string pwd = kfsClient->GetCwd(); 63 | 64 | cout << pwd << endl; 65 | return 0; 66 | } 67 | 68 | -------------------------------------------------------------------------------- /src/cc/tools/KfsRmdir.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: KfsRmdir.cc 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2006/09/21 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2006-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief Tool that implements rmdir 24 | //---------------------------------------------------------------------------- 25 | 26 | #include 27 | #include 28 | #include 29 | 30 | extern "C" { 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | } 38 | 39 | #include "libkfsClient/KfsClient.h" 40 | #include "tools/KfsShell.h" 41 | 42 | using std::cout; 43 | using std::endl; 44 | using std::vector; 45 | using std::string; 46 | 47 | using namespace KFS; 48 | using namespace KFS::tools; 49 | 50 | int 51 | KFS::tools::handleRmdir(const vector &args) 52 | { 53 | if ((args.size() < 1) || (args[0] == "--help") || (args[0] == "")) { 54 | cout << "Usage: rmdir " << endl; 55 | return 0; 56 | } 57 | 58 | return doRmdir(args[0].c_str()); 59 | } 60 | -------------------------------------------------------------------------------- /src/cc/tools/KfsShell.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: KfsShell.h 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2007/09/26 5 | // 6 | // 7 | // Copyright 2008 Quantcast Corp. 8 | // Copyright 2007-2008 Kosmix Corp. 9 | // 10 | // This file is part of Kosmos File System (KFS). 11 | // 12 | // Licensed under the Apache License, Version 2.0 13 | // (the "License"); you may not use this file except in compliance with 14 | // the License. You may obtain a copy of the License at 15 | // 16 | // http://www.apache.org/licenses/LICENSE-2.0 17 | // 18 | // Unless required by applicable law or agreed to in writing, software 19 | // distributed under the License is distributed on an "AS IS" BASIS, 20 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | // implied. See the License for the specific language governing 22 | // permissions and limitations under the License. 23 | // 24 | // \brief A simple shell that lets users navigate KFS directory hierarchy. 25 | // 26 | //---------------------------------------------------------------------------- 27 | 28 | #ifndef TOOLS_KFSSHELL_H 29 | #define TOOLS_KFSSHELL_H 30 | 31 | extern "C" { 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | } 39 | 40 | #include 41 | #include 42 | 43 | namespace KFS 44 | { 45 | namespace tools 46 | { 47 | 48 | typedef int (*cmdHandler)(const std::vector &args); 49 | // cmd handlers 50 | int handleCd(const std::vector &args); 51 | int handleChangeReplication(const std::vector &args); 52 | int handleCopy(const std::vector &args); 53 | int handleFstat(const std::vector &args); 54 | int handleLs(const std::vector &args); 55 | int handleMkdirs(const std::vector &args); 56 | int handleMv(const std::vector &args); 57 | int handleRmdir(const std::vector &args); 58 | int handlePing(const std::vector &args); 59 | int handleRm(const std::vector &args); 60 | int handlePwd(const std::vector &args); 61 | int handleAppend(const std::vector &args); 62 | // utility functions 63 | int doMkdirs(const char *path); 64 | int doRmdir(const char *dirname); 65 | void GetPathComponents(const std::string &path, 66 | std::string &parent, std::string &name); 67 | 68 | } 69 | } 70 | 71 | #endif // TOOLS_KFSSHELL_H 72 | -------------------------------------------------------------------------------- /src/cc/tools/utils.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // $Id: utils.cc 1552 2011-01-06 22:21:54Z sriramr $ 3 | // 4 | // Created 2007/09/20 5 | // 6 | // Copyright 2008 Quantcast Corp. 7 | // Copyright 2007-2008 Kosmix Corp. 8 | // 9 | // This file is part of Kosmos File System (KFS). 10 | // 11 | // Licensed under the Apache License, Version 2.0 12 | // (the "License"); you may not use this file except in compliance with 13 | // the License. You may obtain a copy of the License at 14 | // 15 | // http://www.apache.org/licenses/LICENSE-2.0 16 | // 17 | // Unless required by applicable law or agreed to in writing, software 18 | // distributed under the License is distributed on an "AS IS" BASIS, 19 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 20 | // implied. See the License for the specific language governing 21 | // permissions and limitations under the License. 22 | // 23 | // \brief Common utility functions for KfsShell 24 | // 25 | //---------------------------------------------------------------------------- 26 | 27 | #include 28 | #include 29 | 30 | #include "libkfsClient/KfsClient.h" 31 | #include "tools/KfsShell.h" 32 | 33 | using std::cout; 34 | using std::endl; 35 | using namespace KFS; 36 | 37 | // Make the directory hierarchy in KFS defined by path. 38 | 39 | int 40 | KFS::tools::doMkdirs(const char *path) 41 | { 42 | int res; 43 | KfsClientPtr kfsClient = getKfsClientFactory()->GetClient(); 44 | 45 | res = kfsClient->Mkdirs((char *) path); 46 | if ((res < 0) && (res != -EEXIST)) { 47 | cout << "Mkdir failed: " << ErrorCodeToStr(res) << endl; 48 | return res; 49 | } 50 | return res; 51 | } 52 | 53 | // remove a single directory in kfs 54 | 55 | int 56 | KFS::tools::doRmdir(const char *dirname) 57 | { 58 | int res; 59 | KfsClientPtr kfsClient = getKfsClientFactory()->GetClient(); 60 | 61 | res = kfsClient->Rmdir(dirname); 62 | if (res < 0) { 63 | cout << "unable to rmdir: " << dirname << ':' << ErrorCodeToStr(res) << endl; 64 | return res; 65 | } 66 | return 0; 67 | } 68 | 69 | void 70 | KFS::tools::GetPathComponents(const string &path, string &parent, string &name) 71 | { 72 | string::size_type slash = path.rfind('/'); 73 | 74 | // get everything after the last slash 75 | if (slash != string::npos) { 76 | parent.assign(path, 0, slash); 77 | name.assign(path, slash+1, string::npos); 78 | } else { 79 | name = path; 80 | parent = "/"; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/java/org/kosmix/kosmosfs/access/KfsFileAttr.java: -------------------------------------------------------------------------------- 1 | /** 2 | * $Id: KfsFileAttr.java 92 2008-07-21 21:20:48Z sriramsrao $ 3 | * 4 | * Created 2008/07/20 5 | * 6 | * @author: Sriram Rao 7 | * 8 | * Copyright 2008 Quantcast Corp. 9 | * 10 | * This file is part of Kosmos File System (KFS). 11 | * 12 | * Licensed under the Apache License, Version 2.0 13 | * (the "License"); you may not use this file except in compliance with 14 | * the License. You may obtain a copy of the License at 15 | * 16 | * http://www.apache.org/licenses/LICENSE-2.0 17 | * 18 | * Unless required by applicable law or agreed to in writing, software 19 | * distributed under the License is distributed on an "AS IS" BASIS, 20 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | * implied. See the License for the specific language governing 22 | * permissions and limitations under the License. 23 | * 24 | * \brief A struct that defines the kfs file attributes that get returned back. 25 | */ 26 | 27 | package org.kosmix.kosmosfs.access; 28 | 29 | public class KfsFileAttr 30 | { 31 | public KfsFileAttr() { } 32 | public String filename; 33 | public boolean isDirectory; 34 | public long filesize; 35 | public long modificationTime; 36 | public int replication; 37 | } 38 | -------------------------------------------------------------------------------- /src/java/org/kosmix/kosmosfs/access/Positionable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * $Id: Positionable.java 71 2008-07-07 15:49:14Z sriramsrao $ 3 | * 4 | * Created 2007/09/13 5 | * 6 | * @author: Sriram Rao 7 | * 8 | * Copyright 2008 Quantcast Corp. 9 | * Copyright 2007-2008 Kosmix Corp. 10 | * 11 | * This file is part of Kosmos File System (KFS). 12 | * 13 | * Licensed under the Apache License, Version 2.0 14 | * (the "License"); you may not use this file except in compliance with 15 | * the License. You may obtain a copy of the License at 16 | * 17 | * http://www.apache.org/licenses/LICENSE-2.0 18 | * 19 | * Unless required by applicable law or agreed to in writing, software 20 | * distributed under the License is distributed on an "AS IS" BASIS, 21 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 22 | * implied. See the License for the specific language governing 23 | * permissions and limitations under the License. 24 | * 25 | * \brief Interface for positioning a file pointer. 26 | */ 27 | 28 | package org.kosmix.kosmosfs.access; 29 | 30 | import java.io.IOException; 31 | 32 | public interface Positionable 33 | { 34 | /* 35 | * Position a file pointer at the specified offset from the 36 | * beginning of a file. 37 | */ 38 | int seek(long offset) throws IOException; 39 | 40 | /* 41 | * Return the current position of the file pointer---the position 42 | * is from the beginning of the file. 43 | */ 44 | long tell() throws IOException; 45 | } 46 | -------------------------------------------------------------------------------- /src/python/flogger/flogger.cfg: -------------------------------------------------------------------------------- 1 | # 2 | # $Id: flogger.cfg 15 2007-09-18 07:19:30Z sriramsrao $ 3 | # 4 | # Copyright 2006 Kosmix Corp. 5 | # 6 | # Author: Blake Lewis (Kosmix Corp.) 7 | # 8 | # This file is part of Kosmos File System (KFS). 9 | # 10 | # Licensed under the Apache License, Version 2.0 11 | # (the "License"); you may not use this file except in compliance with 12 | # the License. You may obtain a copy of the License at 13 | # 14 | # http://www.apache.org/licenses/LICENSE-2.0 15 | # 16 | # Unless required by applicable law or agreed to in writing, software 17 | # distributed under the License is distributed on an "AS IS" BASIS, 18 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | # 20 | # This file defines parameters for the KFS test program flogger.py. 21 | # It is read using the ConfigParser module. The values defined here 22 | # are actually the same as the defaults contained in the program, so 23 | # this is just for illustration. 24 | # 25 | # A sample KfsClient.prp file is provided in conf directory. That file 26 | # defines the location of the metaserver. To run this program, a 27 | # suitable KfsClient.prp should be created where this program is executed. 28 | # 29 | # 30 | # XXX The log file should probably be a command-line argument. 31 | # 32 | [Test parameters] 33 | testname = flogger 34 | kfs_properties: KfsClient.prp 35 | log_file: %(testname)s.log 36 | op_count: 5000 37 | max_file_size: 100000 38 | max_tree_depth: 100 39 | max_files: 10000 40 | max_directories: 1000 41 | max_subdirs: 50 42 | max_dirsize: 500 43 | -------------------------------------------------------------------------------- /src/python/flogger/readdirplus.py: -------------------------------------------------------------------------------- 1 | # 2 | # $Id: readdirplus.py 12 2007-09-16 22:35:15Z sriramsrao $ 3 | # 4 | # Copyright 2006 Kosmix Corp. 5 | # 6 | # Author: Blake Lewis (Kosmix Corp.) 7 | # 8 | # This file is part of Kosmos File System (KFS). 9 | # 10 | # Licensed under the Apache License, Version 2.0 11 | # (the "License"); you may not use this file except in compliance with 12 | # the License. You may obtain a copy of the License at 13 | # 14 | # http://www.apache.org/licenses/LICENSE-2.0 15 | # 16 | # Unless required by applicable law or agreed to in writing, software 17 | # distributed under the License is distributed on an "AS IS" BASIS, 18 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | # implied. See the License for the specific language governing 20 | # permissions and limitations under the License. 21 | # 22 | # KFS readdirplus definitions 23 | # 24 | """ 25 | A little module that defines offsets and a few simple functions 26 | for accessing KFS readdirplus data, in the spirit of the standard 27 | 'stat' module. 28 | """ 29 | 30 | RD_NAME = 0 31 | RD_FILEID = 1 32 | RD_MTIME = 2 33 | RD_CTIME = 3 34 | RD_CRTIME = 4 35 | RD_TYPE = 5 36 | RD_SIZE = 6 37 | 38 | def rd_isreg(rdtuple): 39 | return rdtuple[RD_TYPE] == "file" 40 | 41 | def rd_isdir(rdtuple): 42 | return rdtuple[RD_TYPE] == "dir" 43 | 44 | def rd_name(rdtuple): 45 | return rdtuple[RD_NAME] 46 | 47 | def rd_size(rdtuple): 48 | return rdtuple[RD_SIZE] 49 | 50 | def rd_id(rdtuple): 51 | return rdtuple[RD_FILEID] 52 | -------------------------------------------------------------------------------- /src/python/rmr/rmr.py: -------------------------------------------------------------------------------- 1 | # 2 | # $Id: rmr.py 12 2007-09-16 22:35:15Z sriramsrao $ 3 | # 4 | # Copyright 2006 Kosmix Corp. 5 | # 6 | # Author: Blake Lewis (Kosmix Corp.) 7 | # 8 | # This file is part of Kosmos File System (KFS). 9 | # 10 | # Licensed under the Apache License, Version 2.0 11 | # (the "License"); you may not use this file except in compliance with 12 | # the License. You may obtain a copy of the License at 13 | # 14 | # http://www.apache.org/licenses/LICENSE-2.0 15 | # 16 | # Unless required by applicable law or agreed to in writing, software 17 | # distributed under the License is distributed on an "AS IS" BASIS, 18 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 19 | # implied. See the License for the specific language governing 20 | # permissions and limitations under the License. 21 | # 22 | # do a recursive remove on a KFS directory 23 | # 24 | import kfs 25 | from readdirplus import * 26 | from stat import * 27 | 28 | def rmr(client, path): 29 | if S_ISREG(client.stat(path)[ST_MODE]): 30 | client.remove(path) 31 | else: 32 | do_rmr(client, path) 33 | 34 | def do_rmr(client, path): 35 | rdp = client.readdirplus(path) 36 | dot = [r for r in rdp if rd_name(r) == '.'] 37 | my_id = rd_id(dot[0]) 38 | subs = [r for r in rdp if rd_isdir(r) and rd_id(r) != my_id and \ 39 | rd_name(r) != '..'] 40 | plain = [r for r in rdp if rd_isreg(r)] 41 | for r in plain: 42 | client.remove(path + "/" + rd_name(r)) 43 | for r in subs: 44 | do_rmr(client, path + "/" + rd_name(r)) 45 | client.rmdir(path) 46 | -------------------------------------------------------------------------------- /src/python/unittests/KfsClient.prp: -------------------------------------------------------------------------------- 1 | #$Id: KfsClient.prp 182 2008-10-04 00:11:28Z sriramsrao $ 2 | #Sample properties for use with test scripts 3 | # 4 | metaServer.name = localhost 5 | metaServer.port = 20000 6 | -------------------------------------------------------------------------------- /src/set_svnprops.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd $( dirname $0 ) 4 | 5 | for i in $( find . ../examples -name \*.cpp -or -name \*.cc -or -name \*.h -or -name \*.py -or -name \*.java ); do svn propset svn:keywords Id $i; done | grep -v "property 'svn:keywords' set on" 6 | -------------------------------------------------------------------------------- /webui/files/sorttable/arrow-down.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sriramsrao/kfs/8bce116e87f31b1e91f75c09e8aec19c1cafffd4/webui/files/sorttable/arrow-down.gif -------------------------------------------------------------------------------- /webui/files/sorttable/arrow-none.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sriramsrao/kfs/8bce116e87f31b1e91f75c09e8aec19c1cafffd4/webui/files/sorttable/arrow-none.gif -------------------------------------------------------------------------------- /webui/files/sorttable/arrow-up.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sriramsrao/kfs/8bce116e87f31b1e91f75c09e8aec19c1cafffd4/webui/files/sorttable/arrow-up.gif -------------------------------------------------------------------------------- /webui/files/sorttable/sorttable.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sriramsrao/kfs/8bce116e87f31b1e91f75c09e8aec19c1cafffd4/webui/files/sorttable/sorttable.js --------------------------------------------------------------------------------