├── LICENSE ├── README ├── apps └── cloudfs │ ├── Jamfile │ ├── cloudfs.c │ └── replicator.c ├── diffs ├── README.diffs ├── lib_vmkctl.diff ├── scons.diff └── vim.diff ├── lib └── cloudfs │ ├── Jamfile │ ├── Subdir.mk │ ├── Subdir.sc │ ├── cloudfsdb.c │ ├── cloudfsdb.h │ ├── cloudfslib.c │ └── cloudfslib.h ├── modules └── vmkernel │ └── cloudfs │ ├── Jamfile │ ├── Subdir.sc │ ├── aligned.h │ ├── bTreeRange.c │ ├── bTreeRange.h │ ├── binHeap.c │ ├── binHeap.h │ ├── bitOps.h │ ├── bitops.h │ ├── btree.c │ ├── btree.h │ ├── common.c │ ├── common.h │ ├── compressor.c │ ├── conditionvariable.h │ ├── demandFetch.h │ ├── dump.c │ ├── fingerPrint.c │ ├── fingerPrint.h │ ├── globals.h │ ├── graph.c │ ├── graph.h │ ├── hashDb.c │ ├── hashDb.h │ ├── hex.h │ ├── httplib │ ├── Jamfile │ ├── parseHttp.c │ └── parseHttp.h │ ├── log.c │ ├── logCompactor.c │ ├── logModule.c │ ├── logfs.c │ ├── logfsCheckPoint.c │ ├── logfsCheckPoint.h │ ├── logfsConstants.h │ ├── logfsDiskLayout.h │ ├── logfsHash.h │ ├── logfsHttpClient.c │ ├── logfsHttpClient.h │ ├── logfsHttpd.c │ ├── logfsHttpd.h │ ├── logfsIO.c │ ├── logfsIO.h │ ├── logfsLog.h │ ├── logfsNet.c │ ├── logfsNet.h │ ├── logfsPosix.c │ ├── logfsVsi.c │ ├── logfs_int.h │ ├── logfs_vsi.h │ ├── logtypes.h │ ├── metaLog.c │ ├── metaLog.h │ ├── migBSDNet.h │ ├── module.c │ ├── obsoleted.c │ ├── obsoleted.h │ ├── pagedTree.c │ ├── pagedTree.h │ ├── rangemap.c │ ├── rangemap.h │ ├── remoteLog.c │ ├── remoteLog.h │ ├── segmentlist.h │ ├── shalib │ ├── Jamfile │ ├── macros.h │ ├── nettle-meta.h │ ├── nettle-types.h │ ├── sha.h │ ├── sha1-compress.c │ ├── sha1-meta.c │ └── sha1.c │ ├── showlog.c │ ├── system.h │ ├── vDisk.c │ ├── vDisk.h │ ├── vDiskMap.c │ ├── vDiskMap.h │ ├── vebTree.c │ ├── vebTree.h │ └── vmk │ └── Jamrules └── papers └── osr.pdf /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted (subject to the limitations in the 5 | disclaimer below) provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the 13 | distribution. 14 | 15 | * Neither the name of VMware nor the names of its 16 | contributors may be used to endorse or promote products derived 17 | from this software without specific prior written permission. 18 | 19 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 20 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 21 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 22 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 23 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 28 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 29 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 30 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 31 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | -------------------------------------------------------------------------------- /apps/cloudfs/Jamfile: -------------------------------------------------------------------------------- 1 | if ( ! $(cloudfstools) ) { cloudfstools = 1 ; 2 | SubDir TOP bora apps cloudfs ; 3 | 4 | SubDirHdrs $(TOP) bora modules vmkernel cloudfs ; 5 | SubDirHdrs $(TOP) bora modules vmkernel cloudfs httplib ; 6 | SubDirHdrs $(TOP) bora lib cloudfs ; 7 | SubDirHdrs $(TOP) bora public ; 8 | SubDirHdrs $(TOP) bora lib public ; 9 | 10 | UWMain cloudfstool : cloudfs.c ; 11 | LinkLibraries cloudfstool : cloudfslib libsha httplib ; 12 | 13 | UWMain replicator : replicator.c ; 14 | LinkLibraries replicator : cloudfslib libsha httplib ; 15 | 16 | LINKLIBS on replicator cloudfstool += -lpthread -ldl -lcurl -lz -lcrypto -lssl -lvmkuser 17 | $(TOP)/bora/build/scons/build/vmlibs/$(BUILDTYPE)/uw32/SUBDIRS/bora/lib/string/lib-string.a 18 | $(TOP)/bora/build/scons/build/vmlibs/obj/uw32/SUBDIRS/bora/lib/misc/lib-misc.a ; 19 | 20 | LINKFLAGS on replicator cloudfstool += 21 | -L/build/toolchain/lin32/curl-7.19.5-3/lib 22 | -L/build/toolchain/lin32/zlib-1.2.3-3/lib/ 23 | -L/build/toolchain/lin32/openssl-0.9.8m/lib/ 24 | -L$(TOP)/bora/build/scons/build/LIBRARIES/vmkuserlib/uw32/obj 25 | ; 26 | 27 | SubInclude TOP bora lib cloudfs ; 28 | SubInclude TOP bora apps cloudfs ; 29 | SubInclude TOP bora modules vmkernel cloudfs shalib ; 30 | SubInclude TOP bora modules vmkernel cloudfs httplib ; 31 | 32 | } 33 | -------------------------------------------------------------------------------- /diffs/README.diffs: -------------------------------------------------------------------------------- 1 | Files in this directory are relevant for those with VMware source code access, 2 | as they show what must be changed to hook in to the VMware build system, and 3 | how to make CloudFS appear as a datastore in the VMware management stack 4 | ("vim"). 5 | 6 | -------------------------------------------------------------------------------- /diffs/scons.diff: -------------------------------------------------------------------------------- 1 | diff --git a/scons/apps/esx/cloudfstools.sc b/scons/apps/esx/cloudfstools.sc 2 | new file mode 100644 3 | index 0000000..9446bc6 4 | --- /dev/null 5 | +++ b/scons/apps/esx/cloudfstools.sc 6 | @@ -0,0 +1,46 @@ 7 | +import vmware 8 | +import os 9 | + 10 | +env = vmware.DefaultEnvironment() 11 | + 12 | +vmware.LoadTool(env, ['libcurl','libz','vmkuserlib']) 13 | + 14 | + 15 | + 16 | +toolchain = '/build/toolchain' 17 | +bits = '32' if vmware.Host().Is32Bit() else '64' 18 | +opensslDir = 'openssl-0.9.8g' 19 | +cryptoPath = os.path.join(toolchain, 'lin' + bits, opensslDir, 'lib') 20 | + 21 | + 22 | +env.Append(CPPPATH = ['#bora/modules/vmkernel/cloudfs', 23 | + '#bora/modules/vmkernel/cloudfs/httplib', 24 | + '#bora/lib/cloudfs', 25 | + '#bora/public', 26 | + ], 27 | + CPPDEFINES = {'USERLEVEL': None}, 28 | + LIBS = ['dl','z','pthread','curl'], 29 | + LINKFLAGS = ['-Wl,-rpath ' + cryptoPath], 30 | + ) 31 | + 32 | +r = vmware.Executable('replicator', env = env, allowInsecureSymbols = True) 33 | +r.addLibs('vmlibs', ['cloudfs','string', 'misc']) 34 | +r.AddDynamicSubdir('#bora/apps/cloudfs', files=[ 'replicator.c' ] , env = env) 35 | +rnode = r.createProgramNode(env, basename = 'replicator') 36 | + 37 | +vmware.RegisterNode(rnode, 'replicator') 38 | +vmware.Alias('replicator-build', rnode) 39 | +vmware.RegisterNode(rnode, 'esx-apps') 40 | + 41 | + 42 | + 43 | +c = vmware.Executable('cloudfstool', env = env, allowInsecureSymbols = True) 44 | +c.addLibs('vmlibs', ['cloudfs','string', 'misc']) 45 | +c.AddDynamicSubdir('#bora/apps/cloudfs', files=[ 'cloudfs.c' ] , env = env) 46 | +cnode = c.createProgramNode(env, basename = 'cloudfstool') 47 | + 48 | +vmware.RegisterNode(cnode, 'cloudfstool') 49 | +vmware.Alias('cloudfs-build', cnode) 50 | +vmware.RegisterNode(cnode, 'esx-apps') 51 | + 52 | + 53 | diff --git a/scons/modules/hostd.sc b/scons/modules/hostd.sc 54 | index 72ccff6..1b59208 100644 55 | --- a/scons/modules/hostd.sc 56 | +++ b/scons/modules/hostd.sc 57 | @@ -83,6 +83,7 @@ commonBoraLibs = Split(''' 58 | uuid 59 | workerLib 60 | url 61 | + cloudfs 62 | ''') 63 | 64 | if commonEnv.Host().IsWindows(): 65 | diff --git a/scons/modules/vmkModules.sc b/scons/modules/vmkModules.sc 66 | index bf840f8..a217119 100644 67 | --- a/scons/modules/vmkModules.sc 68 | +++ b/scons/modules/vmkModules.sc 69 | @@ -319,6 +319,7 @@ defineModule('applesmc', envAllVmkapi) 70 | defineModule('overlay_test', envVDS) 71 | defineModule('fence_overlay', envVDS) 72 | defineModule('svmmirror', envAllVmkapi) 73 | +defineModule('cloudfs', envAllVmkapi) 74 | 75 | defineModule('dma_mapper_null', envAllVmkapi, baseDir = 'dma/mappers/null') 76 | 77 | diff --git a/scons/modules/vmkctllib.sc b/scons/modules/vmkctllib.sc 78 | index 530b05e..9d9f329 100644 79 | --- a/scons/modules/vmkctllib.sc 80 | +++ b/scons/modules/vmkctllib.sc 81 | @@ -20,6 +20,7 @@ env.Append (CPPPATH = Split('''#bora/lib/vmkctl/include 82 | [ vmware.HeaderDirectory('lib-vmksysinfo-headers'), 83 | '#bora/modules/vmkernel/migrate', 84 | '#bora/modules/vmkernel/nfsclient', 85 | + '#bora/modules/vmkernel/cloudfs', 86 | '#bora/modules/vmkernel/cbrc_filter', 87 | '#bora/modules/vmkernel/lvmdriver', 88 | '#bora/modules/vmkernel/heartbeat', 89 | @@ -33,6 +34,7 @@ env.Append (CPPPATH = Split('''#bora/lib/vmkctl/include 90 | '#bora/modules/vmkernel/iso9660', 91 | vmware.HeaderDirectory('vsi-vmkmod-migrate', host = 'vmkernel64'), 92 | vmware.HeaderDirectory('vsi-vmkmod-nfsclient', host = 'vmkernel64'), 93 | + vmware.HeaderDirectory('vsi-vmkmod-cloudfs', host = 'vmkernel64'), 94 | vmware.HeaderDirectory('vsi-vmkmod-cbrc_filter', host = 'vmkernel64'), 95 | vmware.HeaderDirectory('vsi-vmkmod-vmkibft', host = 'vmkernel64'), 96 | vmware.HeaderDirectory('vsi-vmkmod-vfat', host = 'vmkernel64'), 97 | diff --git a/scons/products/esx.sc b/scons/products/esx.sc 98 | index c50713e..cb647b0 100644 99 | --- a/scons/products/esx.sc 100 | +++ b/scons/products/esx.sc 101 | @@ -163,9 +163,14 @@ class Esx(ProductDefinition): 102 | 'modules/vpxa.sc',], 103 | not skip_hostd_vpxa), 104 | 105 | + 106 | ('uw32', 107 | ['apps/esx/cifsauthd.sc'], 108 | not skip_cifs), 109 | + 110 | + # cloudfs 111 | + ('uw32', 112 | + ['apps/esx/cloudfstools.sc']), 113 | ] 114 | 115 | 116 | -------------------------------------------------------------------------------- /lib/cloudfs/Jamfile: -------------------------------------------------------------------------------- 1 | if ( ! $(cloudfslib) ) { cloudfslib = 1 ; 2 | 3 | SubDir TOP bora lib cloudfs ; 4 | 5 | 6 | SubDirHdrs $(TOP) bora modules vmkernel cloudfs ; 7 | SubDirHdrs $(TOP) bora modules vmkernel cloudfs httplib ; 8 | SubDirHdrs $(TOP) bora public ; 9 | SubDirHdrs $(TOP) bora lib public ; 10 | 11 | UWLibrary cloudfslib : cloudfslib.c cloudfsdb.c sqlite3.c ; 12 | 13 | } 14 | -------------------------------------------------------------------------------- /lib/cloudfs/Subdir.mk: -------------------------------------------------------------------------------- 1 | SPEC_CLIENT := src 2 | SUBDIR_FILES = libLogFS.c 3 | -------------------------------------------------------------------------------- /lib/cloudfs/Subdir.sc: -------------------------------------------------------------------------------- 1 | #env().Append(CPPDEFINES = {'USERLEVEL': None, 2 | # 'LINUX': None, 3 | # '__VMK_DAEMON__':None, 4 | # '_GNU_SOURCE':None, 5 | # 'THREADSAFE=0':None, 6 | # 'HAVE_INTPTR_T':None, 7 | # 'SQLITE_OMIT_LOAD_EXTENSION':None, 8 | # 'SQLITE_OMIT_FAULTINJECTOR':None, 9 | # 'SQLITE_ENABLE_LOCKING_STYLE':None, 10 | # 'SQLITE_FIXED_LOCKING_STYLE=dotlockLockingStyle':None} 11 | # ) 12 | 13 | files += ['cloudfslib.c', 'cloudfsdb.c', 'sqlite3.c', 14 | '../../modules/vmkernel/cloudfs/httplib/parseHttp.c', 15 | '../../modules/vmkernel/cloudfs/shalib/sha1-compress.c', 16 | '../../modules/vmkernel/cloudfs/shalib/sha1-meta.c', 17 | '../../modules/vmkernel/cloudfs/shalib/sha1.c'] 18 | 19 | includes += ['#bora/lib/cloudfs', '#bora/modules/vmkernel/cloudfs', '#bora/modules/vmkernel/cloudfs/httplib'] 20 | 21 | -------------------------------------------------------------------------------- /lib/cloudfs/cloudfsdb.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | typedef struct { 35 | int numIds; 36 | Hash hostIds[5]; 37 | char *hostNames[5]; 38 | } HostIdRecord; 39 | 40 | int initDatabase(void); 41 | int SQL(int (*callback) (void *, int, char **, char **), void *data, 42 | char *format, ...); 43 | int SQL2(int (*callback) (void *, int, char **, char **), void *data, 44 | char *format, ...); 45 | int insertDisk(Hash diskId); 46 | int insertSecretView(Hash diskId, Hash secretView); 47 | int insertPeer(const char *hostName, Hash hostId); 48 | int invalidateView(Hash secretView); 49 | int insertViewMember(Hash publicView, Hash hostId); 50 | Hash getSecretView(Hash diskId); 51 | int getPeerForHost(char *result, Hash hostId); 52 | int getNextPeer(char *result, Hash hostId); 53 | int getRandomPeer(char *result, Hash hostId); 54 | int getRandomViewMember(char *result, Hash hostId); 55 | int selectRandomPeers(char **result, Hash hostId, int howMany); 56 | int selectNRandomPeers(HostIdRecord *hir, int n); 57 | int selectHostsInView(HostIdRecord *hir, Hash publicView, Hash exclude); 58 | int selectHostsNotInView(HostIdRecord *hir, Hash publicView, int howMany); 59 | -------------------------------------------------------------------------------- /lib/cloudfs/cloudfslib.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | struct viewchange { 35 | uint8_t view[20]; 36 | uint8_t invalidates_view[20]; 37 | 38 | uint16_t num_replicas; 39 | uint8_t hosts[5][20]; 40 | 41 | } __attribute__ ((__packed__)); 42 | 43 | Hash getHostId(void); 44 | Hash getCurrentId(Hash diskId); 45 | Hash getEntropy(Hash diskId); 46 | Hash getSecret(Hash diskId); 47 | uint64 getLsn(Hash diskId); 48 | 49 | int appendHead(int fd, Hash parent, Hash id); 50 | int appendBranch(int fd, Hash parent, Hash host0, Hash host1, Hash host2, 51 | Hash secretView, Hash * retSecret); 52 | int forceSecret(int fd, Hash diskId, Hash secretView, Hash hostId, Hash newView, 53 | Hash * retSecret); 54 | int setSecret(Hash diskId, Hash secret, Hash secretView); 55 | 56 | int forwardBranch(struct log_head *head, int numHosts, int threshold, 57 | char **hostNames, Hash secret, Hash secretView); 58 | 59 | int isHostInReplicaSet(Hash hostId, struct viewchange *v); 60 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/Jamfile: -------------------------------------------------------------------------------- 1 | SubDir TOP bora modules vmkernel cloudfs ; 2 | 3 | SubDirHdrs $(TOP) bora lib cloudfs ; 4 | SubDirHdrs $(TOP) bora modules vmkernel cloudfs httplib ; 5 | 6 | VMKModule cloudfs : 7 | bTreeRange.c 8 | binHeap.c 9 | btree.c 10 | log.c 11 | logCompactor.c 12 | logModule.c 13 | logfs.c 14 | logfsHttpd.c 15 | logfsHttpClient.c 16 | logfsIO.c 17 | logfsNet.c 18 | pagedTree.c 19 | logfsPosix.c 20 | logfsCheckPoint.c 21 | metaLog.c 22 | obsoleted.c 23 | rangemap.c 24 | remoteLog.c 25 | vDisk.c 26 | vDiskMap.c 27 | logfsVsi.c 28 | common.c 29 | vebTree.c 30 | fingerPrint.c 31 | hashDb.c 32 | graph.c 33 | compressor.c 34 | ; 35 | 36 | PreprocessVSI logfs_vsi.h ; 37 | 38 | LinkLibraries cloudfs : libvmksha libvsi libvmkhttp ; 39 | 40 | # UWMain showlog : showlog.c ; 41 | 42 | SubInclude TOP bora modules vmkernel cloudfs shalib ; 43 | SubInclude TOP bora modules vmkernel cloudfs httplib ; 44 | SubInclude TOP bora lib cloudfs ; 45 | SubInclude TOP bora apps cloudfs ; 46 | 47 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/Subdir.sc: -------------------------------------------------------------------------------- 1 | files += Split(''' 2 | 3 | shalib/sha1-compress.c shalib/sha1-meta.c shalib/sha1.c 4 | 5 | binHeap.c 6 | btree.c 7 | bTreeRange.c 8 | common.c 9 | fingerPrint.c 10 | graph.c 11 | hashDb.c 12 | httplib/parseHttp.c 13 | log.c 14 | logCompactor.c 15 | logfs.c 16 | logfsCheckPoint.c 17 | logfsHttpClient.c 18 | logfsHttpd.c 19 | logfsIO.c 20 | logfsNet.c 21 | logfsPosix.c 22 | logfsVsi.c 23 | logModule.c 24 | metaLog.c 25 | obsoleted.c 26 | pagedTree.c 27 | rangemap.c 28 | remoteLog.c 29 | vDisk.c 30 | vDiskMap.c 31 | vebTree.c 32 | ''') 33 | 34 | vsi += [(['logfs_vsi.h'], 'cloudfs')] 35 | 36 | includes += [ '#bora/vmkernel/sched', 37 | '#bora/vmkernel/filesystems', 38 | '#bora/modules/vmkernel/cloudfs/httplib', 39 | ] + env()['VMKERNEL_TCPIP_INCLUDES'] 40 | 41 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/aligned.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef __ALIGNED_H__ 35 | #define __ALIGNED_H__ 36 | 37 | /* template for page-size aligned objects */ 38 | 39 | #ifndef VMKERNEL 40 | #include 41 | #include 42 | 43 | #include 44 | 45 | int posix_memalign(void **memptr, size_t alignment, size_t size); 46 | 47 | static inline int is_aligned(const void *ptr) 48 | { 49 | unsigned long va = (unsigned long)ptr; 50 | return ((va & 511) == 0); 51 | } 52 | 53 | static inline void *aligned_malloc(size_t sz) 54 | { 55 | void *mem; 56 | 57 | int r = posix_memalign(&mem, 512, sz); 58 | return (r == 0) ? mem : NULL; 59 | } 60 | 61 | #define aligned_free(_a) free((_a)) 62 | 63 | #else 64 | 65 | #include "system.h" 66 | 67 | static inline int is_aligned(const void *ptr) 68 | { 69 | VA va = (VA) ptr; 70 | return ((va & 7) == 0); 71 | } 72 | 73 | #endif 74 | 75 | #endif /* __ALIGNED_H__ */ 76 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/bTreeRange.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef __BTREE_RANGE_H__ 35 | #define __BTREE_RANGE_H__ 36 | 37 | #include 38 | 39 | #include "logtypes.h" 40 | // #include "lock.h" 41 | #include "logfsHash.h" 42 | #include "obsoleted.h" 43 | 44 | #ifdef __cplusplus 45 | extern "C" { 46 | #endif 47 | #include "rangemap.h" 48 | #ifdef __cplusplus 49 | } 50 | #endif 51 | #define MAX_INSERTS 0x1800 52 | struct LogFS_MetaLog; 53 | 54 | struct ins_elem { 55 | uint64 lsn; 56 | log_block_t from; 57 | log_block_t to; 58 | log_id_t version; 59 | }; 60 | 61 | typedef struct { 62 | 63 | struct LogFS_MetaLog *ml; 64 | 65 | struct ins_elem ins_buffer[MAX_INSERTS]; 66 | 67 | /* We keep these as 32-bit, and do the modulo when accessing 68 | * into the array */ 69 | Atomic_uint32 consumerIndex; 70 | Atomic_uint32 producerIndex; 71 | Atomic_uint32 producerStableIndex; 72 | 73 | Atomic_uint32 numBuffered; 74 | 75 | Hash diskId; 76 | uint64 lsn; 77 | Hash currentId; 78 | Hash entropy; 79 | 80 | log_segment_id_t lastLsnSegment; 81 | 82 | Semaphore sem; 83 | SP_SpinLock lock; 84 | btree_t *tree; 85 | btree_t *lsnTree; 86 | 87 | Bool isDirty; 88 | List_Links next; 89 | 90 | } LogFS_BTreeRangeMap; 91 | 92 | struct _LogFS_VDisk; 93 | struct LogFS_MetaLog; 94 | 95 | void LogFS_BTreeRangeMapPreInit(struct LogFS_MetaLog *ml); 96 | void LogFS_BTreeRangeMapCleanupGlobalState(struct LogFS_MetaLog *ml); 97 | void LogFS_BTreeRangeMapCleanup(LogFS_BTreeRangeMap *bt); 98 | 99 | void LogFS_BTreeRangeMapInit(LogFS_BTreeRangeMap *bt, 100 | struct LogFS_MetaLog *ml, 101 | Hash diskId, 102 | Hash currentId, 103 | Hash entropy); 104 | 105 | void LogFS_BTreeRangeMapMemInit(LogFS_BTreeRangeMap *bt, void *mem); 106 | void LogFS_BTreeRangeMapMemClear(LogFS_BTreeRangeMap *bt); 107 | void LogFS_BTreeRangeMapFlush(LogFS_BTreeRangeMap *bt); 108 | void LogFS_BTreeRangeMapInsert(LogFS_BTreeRangeMap *bt, 109 | log_block_t lsn, 110 | log_block_t from, 111 | log_block_t to, 112 | log_id_t version, 113 | Hash currentId, Hash entropy); 114 | 115 | VMK_ReturnStatus LogFS_BTreeRangeMapInsertSimple(LogFS_BTreeRangeMap *bt, 116 | log_block_t from, 117 | log_block_t to, 118 | log_id_t version); 119 | 120 | log_id_t LogFS_BTreeRangeMapLookup(LogFS_BTreeRangeMap *bt, log_block_t x, 121 | log_block_t * endsat); 122 | 123 | void LogFS_KickFlusher(void); 124 | 125 | VMK_ReturnStatus LogFS_BTreeRangeMapAsyncLookup(LogFS_BTreeRangeMap *bt, 126 | log_block_t x, 127 | void (*callback) (range_t, log_block_t, void *), 128 | void *data, int flags, 129 | range_t* range, 130 | log_block_t * retEndsAt); 131 | 132 | static inline int LogFS_BTreeRangeMapHighWater(LogFS_BTreeRangeMap *bt) 133 | { 134 | return ((MAX_INSERTS - Atomic_Read(&bt->numBuffered) < 2048)); 135 | } 136 | 137 | static inline int LogFS_BTreeRangeMapCanInsert(LogFS_BTreeRangeMap *bt) 138 | { 139 | return (MAX_INSERTS - Atomic_Read(&bt->numBuffered) > 16); 140 | } 141 | 142 | void LogFS_BTreeRangeMapReplace(LogFS_BTreeRangeMap *bt, 143 | log_block_t from, log_block_t to, 144 | log_id_t oldvalue, log_id_t newvalue); 145 | void LogFS_BTreeRangeMapCheckNoRefs(LogFS_BTreeRangeMap *bt, 146 | log_segment_id_t segment); 147 | void LogFS_BTreeRangeMapShow(LogFS_BTreeRangeMap *bt); 148 | void LogFS_BTreeRangeMapImport(LogFS_BTreeRangeMap *bt, 149 | LogFS_BTreeRangeMap *other); 150 | void LogFS_BTreeRangeMapClear(LogFS_BTreeRangeMap *bt); 151 | void LogFS_BTreeRangeStartFlusher(struct LogFS_MetaLog *ml, uint64 generation); 152 | 153 | VMK_ReturnStatus LogFS_BTreeRangeMapLookupLsn( 154 | LogFS_BTreeRangeMap *bt, 155 | uint64 lsn, 156 | log_segment_id_t *segment); 157 | 158 | #endif /* __BTREE_RANGE_H__ */ 159 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/binHeap.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #include "system.h" 35 | #include "binHeap.h" 36 | 37 | static inline int lessThan(LogFS_BinHeap *hp, int a, int b) 38 | { 39 | return (hp->nodes[hp->heap[a]].value < hp->nodes[hp->heap[b]].value); 40 | } 41 | 42 | void LogFS_BinHeapInit(LogFS_BinHeap *hp, int maxElems) 43 | { 44 | hp->maxElems = maxElems; 45 | 46 | hp->heap = malloc(maxElems * sizeof(int)); 47 | ASSERT(hp->heap); 48 | hp->nodes = malloc(maxElems * sizeof(HeapNode)); 49 | ASSERT(hp->nodes); 50 | 51 | int i; 52 | for (i = 0; i < maxElems; i++) { 53 | HeapNode *node = &hp->nodes[i]; 54 | node->value = 0; 55 | node->heapIndex = i; 56 | 57 | hp->heap[i] = i; 58 | } 59 | } 60 | 61 | void LogFS_BinHeapCleanup(LogFS_BinHeap *hp) 62 | { 63 | free(hp->heap); 64 | free(hp->nodes); 65 | } 66 | 67 | static inline void swapNodes(LogFS_BinHeap *hp, int parent, int child) 68 | { 69 | hp->nodes[hp->heap[parent]].heapIndex = child; 70 | hp->nodes[hp->heap[child]].heapIndex = parent; 71 | 72 | int tmp = hp->heap[parent]; 73 | hp->heap[parent] = hp->heap[child]; 74 | hp->heap[child] = tmp; 75 | } 76 | 77 | static inline void SiftUp(LogFS_BinHeap *hp, int child) 78 | { 79 | int parent; 80 | for (; child; child = parent) { 81 | parent = (child - 1) / 2; 82 | 83 | if (lessThan(hp, parent, child)) { 84 | swapNodes(hp, parent, child); 85 | } else 86 | break; 87 | } 88 | } 89 | 90 | static inline void SiftDown(LogFS_BinHeap *hp, int parent) 91 | { 92 | int child; 93 | for (;; parent = child) { 94 | child = 2 * parent + 1; 95 | 96 | if (child >= hp->maxElems) 97 | break; 98 | 99 | /* choose the larger of the two children */ 100 | if (child < hp->maxElems - 1 && lessThan(hp, child, child + 1)) { 101 | ++child; 102 | } 103 | 104 | if (lessThan(hp, child, parent)) 105 | break; 106 | 107 | swapNodes(hp, parent, child); 108 | } 109 | } 110 | 111 | int LogFS_BinHeapAdjustUp(LogFS_BinHeap *hp, int nodeIndex, unsigned howmuch) 112 | { 113 | ASSERT(nodeIndex < hp->maxElems); 114 | 115 | HeapNode *node = hp->nodes + nodeIndex; 116 | node->value += howmuch; 117 | 118 | SiftUp(hp, node->heapIndex); 119 | 120 | return node->value; 121 | } 122 | 123 | uint32 LogFS_BinHeapPopMax(LogFS_BinHeap *hp, int *value) 124 | { 125 | int top = hp->heap[0]; 126 | 127 | HeapNode *node = hp->nodes + top; 128 | *value = node->value; 129 | node->value = 0; 130 | SiftDown(hp, 0); 131 | 132 | return top; 133 | } 134 | 135 | #if 0 136 | int main(int argc, char **argv) 137 | { 138 | int i, j; 139 | int value; 140 | int oldValue; 141 | 142 | LogFS_BinHeap heap; 143 | LogFS_BinHeapInit(&heap, 6400); 144 | 145 | for (j = 0; j < 380; j++) 146 | for (i = 0; i < heap.maxElems; i++) { 147 | LogFS_BinHeapAdjustUp(&heap, i, rand() % 10000); 148 | 149 | #if 1 150 | if ((i % 7) == 0) { 151 | int r = LogFS_BinHeapPopMax(&heap, &value); 152 | if ((i % 14) == 0) { 153 | LogFS_BinHeapAdjustUp(&heap, r, value); 154 | } 155 | } 156 | #endif 157 | } 158 | 159 | int r; 160 | value = oldValue = 0x11111111; 161 | int oldR = r = 0x11111111; 162 | 163 | int n = heap.maxElems; 164 | for (i = 0; i < n; i++) { 165 | oldValue = value; 166 | oldR = r; 167 | r = LogFS_BinHeapPopMax(&heap, &value); 168 | printf("r %d max = %d.\n", r, value); 169 | assert(value <= oldValue); 170 | //assert(r<=oldR); 171 | } 172 | } 173 | #endif 174 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/binHeap.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef __LOGFS_BINHEAP_H__ 35 | #define __LOGFS_BINHEAP_H__ 36 | 37 | /* heapIndex only valid if value != 0 */ 38 | typedef struct { 39 | uint32 value; 40 | int heapIndex; 41 | } HeapNode; 42 | 43 | typedef struct { 44 | int *heap; 45 | HeapNode *nodes; 46 | int maxElems; 47 | } LogFS_BinHeap; 48 | 49 | void LogFS_BinHeapInit(LogFS_BinHeap *hp, int maxElems); 50 | void LogFS_BinHeapCleanup(LogFS_BinHeap *hp); 51 | int LogFS_BinHeapAdjustUp(LogFS_BinHeap *hp, int nodeIndex, unsigned howmuch); 52 | uint32 LogFS_BinHeapPopMax(LogFS_BinHeap *hp, int *value); 53 | 54 | #endif /* __LOGFS_BINHEAP_H__ */ 55 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/bitOps.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | /* Adapted from VMware's bitvector.h */ 35 | 36 | #ifndef __BITOPS_H__ 37 | #define __BITOPS_H__ 38 | 39 | static inline void 40 | BitSet(void *v, int n) 41 | { 42 | #if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) 43 | asm volatile("btsl %1, (%0)" 44 | :: "r" (v), "r" (n) 45 | : "cc", "memory"); 46 | #else 47 | bv->vector[BITVECTOR_INDEX(n)] |= BITVECTOR_MASK(n); 48 | #endif 49 | } 50 | 51 | static inline void 52 | BitClear(void *v, int n) 53 | { 54 | #if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) 55 | asm volatile("btrl %1, (%0)" 56 | :: "r" (v), "r" (n) 57 | : "cc", "memory"); 58 | #else 59 | bv->vector[BITVECTOR_INDEX(n)] &= ~BITVECTOR_MASK(n); 60 | #endif 61 | } 62 | 63 | static inline int 64 | BitTest(const void *v, int n) 65 | { 66 | #if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) 67 | { 68 | uint32_t tmp; 69 | 70 | asm("btl %2, (%1); " 71 | "sbbl %0, %0" 72 | : "=r" (tmp) 73 | : "r" (v), "r" (n) 74 | : "cc"); 75 | return (tmp != 0); 76 | } 77 | #else 78 | return ((bv->vector[BITVECTOR_INDEX(n)] & BITVECTOR_MASK(n)) != 0); 79 | #endif 80 | } 81 | 82 | #endif /* __BITOPS_H__ */ 83 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/bitops.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | /* Adapted from VMware's bitvector.h */ 35 | 36 | #ifndef __BITOPS_H__ 37 | #define __BITOPS_H__ 38 | 39 | static inline void 40 | BitSet(void *v, int n) 41 | { 42 | #if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) 43 | asm volatile("btsl %1, (%0)" 44 | :: "r" (v), "r" (n) 45 | : "cc", "memory"); 46 | #else 47 | bv->vector[BITVECTOR_INDEX(n)] |= BITVECTOR_MASK(n); 48 | #endif 49 | } 50 | 51 | static inline void 52 | BitClear(void *v, int n) 53 | { 54 | #if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) 55 | asm volatile("btrl %1, (%0)" 56 | :: "r" (v), "r" (n) 57 | : "cc", "memory"); 58 | #else 59 | bv->vector[BITVECTOR_INDEX(n)] &= ~BITVECTOR_MASK(n); 60 | #endif 61 | } 62 | 63 | static inline int 64 | BitTest(const void *v, int n) 65 | { 66 | #if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) 67 | { 68 | uint32_t tmp; 69 | 70 | asm("btl %2, (%1); " 71 | "sbbl %0, %0" 72 | : "=r" (tmp) 73 | : "r" (v), "r" (n) 74 | : "cc"); 75 | return (tmp != 0); 76 | } 77 | #else 78 | return ((bv->vector[BITVECTOR_INDEX(n)] & BITVECTOR_MASK(n)) != 0); 79 | #endif 80 | } 81 | 82 | #endif /* __BITOPS_H__ */ 83 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/common.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #include "common.h" 35 | #include "system.h" 36 | #include "heapsort.h" 37 | 38 | /* use a specific heap to help confine memory utiltization */ 39 | #define LogFS_HEAP "vmlogfs" 40 | #define LogFS_HEAP_INITIAL (128*1024) 41 | 42 | #ifdef TRACK_BLOCK_ORIGINS 43 | #define LogFS_HEAP_MAX (196*1024*1024) 44 | #else 45 | //#define LogFS_HEAP_MAX (128*1024*1024) 46 | #define LogFS_HEAP_MAX (384*1024*1024) 47 | #endif 48 | 49 | vmk_HeapID logfsHeap = INVALID_HEAP_ID; 50 | 51 | 52 | void qsort(void *base, size_t nmemb, size_t size, 53 | int (*compar) (const void *, const void *)) 54 | { 55 | void *tmp = malloc(size); 56 | heapsort(base, nmemb, size, compar, tmp); 57 | free(tmp); 58 | } 59 | 60 | /* 61 | *---------------------------------------------------------------------- 62 | * 63 | * LogFS_Alloc() 64 | * 65 | * Allocates memory from the logfsHeap. 66 | * 67 | * Results: 68 | * A pointer to a memory region of size size if the memory 69 | * was available. Otherwise NULL. 70 | * 71 | * Side effects: 72 | * None 73 | * 74 | *---------------------------------------------------------------------- 75 | */ 76 | 77 | //#define TRACE_ALLOCS 78 | #ifdef TRACE_ALLOCS 79 | struct { 80 | void *a; 81 | char place[64]; 82 | } allocs[0x20000]; 83 | int na; 84 | #endif 85 | 86 | void *LogFS_Alloc(size_t size, const char *fn, int line) 87 | { 88 | // zprintf("malloc %ld, from %s:%d\n",size,fn,line); 89 | void *ptr; 90 | if ((ptr = vmk_HeapAlloc(logfsHeap, size))) { 91 | 92 | #ifdef TRACE_ALLOCS 93 | allocs[na].a = ptr; 94 | sprintf(allocs[na].place, "%s:%d\n", fn, line); 95 | ++na; 96 | ASSERT(na < sizeof(allocs)/sizeof(allocs[0])); 97 | #endif 98 | 99 | memset(ptr, '\0', size); 100 | } else { 101 | zprintf("No memory available! Called from %p, %s:%d, %lu bytes", 102 | __builtin_return_address(0), fn, line, size); 103 | NOT_REACHED(); 104 | } 105 | return ptr; 106 | } 107 | 108 | /* 109 | *---------------------------------------------------------------------- 110 | * 111 | * LogFS_Free() 112 | * 113 | * Frees the memory pointed to by ptr. 114 | * 115 | * Results: 116 | * None 117 | * 118 | * Side effects: 119 | * None 120 | * 121 | *---------------------------------------------------------------------- 122 | */ 123 | void LogFS_Free(void *ptr) 124 | { 125 | if (!ptr) 126 | NOT_REACHED(); 127 | 128 | #ifdef TRACE_ALLOCS 129 | int i; 130 | for (i = 0; i < na; i++) { 131 | if (allocs[i].a == ptr) { 132 | allocs[i].a = NULL; 133 | } 134 | 135 | } 136 | #endif 137 | vmk_HeapFree(logfsHeap, ptr); 138 | } 139 | 140 | vmk_HeapID 141 | LogFS_GetHeap(void) 142 | { 143 | return logfsHeap; 144 | } 145 | 146 | 147 | VMK_ReturnStatus 148 | LogFS_CommonInit(void) 149 | { 150 | logfsHeap = Heap_Create("logfs", 151 | VMK_PAGE_SIZE, 152 | LogFS_HEAP_MAX, 153 | MM_PHYS_ANY_CONTIGUITY, 154 | MM_TYPE_ANY); 155 | 156 | if (logfsHeap == INVALID_HEAP_ID) { 157 | Warning("Unable to create heap for logfs module"); 158 | return VMK_NO_MEMORY; 159 | } 160 | 161 | return VMK_OK; 162 | } 163 | 164 | void 165 | LogFS_CommonCleanup(void) 166 | { 167 | 168 | #ifdef TRACE_ALLOCS 169 | int i; 170 | for (i = 0; i < na; i++) { 171 | if (allocs[i].a) 172 | zprintf("missing: %s\n", allocs[i].place); 173 | } 174 | na = 0; 175 | #endif 176 | 177 | if (logfsHeap != INVALID_HEAP_ID) { 178 | vmk_HeapDestroy(logfsHeap); 179 | } 180 | } 181 | 182 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/common.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | /* ********************************************************** 35 | * Copyright 2009 VMware, Inc. All rights reserved. 36 | * **********************************************************/ 37 | 38 | /* 39 | * system.h 40 | * 41 | * Defines structures and declares functions that are used across all files 42 | * in the module. 43 | */ 44 | 45 | #ifndef __COMMON_H__ 46 | #define __COMMON_H__ 47 | 48 | #include "system.h" 49 | 50 | vmk_HeapID LogFS_GetHeap(void); 51 | VMK_ReturnStatus LogFS_CommonInit(void); 52 | void LogFS_CommonCleanup(void); 53 | 54 | void *LogFS_Alloc(size_t size, const char *fn, int line); 55 | void LogFS_Free(void *ptr); 56 | 57 | void qsort(void *base, size_t nmemb, size_t size, 58 | int (*compar) (const void *, const void *)); 59 | 60 | #endif // __COMMON_H__ 61 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/compressor.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #include "logtypes.h" 35 | #include "globals.h" 36 | #include "metaLog.h" 37 | #include "vDisk.h" 38 | #include "vDiskMap.h" 39 | #include "fingerPrint.h" 40 | #include "metaLog.h" 41 | #include "hashDb.h" 42 | #include "vebTree.h" 43 | #include "graph.h" 44 | 45 | static inline Bool checkDuplicate( 46 | LogFS_HashDb *hd, 47 | LogFS_FingerPrint *fp, 48 | int idx, 49 | log_id_t *pos) 50 | { 51 | return LogFS_HashDbLookupHash(hd, LogFS_HashFromRaw(fp->fullHashes + 52 | SHA1_DIGEST_SIZE * idx), pos); 53 | } 54 | 55 | void LogFS_DedupeSegment(LogFS_MetaLog *ml, 56 | LogFS_Log *log, 57 | LogFS_VebTree *vt) 58 | { 59 | zprintf("compress segment %lu\n",LogFS_LogGetSegment(log)); 60 | 61 | VMK_ReturnStatus status; 62 | LogFS_FingerPrint *fp = ml->fingerPrints[ LogFS_LogGetSegment(log) ]; 63 | 64 | int i; 65 | int ranges=0; 66 | int blocks = 0; 67 | mk_invalid_version(last); 68 | int begin; 69 | 70 | 71 | log_id_t pos; 72 | pos.v.segment = LogFS_LogGetSegment(log); 73 | pos.v.blk_offset = 0; 74 | Bool prev = checkDuplicate(ml->hd,ml->fp,0, &pos); 75 | Bool duplicate; 76 | 77 | #if 1 78 | for(i=1, begin=0; ; ++i, prev=duplicate) { 79 | 80 | Bool stop = (i==LOG_MAX_SEGMENT_BLOCKS); 81 | 82 | pos.v.segment = LogFS_LogGetSegment(log); 83 | pos.v.blk_offset = i; 84 | 85 | duplicate = FALSE; 86 | 87 | if (stop || prev != ( duplicate = checkDuplicate(ml->hd, fp, i, &pos) ) || 88 | fp->owners[i-1].vd != fp->owners[i].vd || 89 | last.v.segment != pos.v.segment || 90 | last.v.blk_offset+(i-begin) != pos.v.blk_offset ) { 91 | 92 | /* If previous block was a duplicate, we flush since last begin */ 93 | if (prev) { 94 | 95 | log_block_t from = fp->owners[begin].blkno; 96 | log_block_t to = fp->owners[i-1].blkno + 1; 97 | 98 | if ( fp->owners[begin].vd ) { 99 | log_id_t dst = last; 100 | //dst.v.blk_offset++; /* Not sure why must point at successor? */ 101 | LogFS_BTreeRangeMap *bt = LogFS_VDiskGetVersionsMap( fp->owners[begin].vd ); 102 | 103 | blocks += to - from; 104 | for (;;) { 105 | status = LogFS_BTreeRangeMapInsertSimple(bt,from,to,dst); 106 | if (status != VMK_BUSY) { 107 | break; 108 | } 109 | CpuSched_Sleep(100); 110 | } 111 | 112 | ++ranges; 113 | } 114 | } 115 | 116 | begin = i; 117 | last = pos; 118 | 119 | if (stop) break; 120 | 121 | } 122 | 123 | //fp->owners[i].vd = NULL; 124 | 125 | 126 | } 127 | #endif 128 | 129 | /* Update old-generation VEB tree. */ 130 | LogFS_FingerPrintFinish(fp,vt,LogFS_LogGetSegment(log)); 131 | 132 | zprintf("compressed %u blocks, %u ranges\n", blocks, ranges); 133 | } 134 | 135 | struct data { 136 | LogFS_MetaLog *ml; 137 | LogFS_VebTree *vt; 138 | }; 139 | 140 | static void callback(GraphNode *n, void *data) 141 | { 142 | zprintf("cb %d\n",n->key); 143 | struct data *d = data; 144 | 145 | LogFS_Log *log = LogFS_MetaLogGetLog(d->ml,n->key); 146 | LogFS_DedupeSegment(d->ml, log, d->vt); 147 | LogFS_MetaLogPutLog(d->ml,log); 148 | 149 | } 150 | 151 | void LogFS_Compressor(LogFS_MetaLog *ml, Bool *gcExit) 152 | { 153 | 154 | while(!*gcExit) { 155 | 156 | if(ml->lurt==8) { 157 | ml->lurt = 0; 158 | 159 | zprintf("do graph\n"); 160 | Graph *g = malloc(sizeof(Graph)); 161 | Graph_Init(g); 162 | LogFS_VebTreeMerge(ml->vt,ml->vt,g); 163 | 164 | LogFS_VebTree *vt = malloc(sizeof(LogFS_VebTree)); 165 | LogFS_VebTreeInit(vt,NULL,0x200,0x10000); 166 | LogFS_VebTreeClear(vt); 167 | 168 | struct data d = {ml,vt}; 169 | Graph_Traverse(g,callback,&d); 170 | } else { 171 | CpuSched_Sleep(200); 172 | } 173 | } 174 | 175 | 176 | #if 0 177 | while(!*gcExit) { 178 | 179 | LogFS_Log *log = LogFS_ObsoletedSegmentsGetCandidate(&ml->dupes,ml); 180 | 181 | if(log != NULL) { 182 | LogFS_DedupeSegment(ml, log, vt); 183 | } else { 184 | CpuSched_Sleep(200); 185 | } 186 | 187 | } 188 | #endif 189 | 190 | 191 | } 192 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/conditionvariable.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef __CONDITIONVARIABLE_H__ 35 | #define __CONDITIONVARIABLE_H__ 36 | 37 | #ifndef VMKERNEL 38 | #include 39 | 40 | /* XXX this is some old and very bad code that is rarely used 41 | * and will be phased out soon. */ 42 | 43 | typedef struct { 44 | pthread_cond_t cv; 45 | pthread_mutex_t mutex; 46 | int counter; 47 | } LogFS_ConditionVariable; 48 | 49 | static inline void LogFS_ConditionVariableInit(LogFS_ConditionVariable *cv, 50 | unsigned rank) 51 | { 52 | pthread_mutex_init(&cv->mutex, NULL); 53 | pthread_cond_init(&cv->cv, NULL); 54 | cv->counter = 0; 55 | } 56 | 57 | static inline void LogFS_ConditionVariableWait(LogFS_ConditionVariable *cv) 58 | { 59 | pthread_mutex_lock(&cv->mutex); 60 | int oldcounter = cv->counter; 61 | while (cv->counter == oldcounter) 62 | pthread_cond_wait(&cv->cv, &cv->mutex); 63 | pthread_mutex_unlock(&cv->mutex); 64 | } 65 | 66 | static inline void LogFS_ConditionVariableSignal(LogFS_ConditionVariable *cv) 67 | { 68 | pthread_mutex_lock(&cv->mutex); 69 | ++(cv->counter); 70 | pthread_cond_signal(&cv->cv); 71 | pthread_mutex_unlock(&cv->mutex); 72 | } 73 | 74 | #else 75 | 76 | typedef struct { 77 | List_Links queue; 78 | SP_SpinLock lock; 79 | } LogFS_ConditionVariable; 80 | 81 | static inline void LogFS_ConditionVariableInit(LogFS_ConditionVariable *cv, 82 | unsigned rank) 83 | { 84 | List_Init(&cv->queue); 85 | SP_InitLock("CVLock", &cv->lock, rank); 86 | } 87 | 88 | static inline void LogFS_ConditionVariableWait(LogFS_ConditionVariable *cv) 89 | { 90 | SP_Lock(&cv->lock); 91 | CpuSched_Wait(&cv->queue, CPUSCHED_WAIT_SCSI, &cv->lock); 92 | } 93 | 94 | static inline void LogFS_ConditionVariableSignal(LogFS_ConditionVariable *cv) 95 | { 96 | CpuSched_Wakeup(&cv->queue); 97 | 98 | } 99 | 100 | #endif 101 | #endif /* __CONDITIONVARIABLE_H__ */ 102 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/demandFetch.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #if 0 35 | #include "logfsHash.h" 36 | 37 | typedef enum { LogFS_CmdFetchBlocks, LogFS_CmdFetchSecret } LogFS_CmdType; 38 | 39 | typedef struct { 40 | LogFS_CmdType type; 41 | LogFS_Hash id; 42 | log_block_t blkno; 43 | log_blockcount_t num_blocks; 44 | int ok; 45 | } __attribute__ ((__packed__)) 46 | LogFS_FetchCommand; 47 | 48 | #ifdef VMKERNEL 49 | VMK_ReturnStatus 50 | LogFS_DemandReadUpCall(Async_Token * token, 51 | LogFS_CmdType type, 52 | char *buf, 53 | Hash id, log_block_t blkno, size_t num_blocks); 54 | 55 | VMK_ReturnStatus LogFS_DemandReadDeleteForDisk(Hash diskId); 56 | #endif 57 | #endif 58 | #error "hej" 59 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/fingerPrint.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #include "fingerPrint.h" 35 | #include "vebTree.h" 36 | 37 | struct LogFS_VDisk; 38 | 39 | extern int compare_u64(const void *a, const void *b); 40 | 41 | void LogFS_FingerPrintInit(LogFS_FingerPrint *fp) 42 | { 43 | memset(fp,0,sizeof(LogFS_FingerPrint)); 44 | fp->tmp = malloc(sizeof(uint64) * LOG_MAX_SEGMENT_BLOCKS); 45 | fp->fullHashes = malloc(LOG_MAX_SEGMENT_BLOCKS*SHA1_DIGEST_SIZE); 46 | } 47 | 48 | 49 | Hash 50 | LogFS_FingerPrintLookupHash(LogFS_FingerPrint* fp, Hash h) 51 | { 52 | uint16_t key = *((uint16_t*)h.raw); 53 | 54 | HashEntry* he = fp->hashTable[key]; 55 | HashEntry* prev = NULL; 56 | 57 | while(he!=NULL && !LogFS_HashEquals(he->h,h)) 58 | { 59 | prev = he; 60 | he = he->next; 61 | } 62 | 63 | if(he==NULL) 64 | { 65 | ASSERT(fp->numHashEntriesentries)/sizeof(fp->entries[0])); 66 | he = &fp->entries[fp->numHashEntries++]; 67 | if(prev==NULL) 68 | { 69 | fp->hashTable[key] = he; 70 | } 71 | else 72 | { 73 | prev->next = he; 74 | } 75 | he->h = h; 76 | he->c = h; 77 | he->next = NULL; 78 | } 79 | else 80 | { 81 | LogFS_HashReapply(&he->c); 82 | } 83 | 84 | return he->c; 85 | 86 | } 87 | 88 | void LogFS_FingerPrintAddHash( 89 | LogFS_FingerPrint* fp, 90 | Hash h, 91 | struct LogFS_VDisk *vd, 92 | log_block_t blkno) 93 | { 94 | ASSERT(fp->numFullHashesnumFullHashes++; 97 | /* Store full hash for use in log manifest */ 98 | uint8* dst = fp->fullHashes + SHA1_DIGEST_SIZE * n; 99 | memcpy(dst,h.raw,SHA1_DIGEST_SIZE); 100 | fp->owners[n].vd = vd; 101 | fp->owners[n].blkno = blkno; 102 | 103 | if (vd==NULL) { 104 | return; 105 | } 106 | 107 | } 108 | 109 | void LogFS_FingerPrintFinish(LogFS_FingerPrint *fp, 110 | LogFS_VebTree* vt, 111 | uint32 value) 112 | { 113 | int i; 114 | uint64* tmp = fp->tmp; 115 | 116 | /* Mangle repeated hashes to ensure proportional representation */ 117 | 118 | for(i=0; ifullHashes + SHA1_DIGEST_SIZE * i)); 122 | 123 | /* We only have room for 40 of the 160 bits in the hash */ 124 | uint64_t key; 125 | memcpy(&key,h.raw,sizeof(key)); 126 | key>>=24; 127 | 128 | /* Subsample to only store hashes with a certain bit pattern. 129 | * This is based on the observations in the HP FAST 2009 dedupe paper */ 130 | 131 | if((h.raw[9]&0xf)==0) /* Arbitrarily subsample hashes 1:16 */ 132 | { 133 | ASSERT(fp->numHashesnumHashes++] = key; 135 | } 136 | 137 | } 138 | 139 | int q = fp->numHashes; 140 | qsort(tmp,q,sizeof(uint64_t),compare_u64); /* XXX use a minheap */ 141 | 142 | int j,k; 143 | for(j=0,k=0 ; j>8, value ); 149 | ++k; 150 | } 151 | } 152 | 153 | } 154 | 155 | void LogFS_FingerPrintCleanup(LogFS_FingerPrint* fp) 156 | { 157 | free(fp->tmp); 158 | fp->tmp = NULL; 159 | free(fp->fullHashes); 160 | fp->fullHashes = NULL; 161 | } 162 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/fingerPrint.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #include "logfsHash.h" 35 | 36 | typedef struct _HashEntry { 37 | Hash h; 38 | Hash c; 39 | struct _HashEntry* next; 40 | 41 | } HashEntry; 42 | 43 | #define LogFS_FingerPrintMaxKeys 512 44 | 45 | struct LogFS_VDisk; 46 | 47 | typedef struct LogFS_FingerPrint { 48 | HashEntry entries[LOG_MAX_SEGMENT_BLOCKS]; 49 | HashEntry* hashTable[0x10000]; 50 | int numHashEntries; 51 | uint64 *tmp; 52 | uint64 keys[LogFS_FingerPrintMaxKeys]; 53 | int numHashes; 54 | 55 | int numFullHashes; 56 | uint8* fullHashes; 57 | struct FingerPrintOwner { 58 | struct LogFS_VDisk *vd; 59 | log_block_t blkno; 60 | } owners[LOG_MAX_SEGMENT_BLOCKS]; 61 | 62 | } LogFS_FingerPrint; 63 | 64 | static inline int 65 | LogFS_FingerPrintIsFull(LogFS_FingerPrint* fp) 66 | { 67 | return fp->numHashesrefCount, 1); 64 | rb->buffer = buffer; 65 | rb->count = sz; 66 | return rb; 67 | } 68 | 69 | static inline void LogFS_RefCountedBufferRef(LogFS_RefCountedBuffer *rb) 70 | { 71 | Atomic_Inc(&rb->refCount); 72 | } 73 | static inline void LogFS_RefCountedBufferRelease(LogFS_RefCountedBuffer *rb) 74 | { 75 | if (Atomic_FetchAndDec(&rb->refCount) == 1) { 76 | aligned_free(rb->buffer); 77 | free(rb); 78 | } 79 | } 80 | 81 | #endif 82 | 83 | #endif /* __GLOBALS_H__ */ 84 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/graph.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef VMKERNEL 35 | #include 36 | #include 37 | #include 38 | #else 39 | #include "system.h" 40 | #endif 41 | 42 | #include "graph.h" 43 | 44 | void Graph_InitNode(Graph *g, GraphNode *n, int key) 45 | { 46 | PriQueue_Init(&n->edges); 47 | n->marked = 0; 48 | n->key = key; 49 | 50 | List_Insert(&n->nodesLinks, LIST_ATREAR(&g->nodesList)); 51 | } 52 | 53 | void Graph_Init(Graph *g) 54 | { 55 | List_Init(&g->nodesList); 56 | g->generation = 0; 57 | } 58 | 59 | void Graph_AddEdge(Graph *g, GraphNode *from, GraphNode *to, int weight) 60 | { 61 | GraphEdge *e = malloc(sizeof(GraphEdge)); 62 | e->n = to; 63 | PriQueue_Insert(&from->edges,&e->queueLinks, weight); 64 | } 65 | 66 | void Graph_Connect(Graph *g, GraphNode *from, GraphNode *to, int weight) 67 | { 68 | Graph_AddEdge(g,from,to,weight); 69 | Graph_AddEdge(g,to,from,weight); 70 | } 71 | 72 | void 73 | Graph_Traverse(Graph *g, void (*callback) (GraphNode*,void*), void *data) 74 | { 75 | List_Links *curr, *next; 76 | List_Links queue; 77 | List_Init(&queue); 78 | 79 | ++(g->generation); 80 | 81 | LIST_FORALL_SAFE(&g->nodesList, curr, next) { 82 | 83 | GraphNode *root = List_Entry( curr, GraphNode, nodesLinks); 84 | 85 | if (root->marked == g->generation) continue; 86 | root->marked = g->generation; 87 | 88 | List_Insert(&root->listLinks, LIST_ATREAR(&queue)); 89 | 90 | 91 | while(!List_IsEmpty(&queue)) { 92 | GraphNode *n; 93 | PriQueue_Links *qe; 94 | 95 | n = List_Entry(List_First(&queue), GraphNode, listLinks); 96 | 97 | PRIQUEUE_FORALL(&n->edges, qe) { 98 | 99 | GraphEdge *e = PriQueue_Entry(qe, GraphEdge, queueLinks); 100 | GraphNode *p = e->n; 101 | if(p->marked != g->generation) { 102 | p->marked = g->generation; 103 | List_Insert(&p->listLinks, LIST_ATREAR(&queue)); 104 | 105 | /* To save some looping over the nodeList, we remove the element 106 | * and move it to the front of the list. In that we do not loose 107 | * list contents, but save traversing over already visited nodes 108 | * in this run. */ 109 | 110 | List_Remove(curr); 111 | List_Insert(curr, LIST_ATFRONT(&g->nodesList)); 112 | } 113 | } 114 | 115 | callback(n,data); 116 | 117 | List_Remove(&n->listLinks); 118 | } 119 | } 120 | } 121 | static void graphCleanCallback(GraphNode *n, void *data) 122 | { 123 | PriQueue_Links *qe, *next; 124 | PRIQUEUE_FORALL_SAFE(&n->edges, qe, next) { 125 | GraphEdge *e = PriQueue_Entry(qe, GraphEdge, queueLinks); 126 | PriQueue_Remove(qe); 127 | free(e); 128 | } 129 | free(n); 130 | } 131 | 132 | void Graph_Cleanup(Graph *g) 133 | { 134 | 135 | Graph_Traverse(g, graphCleanCallback, NULL); 136 | 137 | 138 | } 139 | 140 | 141 | #ifndef VMKERNEL 142 | void callback(GraphNode *n, void *data) 143 | { 144 | printf("call %d\n",n->key); 145 | 146 | } 147 | 148 | int main(int argc, char** argv) 149 | { 150 | Graph *g = malloc(sizeof(Graph)); 151 | 152 | GraphNode* a = malloc(sizeof(GraphNode)); 153 | GraphNode* b = malloc(sizeof(GraphNode)); 154 | GraphNode* c = malloc(sizeof(GraphNode)); 155 | GraphNode* d = malloc(sizeof(GraphNode)); 156 | GraphNode* e = malloc(sizeof(GraphNode)); 157 | 158 | Graph_Init(g); 159 | Graph_InitNode(g,a,1); 160 | Graph_InitNode(g,b,2); 161 | Graph_InitNode(g,c,3); 162 | Graph_InitNode(g,d,4); 163 | 164 | Graph_InitNode(g,e,5); 165 | 166 | Graph_Connect(g,a,b,1); 167 | Graph_Connect(g,a,c,2); 168 | // Graph_Connect(g,c,d,19); 169 | 170 | 171 | Graph_Traverse(g,callback,NULL); 172 | Graph_Cleanup(g); 173 | 174 | return 0; 175 | } 176 | /* 177 | * Panic -- 178 | * Print a message to stderr and die. 179 | */ 180 | void 181 | Panic(const char *fmt, ...) 182 | { 183 | va_list ap; 184 | va_start(ap, fmt); 185 | fprintf(stderr, "cloudfs PANIC: "); 186 | vfprintf(stderr, fmt, ap); 187 | va_end(ap); 188 | exit(1); 189 | } 190 | #endif 191 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/graph.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef __GRAPH_H__ 35 | #define __GRAPH_H__ 36 | 37 | #include "list.h" 38 | #include "priQueue.h" 39 | 40 | typedef struct { 41 | 42 | PriQueue_Links edges; 43 | List_Links listLinks; 44 | List_Links nodesLinks; 45 | int key; 46 | int marked; 47 | 48 | } GraphNode ; 49 | 50 | typedef struct { 51 | GraphNode *n; 52 | PriQueue_Links queueLinks; 53 | } GraphEdge ; 54 | 55 | typedef struct Graph { 56 | List_Links nodesList; 57 | int generation; 58 | } Graph ; 59 | 60 | void Graph_InitNode(Graph *g, GraphNode *n, int key); 61 | void Graph_Init(Graph *g); 62 | void Graph_AddEdge(Graph *g, GraphNode *from, GraphNode *to, int weight); 63 | void Graph_Connect(Graph *g, GraphNode *from, GraphNode *to, int weight); 64 | void Graph_Traverse(Graph *g, void (*callback) (GraphNode*,void*), void *data); 65 | 66 | #endif /* __GRAPH_H__ */ 67 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/hashDb.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | /* ********************************************************** 35 | * **********************************************************/ 36 | 37 | /* 38 | * hashDb.c -- 39 | * 40 | * hash table with pseudo-LRU replacement. 41 | * Resolves from block hashes to their log offsets, and 42 | * updates hash database at the same time, if this is 43 | * a hash not seen before. 44 | * 45 | */ 46 | 47 | #include "logtypes.h" 48 | #include "hashDb.h" 49 | 50 | int LogFS_HashDbLookupHash(LogFS_HashDb* hd, Hash h, log_id_t *pos) 51 | { 52 | uint16_t key = *((uint16_t*)h.raw); 53 | LogFS_HashDbEntry *he = hd->hashTable[key]; 54 | LogFS_HashDbEntry *prev = NULL; 55 | 56 | /* If hash collision, follow linked linked list until the 57 | * end or a match is found. */ 58 | 59 | while(he!=NULL && memcmp(he->hash,h.raw,SHA1_DIGEST_SIZE)!=0) 60 | { 61 | prev = he; 62 | he = he->next; 63 | } 64 | 65 | /* If not found, insert into hashtable. Otherwise return 66 | * found pos. */ 67 | 68 | if(he==NULL) 69 | { 70 | /* Find existing element to evict, using pseudo-LRU */ 71 | 72 | int i; 73 | int child; 74 | for (i = 0, child = 0; i < LogFS_HashDb_LOGLINES; i++) { 75 | int parent = child; 76 | child = 2 * parent + 1 + hd->bits[parent]; 77 | hd->bits[parent] ^= 1; 78 | } 79 | int line = child - LogFS_HashDb_INNER_NODES; 80 | he = &hd->entries[line]; 81 | 82 | /* Unlink existing element from linked list */ 83 | 84 | if(he->prev) { 85 | he->prev->next = he->next; 86 | } 87 | if(he->next) { 88 | he->next->prev = he->prev; 89 | } 90 | 91 | if(prev==NULL) { 92 | /* First element with this hash */ 93 | 94 | hd->hashTable[key] = he; 95 | he->next = NULL; 96 | 97 | /* Set 'prev' to point back to hash table entry, so that 98 | * it gets unlinked like other elements upon replace. */ 99 | 100 | he->prev = (struct LogFS_HashDbEntry *) &hd->hashTable[key]; 101 | 102 | 103 | } else { 104 | 105 | /* Part of a linked list of entries */ 106 | 107 | prev->next = he; 108 | he->prev = prev; 109 | } 110 | LogFS_HashCopy(he->hash,h); 111 | he->pos = *pos; 112 | he->next = NULL; 113 | 114 | return 0; 115 | } 116 | else 117 | { 118 | /* Flip the bits in the reverse path from leaf to root */ 119 | 120 | int line = he - hd->entries; 121 | int child; 122 | for (child = line + LogFS_HashDb_INNER_NODES; child != 0;) { 123 | int parent = (child - 1) / 2; 124 | 125 | ASSERT(parent < sizeof(hd->bits)); 126 | 127 | hd->bits[parent] = (child == (2 * parent + 1)); /* inverse test to save xor */ 128 | child = parent; 129 | } 130 | 131 | *pos = he->pos; 132 | return 1; 133 | } 134 | 135 | } 136 | 137 | void LogFS_HashDbInit(LogFS_HashDb *hd) 138 | { 139 | memset(hd->hashTable,0,sizeof(hd->hashTable)); 140 | } 141 | 142 | #if 0 143 | int main(int argc,char** argv) 144 | { 145 | 146 | LogFS_HashDb *hd = malloc(sizeof(LogFS_HashDb)); 147 | 148 | printf("sz %lu\n",sizeof(LogFS_HashDb)); 149 | LogFS_HashDbInit(hd); 150 | 151 | Hash h; 152 | LogFS_HashRandomize(&h); 153 | log_id_t v; 154 | v.v.segment = 117; 155 | v.v.blk_offset = 546; 156 | 157 | LogFS_HashDbLookupHash(hd,h,v); 158 | 159 | Hash a; 160 | LogFS_HashRandomize(&a); 161 | 162 | for(;;) { 163 | LogFS_HashRandomize(&h); 164 | log_id_t v2; 165 | v2.v.segment = 0; 166 | v2.v.blk_offset = 547; 167 | v2 = LogFS_HashDbLookupHash(hd,h,v2); 168 | 169 | log_id_t v3; 170 | v3.v.segment = 7; 171 | v3.v.blk_offset = 1234; 172 | 173 | v3 = LogFS_HashDbLookupHash(hd,a,v3); 174 | } 175 | 176 | 177 | return 0; 178 | } 179 | #endif 180 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/hashDb.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #include "logfsHash.h" 35 | #define LogFS_HashDb_LOGLINES 18 36 | #define LogFS_HashDb_INNER_NODES ((1<= '0' && c <= '9') 51 | c -= '0'; 52 | else if (c >= 'a' && c <= 'f') 53 | c -= ('a' - 0xa); 54 | else if (c >= 'A' && c <= 'F') 55 | c -= ('A' - 0xa); 56 | else 57 | return -1; 58 | 59 | digit |= c << shift; 60 | shift ^= 4; 61 | 62 | if (shift) { 63 | *out++ = digit; 64 | digit = 0; 65 | } 66 | } 67 | return 0; 68 | } 69 | 70 | static inline void hex(char *out, char *in) 71 | { 72 | int i; 73 | char *o = out; 74 | char digits[] = "0123456789abcdef"; 75 | for (i = 0; i < 20; i++) { 76 | char c = *in++; 77 | *o++ = digits[(c & 0xf0) >> 4]; 78 | *o++ = digits[c & 0xf]; 79 | } 80 | *o = '\0'; 81 | } 82 | #endif /* __HEX_H__ */ 83 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/httplib/Jamfile: -------------------------------------------------------------------------------- 1 | if ( ! $(httplib) ) { httplib = 1 ; 2 | 3 | LOCATION = bora modules vmkernel cloudfs ; 4 | 5 | SubDir TOP $(LOCATION) httplib ; 6 | SubDirHdrs $(TOP) $(LOCATION) ; 7 | 8 | 9 | VMKLibrary libvmkhttp : parseHttp.c ; 10 | UWLibrary httplib : parseHttp.c ; 11 | 12 | 13 | } 14 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/httplib/parseHttp.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #include "logfsHash.h" 35 | 36 | typedef struct HTTPTransition { 37 | char *key; 38 | struct HTTPState *state; 39 | int (*callback) (char *c, void *data); 40 | } HTTPTransition; 41 | 42 | typedef struct HTTPState { 43 | char *name; 44 | HTTPTransition transitions[16]; 45 | } HTTPState; 46 | 47 | typedef enum { 48 | HTTP_GET = 1, 49 | HTTP_PUT = 2, 50 | } HTTPVerb; 51 | 52 | typedef struct { 53 | int status; 54 | HTTPVerb verb; 55 | int complete; 56 | int keepAlive; 57 | char hostName[256]; 58 | char fileName[512]; 59 | uint16_t port; 60 | size_t contentLength; 61 | 62 | size_t from; 63 | size_t to; 64 | 65 | Hash id; 66 | Hash secret; 67 | Hash secretView; 68 | 69 | } HTTPSession; 70 | 71 | typedef struct { 72 | HTTPState *state; 73 | char buf[512]; 74 | size_t putOffset; 75 | size_t bufOffset; 76 | size_t totalParsed; 77 | size_t justParsed; 78 | } HTTPParserState; 79 | 80 | int parseHttp(HTTPParserState *ps, char *in, int inputLeft, void *data); 81 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/logModule.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | /* ********************************************************** 35 | * **********************************************************/ 36 | 37 | #include "vm_types.h" 38 | #include "vm_libc.h" 39 | #include "vmkernel.h" 40 | #include "splock.h" 41 | #include "libc.h" 42 | #include "mod_loader_public.h" 43 | 44 | #include "fss_int.h" 45 | #include "logfs_int.h" 46 | #include "fs_common.h" 47 | #include "fsSwitch.h" 48 | 49 | #include "devfs.h" 50 | #include "deviceLib.h" 51 | #include "vmkapi.h" 52 | #include "module_ns.h" 53 | 54 | #define LOGLEVEL_MODULE FS3 55 | #define LOGLEVEL_MODULE_LEN 3 56 | #include "log.h" 57 | 58 | VMK_VERSION_INFO("Version" VMK_STRINGIFY(CBRCFILTER_MAJOR_VERSION) "." 59 | VMK_STRINGIFY(CBRCFILTER_MINOR_VERSION) 60 | ", Built on: " __DATE__); 61 | VMK_LICENSE_INFO(VMK_MODULE_LICENSE_VMWARE); 62 | VMK_NAMESPACE_REQUIRED(MOD_VMKERNEL_NAMESPACE, MOD_VMKERNEL_NAMESPACE_VERSION); 63 | 64 | 65 | #ifdef STATIC_LINK_VMLOGFS 66 | /* 67 | * Linking the module in directly with vmkernel. This means we need 68 | * unique names for the init and cleanup routines. (cleanup won't be called). 69 | * 70 | * The "LOGFS" prefix is also used by bootUser to find these functions. 71 | */ 72 | #define INIT_FUNC_NAME LogFS_init_module 73 | #define CLEANUP_FUNC_NAME LogFS_cleanup_module 74 | #else 75 | /* 76 | * Default dynamic module init/cleanup function names 77 | */ 78 | #define INIT_FUNC_NAME init_module 79 | #define CLEANUP_FUNC_NAME cleanup_module 80 | #endif 81 | 82 | extern FSS_FSOps logfsOps; 83 | 84 | // Registration ID obtained from FSS 85 | // Module ID for this module 86 | vmk_ModuleID logfsModuleID; 87 | 88 | /* 89 | *----------------------------------------------------------------------------- 90 | * 91 | * init_module -- 92 | * Register this FS implementation with the VMKernel FS switch. 93 | * 94 | * Results: 95 | * 0 on success, -1 on error. 96 | * 97 | * Side effects: 98 | * None. 99 | * 100 | *----------------------------------------------------------------------------- 101 | */ 102 | int INIT_FUNC_NAME(void) 103 | { 104 | VMK_ReturnStatus status; 105 | 106 | logfsModuleID = World_GetLastModID(); 107 | Log("logfsModuleID %d\n", logfsModuleID); 108 | status = LogFS_Init(); 109 | if (status != VMK_OK) { 110 | return -1; 111 | } 112 | return 0; 113 | } 114 | 115 | /* 116 | *----------------------------------------------------------------------------- 117 | * 118 | * cleanup_module -- 119 | * Called at module unload time. Cleanup module local data structures. 120 | * 121 | * Results: 122 | * None. 123 | * 124 | * 125 | * Side effects: 126 | * None. 127 | * 128 | *----------------------------------------------------------------------------- 129 | */ 130 | void CLEANUP_FUNC_NAME(void) 131 | { 132 | Log("cleanup!\n"); 133 | LogFS_Cleanup(); 134 | } 135 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/logfsCheckPoint.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef __LOGFS_CHECKPOINT_H__ 35 | #define __LOGFS_CHECKPOINT_H__ 36 | 37 | #include "logfsConstants.h" 38 | #include "logtypes.h" 39 | 40 | struct SP_SpinLock; 41 | 42 | extern uint8 nodesBitmap[TREE_MAX_BLOCKS / 8 + 1]; 43 | extern struct SP_SpinLock nodesLock; 44 | 45 | typedef struct { 46 | uint8_t checksum[20]; 47 | uint64 generation; 48 | log_id_t logEnd; 49 | disk_block_t superTreeRoot; 50 | uint8 bitmap[MAX_NUM_SEGMENTS / 8 + 1]; 51 | uint16 heap[MAX_NUM_SEGMENTS]; 52 | uint8 nodesBitmap[TREE_MAX_BLOCKS / 8 + 1]; 53 | } __attribute__ ((__packed__)) 54 | LogFS_CheckPoint; 55 | 56 | typedef struct { 57 | disk_block_t from,to; 58 | List_Links list; 59 | } MovedNode; 60 | 61 | #define LOGFS_CHECKPOINT_SIZE (BLKSIZE_ALIGNUP(sizeof(LogFS_CheckPoint))) 62 | 63 | struct LogFS_MetaLog; 64 | 65 | VMK_ReturnStatus 66 | LogFS_CheckPointPrepare(struct LogFS_MetaLog *ml, 67 | LogFS_CheckPoint *cp, 68 | uint64 generation); 69 | 70 | VMK_ReturnStatus LogFS_CheckPointCommit(struct LogFS_MetaLog *ml, 71 | LogFS_CheckPoint *cp, 72 | int buffer, 73 | disk_block_t superTreeRoot, 74 | List_Links *freedList); 75 | 76 | VMK_ReturnStatus LogFS_RecoverCheckPoint(struct LogFS_MetaLog *ml, 77 | uint64 *generation, log_id_t * logEnd, disk_block_t *superTreeRoot); 78 | 79 | VMK_ReturnStatus LogFS_ReplayFromCheckPoint(struct LogFS_MetaLog *ml, 80 | log_id_t logEnd); 81 | 82 | #endif /* __LOGFS_CHECKPOINT_H__ */ 83 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/logfsConstants.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef __LOGFSCONSTANTS_H__ 35 | #define __LOGFSCONSTANTS_H__ 36 | 37 | #define MAX_NUM_SEGMENTS 0x1000 38 | 39 | #define TREE_MAX_BLOCKS 2048 40 | #define TREE_BLOCK_SIZE (8*4096) 41 | #define MAX_FILE_SIZE (TREE_BLOCK_SIZE*TREE_MAX_BLOCKS) 42 | 43 | // KJO: copied from older version.. Revisit the lock hierarchy and see what we really need 44 | 45 | 46 | #define SP_RANK_POSIX SP_RANK_FSDRIVER_LOWEST 47 | 48 | #define SP_RANK_VDISK SP_RANK_FSDRIVER_LOWEST 49 | #define SP_RANK_DEMANDFETCH (SP_RANK_VDISK+1) 50 | 51 | #define SP_RANK_VDISKBUFFEREDRANGES (SP_RANK_VDISK+1) 52 | 53 | #define SP_RANK_METALOG (SP_RANK_VDISK+1) 54 | #define SP_RANK_APPENDLOG (SP_RANK_METALOG+1) 55 | #define SP_RANK_REFCOUNTS (SP_RANK_METALOG+1) 56 | #define SP_RANK_SEGMENTLIST (SP_RANK_METALOG+1) 57 | 58 | #define SP_RANK_DDISK (SP_RANK_VDISK+1) 59 | #define SP_RANK_REMOTELOG (SP_RANK_VDISK+1) 60 | 61 | #define SP_RANK_BTREERANGE (SP_RANK_VDISK+1) 62 | #define SP_RANK_RANGEMAP (SP_RANK_BTREERANGE+1) 63 | #define SP_RANK_RANGEMAPCACHE (SP_RANK_RANGEMAP+1) 64 | #define SP_RANK_RANGEMAPQUEUES (SP_RANK_RANGEMAPCACHE+1) 65 | #define SP_RANK_RANGEMAPNODES (SP_RANK_RANGEMAPCACHE+1) 66 | 67 | #define SP_RANK_OBSOLETED (SP_RANK_BTREERANGE+1) 68 | 69 | #endif /* __LOGFSCONSTANTS_H__ */ 70 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/logfsDiskLayout.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef __LOGFSDISKLAYOUT_H__ 35 | #define __LOGFSDISKLAYOUT_H__ 36 | 37 | #include "logtypes.h" 38 | #include "globals.h" 39 | #include "btree.h" 40 | #include "logfsCheckPoint.h" 41 | 42 | typedef enum { 43 | LogFS_DiskHeaderSection = 0, 44 | LogFS_CheckPointASection, 45 | LogFS_CheckPointBSection, 46 | LogFS_BTreeSection, 47 | LogFS_VebTreeSection, 48 | LogFS_LogSegmentsSection, 49 | 50 | LogFS_LogNumDiskSegments, 51 | } LogFS_DiskSegmentType; 52 | 53 | struct __section { 54 | uint32 type; 55 | log_offset_t offset; 56 | } __attribute__ ((__packed__)); 57 | 58 | typedef struct __LogFS_DiskLayout { 59 | char magic[8]; 60 | struct __section sections[LogFS_LogNumDiskSegments]; 61 | } __attribute__ ((__packed__)) 62 | LogFS_DiskLayout; 63 | 64 | typedef struct { 65 | uint8 key[SHA1_DIGEST_SIZE]; 66 | struct { 67 | disk_block_t root; 68 | disk_block_t lsnRoot; 69 | uint64 lsn; 70 | uint8 currentId[SHA1_DIGEST_SIZE]; 71 | uint8 entropy[SHA1_DIGEST_SIZE]; 72 | } __attribute__ ((__packed__)) value; 73 | } __attribute__ ((__packed__)) 74 | SuperTreeElement; 75 | 76 | static inline VMK_ReturnStatus 77 | LogFS_DiskLayoutInit(LogFS_DiskLayout *dl, log_size_t diskCapacity) 78 | { 79 | log_size_t headerSize = sizeof(LogFS_DiskLayout); 80 | log_size_t checkPointSize = LOGFS_CHECKPOINT_SIZE; 81 | log_size_t bTreeSize = diskCapacity / 128; 82 | 83 | log_size_t sizes[] = { 84 | headerSize, 85 | checkPointSize, checkPointSize, /* Double buffered */ 86 | bTreeSize, 87 | bTreeSize, /* XXX used by VebTree */ 88 | }; 89 | 90 | LogFS_DiskSegmentType type; 91 | log_offset_t pos; 92 | 93 | strcpy(dl->magic, "CloudFS"); 94 | 95 | for (type = LogFS_DiskHeaderSection, pos = 0; 96 | type != LogFS_LogNumDiskSegments; type++) { 97 | dl->sections[type].type = type; 98 | dl->sections[type].offset = pos; 99 | pos += BLKSIZE_ALIGNUP(sizes[type]); 100 | } 101 | 102 | return VMK_OK; 103 | } 104 | 105 | static inline log_offset_t 106 | LogFS_DiskLayoutGetOffset(LogFS_DiskLayout *dl, LogFS_DiskSegmentType type) 107 | { 108 | return dl->sections[type].offset; 109 | } 110 | 111 | #endif /* __LOGFSDISKLAYOUT_H__ */ 112 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/logfsHash.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef __HASH_H__ 35 | #define __HASH_H__ 36 | 37 | #include "system.h" 38 | #include "shalib/sha.h" 39 | 40 | #include "hex.h" 41 | #include "logtypes.h" 42 | 43 | typedef struct __LogFS_Hash { 44 | unsigned char raw[SHA1_DIGEST_SIZE]; 45 | int isValid; 46 | 47 | #ifdef __cplusplus 48 | bool operator <(struct __LogFS_Hash other) const { 49 | return (memcmp(raw, other.raw, SHA1_DIGEST_SIZE) < 0); 50 | } 51 | #endif 52 | } 53 | LogFS_Hash; 54 | 55 | static inline void LogFS_HashSetRaw(LogFS_Hash * h, const unsigned char *in) 56 | { 57 | memcpy(h->raw, in, SHA1_DIGEST_SIZE); 58 | h->isValid = 1; 59 | } 60 | 61 | static inline LogFS_Hash LogFS_HashFromRaw(const unsigned char *in) 62 | { 63 | LogFS_Hash h; 64 | memcpy(h.raw, in, SHA1_DIGEST_SIZE); 65 | h.isValid = 1; 66 | return h; 67 | } 68 | 69 | static inline LogFS_Hash LogFS_HashChecksum(const void *in, size_t sz) 70 | { 71 | LogFS_Hash h; 72 | struct sha1_ctx ctx; 73 | sha1_init(&ctx); 74 | sha1_update(&ctx, sz, (const uint8_t *)in); 75 | sha1_digest(&ctx, SHA1_DIGEST_SIZE, h.raw); 76 | h.isValid = 1; 77 | return h; 78 | } 79 | 80 | static inline void LogFS_HashCopy(unsigned char *buf, LogFS_Hash h) 81 | { 82 | #ifdef VMKERNEL 83 | ASSERT(h.isValid); 84 | #endif 85 | memcpy(buf, h.raw, SHA1_DIGEST_SIZE); 86 | } 87 | 88 | static inline void LogFS_HashClear(LogFS_Hash * h) 89 | { 90 | memset(h->raw, 0, SHA1_DIGEST_SIZE); 91 | h->isValid = 0; 92 | } 93 | 94 | static inline int LogFS_HashZero(LogFS_Hash * h) 95 | { 96 | memset(h->raw, 0, SHA1_DIGEST_SIZE); 97 | h->isValid = 1; 98 | return h->isValid; 99 | } 100 | 101 | static inline int LogFS_HashSetString(LogFS_Hash * h, const char *in) 102 | { 103 | if (strlen(in) < SHA1_HEXED_SIZE_UNTERMINATED) { 104 | #if 0 105 | printf("invalid hash input %s\n", in); 106 | #endif 107 | h->isValid = 0; 108 | } else if (unhex(h->raw, (char *)in) == 0) { 109 | h->isValid = 1; 110 | } 111 | return h->isValid; 112 | } 113 | 114 | static inline void LogFS_HashRandomize(LogFS_Hash * h) 115 | { 116 | #ifdef VMKERNEL 117 | zprintf("weak randomize in kernel\n"); 118 | uint32 seed = Util_RandSeed(); 119 | int i; 120 | for (i = 0; i < SHA1_DIGEST_SIZE; i++) 121 | h->raw[i] = seed = Util_FastRand(seed); 122 | h->isValid = 1; 123 | #else 124 | int r; 125 | int f = open("/dev/urandom", O_RDONLY); 126 | if (f < 0) { 127 | perror("could not open /dev/urandom!\n"); 128 | exit(1); 129 | } 130 | r = read(f, h->raw, sizeof(h->raw)); 131 | assert(r==sizeof(h->raw)); 132 | close(f); 133 | h->isValid = 1; 134 | #endif 135 | } 136 | 137 | static inline void LogFS_HashReapply(LogFS_Hash * h) 138 | { 139 | struct sha1_ctx ctx; 140 | sha1_init(&ctx); 141 | sha1_update(&ctx, SHA1_DIGEST_SIZE, (const uint8_t *)h->raw); 142 | sha1_digest(&ctx, SHA1_DIGEST_SIZE, h->raw); 143 | } 144 | 145 | static inline LogFS_Hash LogFS_HashApply(LogFS_Hash h) 146 | { 147 | #ifdef VMKERNEL 148 | ASSERT(h.isValid); 149 | #endif 150 | LogFS_Hash r; 151 | struct sha1_ctx ctx; 152 | sha1_init(&ctx); 153 | sha1_update(&ctx, SHA1_DIGEST_SIZE, (const uint8_t *)h.raw); 154 | sha1_digest(&ctx, SHA1_DIGEST_SIZE, r.raw); 155 | r.isValid = 1; 156 | return r; 157 | } 158 | 159 | static inline LogFS_Hash LogFS_HashXor(LogFS_Hash a, LogFS_Hash b) 160 | { 161 | int i; 162 | LogFS_Hash r; 163 | 164 | #ifdef VMKERNEL 165 | ASSERT(a.isValid && b.isValid); 166 | #else 167 | assert(a.isValid && b.isValid); 168 | #endif 169 | 170 | for (i = 0; i < SHA1_DIGEST_SIZE; i++) { 171 | r.raw[i] = a.raw[i] ^ b.raw[i]; 172 | } 173 | r.isValid = a.isValid & b.isValid; 174 | return r; 175 | } 176 | 177 | static inline int LogFS_HashIsValid(LogFS_Hash h) 178 | { 179 | return h.isValid; 180 | } 181 | 182 | static inline int LogFS_HashIsNull(LogFS_Hash h) 183 | { 184 | if (h.isValid) { 185 | int i; 186 | for (i = 0; i < SHA1_DIGEST_SIZE; i++) 187 | if (h.raw[i] != 0) 188 | return 0; 189 | return 1; 190 | } else 191 | return 0; 192 | } 193 | 194 | static inline int LogFS_HashCompare(const LogFS_Hash * a, const LogFS_Hash * b) 195 | { 196 | #ifdef VMKERNEL 197 | ASSERT(a->isValid && b->isValid); 198 | #endif 199 | return memcmp(a->raw, b->raw, SHA1_DIGEST_SIZE); 200 | } 201 | 202 | static inline int LogFS_HashEquals(const LogFS_Hash a, const LogFS_Hash b) 203 | { 204 | if (a.isValid == 0 || b.isValid == 0) 205 | return 0; 206 | return (LogFS_HashCompare(&a, &b) == 0); 207 | } 208 | 209 | static inline char *LogFS_HashPrint(char *out, const LogFS_Hash * h) 210 | { 211 | if (h->isValid) 212 | hex(out, (char *)h->raw); 213 | else 214 | strcpy(out, "++++++++++[ INVALID HASH ]+++++++++++++\0"); 215 | return out; 216 | } 217 | 218 | static inline char *LogFS_HashShow(const LogFS_Hash * h) 219 | { 220 | char *out = (char *)malloc(SHA1_HEXED_SIZE); 221 | return LogFS_HashPrint(out, h); 222 | } 223 | 224 | static inline char *LogFS_HashShow2(const LogFS_Hash h) 225 | { 226 | #ifdef VMKERNEL 227 | NOT_REACHED(); 228 | #endif 229 | char *out = (char *)malloc(SHA1_HEXED_SIZE); 230 | return LogFS_HashPrint(out, &h); 231 | } 232 | 233 | typedef LogFS_Hash Hash; 234 | 235 | #endif 236 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/logfsHttpClient.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #include "logfsNet.h" 35 | #include "metaLog.h" 36 | 37 | VMK_ReturnStatus 38 | LogFS_HttpClientRequest(Hash diskId, 39 | Hash id, 40 | Async_Token * token, 41 | char *buf, 42 | log_block_t blkno, size_t num_blocks, int flags); 43 | 44 | VMK_ReturnStatus LogFS_InitHttpClient(void); 45 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/logfsHttpd.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #include "logfsNet.h" 35 | #include "metaLog.h" 36 | 37 | struct LogFS_MetaLog; 38 | 39 | VMK_ReturnStatus LogFS_InitHttpd(struct LogFS_MetaLog *); 40 | void LogFS_CleanupHttp(void); 41 | void *LogFS_HandleHttpClient(Net_Socket client_sock, LogFS_MetaLog *log); 42 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/logfsIO.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | 35 | #include "system.h" 36 | #include "logtypes.h" 37 | #include "logfsIO.h" 38 | 39 | VMK_ReturnStatus LogFS_DeviceRead(LogFS_Device *device, 40 | Async_Token * token, 41 | void *buf, log_size_t length, 42 | log_offset_t offset, 43 | LogFS_DiskSegmentType type) 44 | { 45 | VMK_ReturnStatus status; 46 | FDS_Handle *fdsHandleArray[1]; 47 | Async_Token *newToken = NULL; 48 | 49 | log_offset_t diskOffset = 50 | LogFS_DiskLayoutGetOffset(&device->diskLayout, type) + offset; 51 | //if(type != LogFS_LogSegmentsSection) ASSERT(diskOffset < LogFS_DiskLayoutGetOffset(&device->diskLayout,type+1)); 52 | 53 | if (token == NULL) { 54 | newToken = token = Async_AllocToken(0); 55 | } 56 | 57 | SG_Array sg; 58 | SG_SingletonSGArray(&sg, diskOffset, (VA) buf, length, SG_VIRT_ADDR); 59 | fdsHandleArray[0] = device->fd; 60 | 61 | do { 62 | status = FDS_AsyncIO(fdsHandleArray, &sg, FS_READ_OP, token); 63 | } while (status == VMK_STORAGE_RETRY_OPERATION); 64 | 65 | if (newToken != NULL && status == VMK_OK) { 66 | Async_WaitForIO(token); 67 | Async_ReleaseToken(token); 68 | } 69 | 70 | return status; 71 | } 72 | 73 | VMK_ReturnStatus LogFS_DeviceWrite(LogFS_Device *device, 74 | Async_Token * token, 75 | SG_Array *sgArr, 76 | LogFS_DiskSegmentType type) 77 | { 78 | VMK_ReturnStatus status; 79 | int i; 80 | log_offset_t diskOffset = 81 | LogFS_DiskLayoutGetOffset(&device->diskLayout, type); 82 | 83 | ASSERT(token); 84 | FDS_Handle **fdsHandleArray = malloc(sgArr->length*sizeof(FDS_Handle*)); 85 | 86 | for(i=0;ilength;++i) { 87 | sgArr->sg[i].offset += diskOffset; 88 | fdsHandleArray[i] = device->fd; 89 | } 90 | 91 | do { 92 | status = FDS_AsyncIO(fdsHandleArray, sgArr, FS_WRITE_OP, token); 93 | } while (status == VMK_STORAGE_RETRY_OPERATION); 94 | 95 | free(fdsHandleArray); 96 | 97 | return status; 98 | } 99 | 100 | VMK_ReturnStatus LogFS_DeviceWriteSimple(LogFS_Device *device, 101 | Async_Token * token, 102 | const void *buf, log_size_t length, 103 | log_offset_t offset, 104 | LogFS_DiskSegmentType type) 105 | { 106 | VMK_ReturnStatus status; 107 | 108 | ASSERT(device); 109 | log_offset_t diskOffset = 110 | LogFS_DiskLayoutGetOffset(&device->diskLayout, type) + offset; 111 | FDS_Handle *fdsHandleArray[1]; 112 | Async_Token *newToken = NULL; 113 | 114 | if (token == NULL) { 115 | newToken = token = Async_AllocToken(0); 116 | } 117 | 118 | SG_Array sg; 119 | 120 | SG_SingletonSGArray(&sg, diskOffset, (VA) buf, length, SG_VIRT_ADDR); 121 | 122 | fdsHandleArray[0] = device->fd; 123 | do { 124 | status = FDS_AsyncIO(fdsHandleArray, &sg, FS_WRITE_OP, token); 125 | } while (status == VMK_STORAGE_RETRY_OPERATION); 126 | 127 | if (newToken != NULL && status == VMK_OK) { 128 | Async_WaitForIO(token); 129 | Async_ReleaseToken(token); 130 | } 131 | 132 | 133 | return status; 134 | } 135 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/logfsIO.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef __LOGFSIO_H__ 35 | #define __LOGFSIO_H__ 36 | 37 | #include "logfsDiskLayout.h" 38 | 39 | typedef struct LogFS_Device { 40 | log_device_handle_t fd; 41 | LogFS_DiskLayout diskLayout; 42 | struct btree *superTree; 43 | } LogFS_Device; 44 | 45 | static inline void 46 | LogFS_DeviceInit(LogFS_Device *dev, FDS_Handle * logfsFDSHandle) 47 | { 48 | dev->fd = logfsFDSHandle; 49 | } 50 | 51 | VMK_ReturnStatus LogFS_DeviceRead(LogFS_Device *device, Async_Token * token, 52 | void *buf, log_size_t length, 53 | log_offset_t offset, 54 | LogFS_DiskSegmentType type); 55 | 56 | VMK_ReturnStatus LogFS_DeviceWrite(LogFS_Device *device, 57 | Async_Token * token, 58 | SG_Array *sgArr, 59 | LogFS_DiskSegmentType type); 60 | 61 | VMK_ReturnStatus LogFS_DeviceWriteSimple(LogFS_Device *device, 62 | Async_Token * token, 63 | const void *buf, log_size_t length, 64 | log_offset_t offset, 65 | LogFS_DiskSegmentType type); 66 | #endif /* __LOGFSIO_H__ */ 67 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/logfsLog.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef __LOG_H__ 35 | #define __LOG_H__ 36 | 37 | #include "bTreeRange.h" 38 | #include "logfsHash.h" 39 | 40 | #include "logtypes.h" 41 | // #include "lock.h" 42 | 43 | typedef struct LogFS_Log { 44 | void *metaLog; 45 | int alive; 46 | log_segment_id_t index; 47 | Atomic_uint32 isAppendLog; 48 | Atomic_uint32 refCount; 49 | char *buffer; 50 | 51 | /* if AppendLog */ 52 | SP_SpinLock writeLock; /* protects write contexts chain */ 53 | Atomic_uint32 end; 54 | log_offset_t stableEnd; 55 | 56 | void *prevWriteContext; 57 | } LogFS_Log; 58 | 59 | static inline log_offset_t _cursor(LogFS_Log *log, log_offset_t offset) 60 | { 61 | if (!log->alive) { 62 | zprintf("log %ld is dead\n", log->index); 63 | zprintf("appendlog? %d\n", Atomic_Read(&log->isAppendLog)); 64 | } 65 | ASSERT(log->alive); 66 | return log->index * LOG_MAX_SEGMENT_SIZE + offset; 67 | } 68 | 69 | static inline log_id_t 70 | LogFS_AppendLogGetEnd(LogFS_Log *log) 71 | { 72 | log_id_t v; 73 | v.v.segment = log->index; 74 | uint32 end = Atomic_Read(&log->end); 75 | v.v.blk_offset = end/BLKSIZE; 76 | return v; 77 | } 78 | 79 | struct LogFS_MetaLog; 80 | 81 | void LogFS_LogInit(LogFS_Log *log, struct LogFS_MetaLog *metaLog, 82 | log_segment_id_t index); 83 | 84 | void LogFS_LogClose(LogFS_Log *log); 85 | 86 | log_segment_id_t LogFS_LogGetSegment(LogFS_Log *log); 87 | char *LogFS_LogEnableBuffering(LogFS_Log *log); 88 | void LogFS_AppendLogInit(LogFS_Log *log, struct LogFS_MetaLog *metaLog, 89 | log_segment_id_t index, log_offset_t end); 90 | 91 | VMK_ReturnStatus 92 | LogFS_LogWriteBody(LogFS_Log *log, Async_Token *token, SG_Array *sg, int flags); 93 | 94 | VMK_ReturnStatus LogFS_LogReadBody(LogFS_Log *log, Async_Token * token, 95 | void *buf, log_size_t count, 96 | log_offset_t offset); 97 | VMK_ReturnStatus LogFS_LogForceReadBody(LogFS_Log *log, Async_Token * token, 98 | void *buf, log_size_t count, 99 | log_offset_t offset); 100 | 101 | VMK_ReturnStatus LogFS_AppendLogAppend(LogFS_Log *log, Async_Token * token, 102 | SG_Array *sgArr, log_id_t * result, int flags); 103 | 104 | VMK_ReturnStatus LogFS_AppendLogAppendSimple(LogFS_Log *log, Async_Token * 105 | token, const void *buf, log_size_t count, log_id_t *result, int flags); 106 | 107 | int LogFS_LogIsAppendLog(LogFS_Log *log); 108 | VMK_ReturnStatus LogFS_AppendLogClose(LogFS_Log *log, Async_Token * token, 109 | int flags); 110 | void LogFS_AppendLogPushEnd(LogFS_Log *log, log_offset_t end); 111 | 112 | #endif /* __LOG_H__ */ 113 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/logfsNet.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | 35 | #include "logfsNet.h" 36 | 37 | #include "globals.h" 38 | #include "metaLog.h" 39 | 40 | #include "socket.h" 41 | 42 | VMK_ReturnStatus LogFS_NetSetSocketOptions(Net_Socket sock) 43 | { 44 | uint32 sndSize; 45 | uint32 actualSockSize = 0; 46 | uint32 sockSizeLen = 0; 47 | int optVal = 1; 48 | VMK_ReturnStatus status; 49 | 50 | for (sndSize = 0x40000; sndSize != 0; sndSize >>= 1) { 51 | status = Net_SetSockOpt(sock, SOL_SOCKET, SO_SNDBUF, 52 | &sndSize, sizeof(sndSize), DEFAULT_STACK); 53 | 54 | if (status != VMK_OK) { 55 | continue; 56 | } 57 | 58 | sockSizeLen = sizeof(actualSockSize); 59 | status = Net_GetSockOpt(sock, SOL_SOCKET, SO_SNDBUF, &actualSockSize, 60 | &sockSizeLen, DEFAULT_STACK); 61 | break; 62 | } 63 | 64 | status = Net_SetSockOpt(sock, IPPROTO_TCP, TCP_NODELAY, 65 | &optVal, sizeof(optVal), DEFAULT_STACK); 66 | 67 | return status; 68 | } 69 | 70 | VMK_ReturnStatus LogFS_NetCreateAcceptSocket(Net_Socket * acceptSocketOut) 71 | { 72 | VMK_ReturnStatus status; 73 | sockaddr_in_bsd myAddr; 74 | int optval; 75 | Net_Socket acceptSocket; 76 | 77 | Net_SetSockVal(acceptSocketOut, NULL); 78 | 79 | status = Net_CreateSocket(PF_INET, SOCK_STREAM, IPPROTO_TCP, 80 | &acceptSocket, DEFAULT_STACK); 81 | if (status != VMK_OK) { 82 | if (status == VMK_NOT_SUPPORTED) { 83 | printf("Can't create accept socket. Is the TCP/IP module loaded?"); 84 | } else { 85 | printf("Can't create accept socket: %s", 86 | VMK_ReturnStatusToString(status)); 87 | } 88 | return status; 89 | } 90 | 91 | memset(&myAddr, 0, sizeof(myAddr)); 92 | myAddr.sin_len = sizeof(myAddr); 93 | myAddr.sin_family = AF_INET; 94 | myAddr.sin_port = htons(8090); 95 | 96 | optval = 1; 97 | status = Net_SetSockOpt(acceptSocket, SOL_SOCKET, SO_REUSEPORT, 98 | &optval, sizeof optval, DEFAULT_STACK); 99 | if (status != VMK_OK) { 100 | printf("Reuse port sockopt failed: %s", VMK_ReturnStatusToString(status)); 101 | Net_CloseSocket(acceptSocket, DEFAULT_STACK); 102 | return status; 103 | } 104 | 105 | status = Net_Bind(acceptSocket, (struct sockaddr *)&myAddr, sizeof myAddr, 106 | DEFAULT_STACK); 107 | if (status != VMK_OK) { 108 | printf("Bind failed: %s", VMK_ReturnStatusToString(status)); 109 | Net_CloseSocket(acceptSocket, DEFAULT_STACK); 110 | return status; 111 | } 112 | 113 | status = Net_Listen(acceptSocket, 5, DEFAULT_STACK); 114 | if (status != VMK_OK) { 115 | printf("Error trying to listen: %s", VMK_ReturnStatusToString(status)); 116 | Net_CloseSocket(acceptSocket, DEFAULT_STACK); 117 | return status; 118 | } 119 | 120 | *acceptSocketOut = acceptSocket; 121 | return VMK_OK; 122 | } 123 | 124 | VMK_ReturnStatus LogFS_NetRead(Net_Socket socket, void *data, size_t len, 125 | int *bytesReceived) 126 | { 127 | int b; 128 | return Net_RecvFrom(socket, 0, data, len, NULL, 129 | NULL, bytesReceived ? bytesReceived : &b, DEFAULT_STACK); 130 | } 131 | 132 | VMK_ReturnStatus LogFS_NetWrite(Net_Socket socket, void *data, size_t len, 133 | int *bytesSent) 134 | { 135 | int b; 136 | return Net_SendTo(socket, 0, NULL, data, len, bytesSent ? bytesSent : &b, 137 | DEFAULT_STACK); 138 | } 139 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/logfsNet.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef __LOGFSNET_H__ 35 | #define __LOGFSNET_H__ 36 | 37 | #ifdef VMKERNEL 38 | #include "migBSDNet.h" 39 | 40 | #include "sys/types.h" 41 | #include "netinet/in.h" 42 | #include "netinet/tcp.h" 43 | #define FIXED_FOR_ESX 44 | #include "sys/socket.h" 45 | 46 | #define _SYS_INTTYPES_H_ 47 | #define __FreeBSD__ 48 | 49 | #include "vm_types.h" 50 | #include "vm_libc.h" 51 | #include "vmkernel.h" 52 | #include "return_status.h" 53 | #include "world.h" 54 | #include "vmk_net.h" 55 | #include "vmkcfgopts_public.h" 56 | #include "net.h" 57 | #include "timer.h" 58 | #include "bh_dist.h" 59 | #include "util.h" 60 | #include "memalloc_dist.h" 61 | #include "vob.h" 62 | 63 | #else 64 | #include 65 | #include 66 | #include 67 | #include 68 | #include 69 | #include 70 | #include 71 | #include 72 | #include 73 | #include 74 | 75 | #include 76 | 77 | #include "system.h" 78 | 79 | typedef int Net_Socket; 80 | 81 | #define Net_CloseSocket(_a,_b) close((_a)) 82 | 83 | #endif 84 | 85 | struct LogFS_MetaLog; 86 | 87 | VMK_ReturnStatus LogFS_NetCreateAcceptSocket(Net_Socket * acceptSocketOut); 88 | VMK_ReturnStatus LogFS_NetSetSocketOptions(Net_Socket sock); 89 | VMK_ReturnStatus LogFS_NetRead(Net_Socket socket, void *data, size_t len, 90 | int *bytesReceived); 91 | VMK_ReturnStatus LogFS_NetWrite(Net_Socket socket, void *data, size_t len, 92 | int *bytesSent); 93 | 94 | #endif /* __LOGFSNET_H__ */ 95 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/logfsVsi.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | 35 | #include "globals.h" 36 | #include "vmkernel.h" 37 | #include "log.h" 38 | 39 | #include "vm_basic_types.h" 40 | #include "vm_basic_defs.h" 41 | #include "vm_assert.h" 42 | #include "vm_libc.h" 43 | 44 | #include "vsiDefs.h" 45 | #include "logfs_vsi.h" 46 | #include "parse.h" 47 | 48 | VMK_ReturnStatus LogFS_AddPhysicalDevice(const char *deviceName); 49 | 50 | VMK_ReturnStatus 51 | LogFS_VSIDeviceGet(VSI_NodeID nodeID, 52 | VSI_ParamList * instanceArgs, VSI_LogDeviceStruct * data) 53 | { 54 | return VMK_OK; 55 | } 56 | 57 | VMK_ReturnStatus 58 | LogFS_VSIFastDeviceGet(VSI_NodeID nodeID, 59 | VSI_ParamList * instanceArgs, VSI_LogDeviceStruct * data) 60 | { 61 | return VMK_OK; 62 | } 63 | 64 | VMK_ReturnStatus 65 | LogFS_VSIDeviceSet(VSI_NodeID nodeID, 66 | VSI_ParamList * instanceArgs, 67 | VSI_ParamList * inputArgs, VSI_Empty_Output * data) 68 | { 69 | if (VSI_ParamListUsedCount(inputArgs) != 1) 70 | return VMK_BAD_PARAM_TYPE; 71 | 72 | VSI_Param *param = VSI_ParamListGetParam(inputArgs, 0); 73 | 74 | return LogFS_AddPhysicalDevice(VSI_ParamGetString(param)); 75 | 76 | } 77 | 78 | VMK_ReturnStatus 79 | LogFS_VSIFastDeviceSet(VSI_NodeID nodeID, 80 | VSI_ParamList * instanceArgs, 81 | VSI_ParamList * inputArgs, VSI_Empty_Output * data) 82 | { 83 | if (VSI_ParamListUsedCount(inputArgs) != 1) 84 | return VMK_BAD_PARAM_TYPE; 85 | 86 | VSI_Param *param = VSI_ParamListGetParam(inputArgs, 0); 87 | 88 | return LogFS_AddPhysicalDevice(VSI_ParamGetString(param)); 89 | 90 | } 91 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/logfs_int.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | VMK_ReturnStatus LogFS_Init(void); 35 | void LogFS_Cleanup(void); 36 | extern ModuleID logfsModuleID; 37 | 38 | #define LOGFS_TYPE_STRING "cloudfs" 39 | #define LOGFS_VERSION_NUMBER 1 40 | #define LOGFS_CURRENT_MINORVER 1 41 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/logfs_vsi.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | /************************************************************ 35 | * **********************************************************/ 36 | 37 | /* 38 | * logfs_vsi.h -- 39 | * 40 | * Define vsi related vsi nodes. 41 | */ 42 | 43 | #ifndef _LogFS_VSI_H 44 | #define _LogFS_VSI_H 45 | 46 | #define INCLUDE_ALLOW_VMKERNEL 47 | #define INCLUDE_ALLOW_VMKERNEL_MODULE 48 | #include "includeCheck.h" 49 | #include "vsiDefs.h" 50 | 51 | VSI_DEF_STRUCT(VSI_LogHostIdStruct, "LogFS Info Struct") 52 | { 53 | VSI_DEF_STRUCT_FIELD(VSI_DEC_U32, fsCheck, "fsCheck"); 54 | }; 55 | 56 | VSI_DEF_STRUCT(VSI_LogDeviceStruct, "LogFS Info Struct") 57 | { 58 | VSI_DEF_STRUCT_FIELD(VSI_DEC_U32, fsCheck, "fsCheck"); 59 | }; 60 | 61 | VSI_DEF_LEAF(device, root, 62 | LogFS_VSIDeviceGet, VSI_LogDeviceStruct, 63 | LogFS_VSIDeviceSet, VSI_Empty_Output, "device"); 64 | 65 | #endif 66 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/logtypes.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | /* ********************************************************** 35 | * **********************************************************/ 36 | 37 | /* 38 | * logtypes.h -- 39 | * 40 | * Core data structures and macros for CloudFS network protocol and disk 41 | * format. 42 | * 43 | * we use stdint.h types such as uint64_t, because we intend to include 44 | * this file in user space tools etc. outside of VMware's normal build 45 | * infrastructure. 46 | * 47 | * XXX we currently ignore all endianness issues. 48 | * 49 | * XXX we currently rely on bitops.h and the libnettle SHA-1 functions, 50 | * both of which are GPL. We need to switch to our own bitops, and to 51 | * find an equally fast SHA-1 that we can legally ship. 52 | * 53 | */ 54 | 55 | #ifndef __LOGTYPES_H__ 56 | #define __LOGTYPES_H__ 57 | 58 | #include "shalib/sha.h" 59 | #include "bitops.h" 60 | 61 | typedef uint64_t log_segment_id_t; 62 | typedef uint64_t log_size_t; 63 | typedef int64_t log_ssize_t; 64 | typedef uint64_t log_offset_t; 65 | typedef uint64_t log_block_t; 66 | typedef uint8_t log_ref_t; 67 | 68 | #define MAXBLOCK (~0ULL) 69 | 70 | #ifdef VMKERNEL 71 | typedef FDS_Handle *log_device_handle_t; 72 | #else 73 | typedef int log_device_handle_t; 74 | #endif 75 | 76 | /* We do everything at disk sector precision because we can. Longer term the 77 | * advent of 4kB sector disks may force us to align our disk accesses at 4kB, 78 | * but we believe it is going to be feasible to emulate a 512B interface for 79 | * VMs and applications. */ 80 | 81 | #define BLKSIZE 512 82 | #define BLKSIZE_ALIGNUP(_a) ( (_a+BLKSIZE-1)/BLKSIZE * BLKSIZE ) 83 | 84 | /* Log segments are 16MB each. We are planning to change this to be 32MB soon. */ 85 | 86 | #define LOG_MAX_SEGMENT_BLOCKS (0x8000) 87 | #define LOG_MAX_SEGMENT_SIZE (LOG_MAX_SEGMENT_BLOCKS*BLKSIZE) 88 | 89 | typedef struct __log_id { 90 | union { 91 | struct { 92 | unsigned short blk_offset:16; 93 | unsigned long long segment:48; 94 | } v; 95 | uint64_t raw; 96 | }; 97 | } 98 | log_id_t; 99 | 100 | #define METADATA_BLOCK (0xff00000000000000ULL/BLKSIZE) 101 | 102 | #define LOGID_TO_UINT64(_a) (_a.raw) 103 | 104 | #define INVALID_SEGMENT 0xffffffffffffULL 105 | #define INVALID_BLK_OFFSET 0xffff 106 | #define mk_invalid_version(__name) log_id_t __name; __name.raw = 0xffffffffffffffffULL; 107 | #define equal_version(__a,__b) ( __a.raw==__b.raw ) 108 | #define is_invalid_version(__a) ( (__a).raw==0xffffffffffffffffULL ) 109 | 110 | typedef enum { log_eof = 0, 111 | log_pointer_type, 112 | log_entry_type, 113 | } log_tag_t; 114 | 115 | typedef enum { log_prev_ptr = 1, log_next_ptr } log_pointer_t; 116 | 117 | struct log_head { 118 | 119 | log_tag_t tag:32; 120 | 121 | uint8_t disk[20]; 122 | uint8_t parent[20]; 123 | uint8_t id[20]; 124 | uint8_t entropy[20]; 125 | 126 | union { 127 | 128 | struct { /* log_entry_type */ 129 | uint8_t checksum[20]; /* actual checksum for this log entry */ 130 | uint64_t lsn; 131 | 132 | /* Extent info */ 133 | log_block_t blkno; 134 | unsigned int num_blocks:16; 135 | 136 | /* For potential RAIN uses in the future */ 137 | uint16_t slice; 138 | uint16_t slices_total; 139 | uint16_t num_parity; 140 | uint16_t ununsed; 141 | /* We conclude with a bit-vector used for compressing away all-zero blocks. 142 | * This is both to save space and bandwidth, but also to simplify log- 143 | * compaction, where blocks that are later overwritten can be compressed 144 | * down to a single zero bit here. */ 145 | log_ref_t refs[0]; 146 | 147 | } __attribute__ ((__packed__)) update; 148 | 149 | struct { /* log_pointer_type */ 150 | log_pointer_t direction; 151 | log_id_t target; 152 | } __attribute__ ((__packed__)); 153 | }; 154 | 155 | } __attribute__ ((__packed__)); 156 | 157 | 158 | #define LOG_HEAD_SIZE BLKSIZE 159 | #define LOG_HEAD_MAX_BLOCKS (8*(LOG_HEAD_SIZE- (unsigned long)( ((struct log_head*)NULL)->update.refs) ) ) 160 | /* FixME: use offsetof above! */ 161 | 162 | static inline void init_forward_pointer(struct log_head *head, log_id_t target) 163 | { 164 | memset(head, 0, LOG_HEAD_SIZE); 165 | head->tag = log_pointer_type; 166 | head->direction = log_next_ptr; 167 | head->target = target; 168 | } 169 | 170 | static inline void init_backward_pointer(struct log_head *head, log_id_t target) 171 | { 172 | memset(head, 0, LOG_HEAD_SIZE); 173 | head->tag = log_pointer_type; 174 | head->direction = log_prev_ptr; 175 | head->target = target; 176 | } 177 | 178 | static inline size_t log_body_size(struct log_head *head) 179 | { 180 | size_t sum; 181 | log_block_t i; 182 | 183 | if (head->tag != log_entry_type) 184 | return 0; 185 | 186 | for (i = 0, sum = 0; i < head->update.num_blocks; i++) { 187 | if (BitTest(head->update.refs,i)) 188 | sum += BLKSIZE; 189 | } 190 | 191 | return sum; 192 | } 193 | 194 | static inline size_t log_entry_size(struct log_head *head) 195 | { 196 | return log_body_size(head) + LOG_HEAD_SIZE; 197 | } 198 | 199 | static inline int is_block_zero(char *blkdata) 200 | { 201 | unsigned int i; 202 | int *data = (int *)blkdata; 203 | 204 | for (i = 0; i < BLKSIZE / sizeof(int); i++) { 205 | if (data[i]) 206 | return 0; 207 | } 208 | return 1; 209 | } 210 | 211 | static inline void log_entry_checksum(unsigned char *sum, struct log_head *head, 212 | const void *body, size_t sz) 213 | { 214 | struct sha1_ctx ctx; 215 | 216 | /* We hash from copies on the stack to prevent the compiler messing with the 217 | * field sizes. */ 218 | 219 | uint64_t b = head->update.blkno; 220 | uint16_t n = head->update.num_blocks; 221 | uint64_t l = head->update.lsn; 222 | 223 | sha1_init(&ctx); 224 | sha1_update(&ctx, sizeof(l), (const uint8_t *)&l); 225 | sha1_update(&ctx, sizeof(b), (const uint8_t *)&b); 226 | sha1_update(&ctx, sizeof(n), (const uint8_t *)&n); 227 | sha1_update(&ctx, sz, (const uint8_t *)body); 228 | 229 | /* digest the refs last for practical reasons */ 230 | sha1_update(&ctx, LOG_HEAD_SIZE - sizeof(struct log_head), 231 | (const uint8_t *)head->update.refs); 232 | 233 | sha1_digest(&ctx, SHA1_DIGEST_SIZE, sum); 234 | } 235 | 236 | #endif /* __LOGTYPES_H__ */ 237 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/metaLog.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef __METALOG_H__ 35 | #define __METALOG_H__ 36 | 37 | #include "logfsLog.h" 38 | #include "segmentlist.h" 39 | #include "logfsIO.h" 40 | #include "obsoleted.h" 41 | 42 | #define MAX_OPEN_LOGS 128 43 | 44 | struct LogFS_FingerPrint; 45 | 46 | typedef struct LogFS_MetaLog { 47 | btree_t *superTree; 48 | Semaphore superTreeSemaphore; 49 | 50 | LogFS_SegmentList segment_list; 51 | LogFS_Device *device; 52 | 53 | LogFS_ObsoletedSegments obsoleted; 54 | LogFS_ObsoletedSegments dupes; 55 | 56 | LogFS_Log *openLogs[MAX_OPEN_LOGS]; 57 | 58 | SP_SpinLock append_lock; 59 | SP_SpinLock refcounts_lock; 60 | 61 | List_Links remoteWaiters; 62 | 63 | LogFS_Log *activeLog; 64 | log_size_t spaceLeft; 65 | 66 | int lurt; 67 | 68 | struct LogFS_DiskLayout *diskLayout; 69 | 70 | Bool compactionInProgress; 71 | 72 | /* Dedupe related */ 73 | struct LogFS_VebTree *vt; 74 | struct LogFS_HashDb *hd; 75 | struct LogFS_FingerPrint *fp; 76 | struct LogFS_FingerPrint *fingerPrints[0x100]; 77 | 78 | } LogFS_MetaLog; 79 | 80 | void LogFS_MetaLogInit(LogFS_MetaLog *ml, struct LogFS_Device *device); 81 | void LogFS_MetaLogCleanup(LogFS_MetaLog *ml); 82 | VMK_ReturnStatus LogFS_MetaLogReopen(LogFS_MetaLog *ml, log_id_t position); 83 | void LogFS_MetaLogFreeLog(LogFS_MetaLog *ml, LogFS_Log *log); 84 | LogFS_Log *LogFS_MetaLogGetLog(LogFS_MetaLog *ml, log_segment_id_t segment); 85 | LogFS_Log *LogFS_MetaLogPutLog(LogFS_MetaLog *ml, LogFS_Log *log); 86 | 87 | VMK_ReturnStatus LogFS_MetaLogAppend(LogFS_MetaLog *ml, Async_Token * token, 88 | SG_Array *sgArr, log_id_t *retVersion, int flags); 89 | 90 | void LogFS_MetaLogSignalNewData(LogFS_MetaLog *ml); 91 | 92 | /* logCompactor.c */ 93 | log_id_t LogFS_MetaLogLookupIndirection(LogFS_MetaLog *ml, log_id_t position); 94 | 95 | //void LogFS_MetaLogGC(LogFS_MetaLog* ml); 96 | 97 | #endif /* __METALOG_H__ */ 98 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/migBSDNet.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef _MIG_BSD_NET_H_ 35 | #define _MIG_BSD_NET_H_ 36 | 37 | #include "vm_basic_types.h" 38 | #define _INTPTR_T_DECLARED // prevents from typedef'ing intptr_t in freebsd 39 | #define _SIZE_T_DECLARED // prevents from typedef'ing size_t in freebsd 40 | 41 | #endif //_MIG_BSD_NET_H_ 42 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/module.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | /* ***************************************************************************** 35 | * ****************************************************************************/ 36 | 37 | /* 38 | * module.c -- 39 | * 40 | * Logfs based LSOM initialization and cleanup 41 | * 42 | */ 43 | #include "thinDom.h" 44 | #include "vm_types.h" 45 | #include "vm_libc.h" 46 | #include "vmkernel.h" 47 | #include "mod_loader_public.h" 48 | #include "world.h" 49 | #include "system.h" 50 | #include "common.h" 51 | 52 | /* 53 | *----------------------------------------------------------------------------- 54 | * 55 | * LogfsInit -- 56 | * Entry point for logfs initialization. 57 | * 58 | * Results: 59 | * 0 on success, -1 on error. 60 | * 61 | * Side effects: 62 | * None. 63 | * 64 | *----------------------------------------------------------------------------- 65 | */ 66 | 67 | static int 68 | InitLogFS() 69 | { 70 | VMK_ReturnStatus status; 71 | 72 | // FixME: VMK_CheckVMKernelID is obsolete. Use VMK_GetHostID instead 73 | if (!VMK_CheckVMKernelID()) { 74 | Warning("Invalid vmkernel ID %#x. Can't load LogFS driver", 75 | VMK_GetVMKernelID()); 76 | return VMK_FAILURE; 77 | } 78 | 79 | status = LogFS_CommonInit(); 80 | if (status != VMK_OK) { 81 | Warning("Logfs_CommonInit failed with %s", 82 | VMK_ReturnStatusToString(status)); 83 | return -1; 84 | } 85 | 86 | status = Dom_Init(World_GetLastModID()); 87 | if (status != VMK_OK) { 88 | Warning("Logfs init failed with %s", VMK_ReturnStatusToString(status)); 89 | LogFS_CommonCleanup(); 90 | return -1; 91 | } 92 | 93 | return 0; 94 | } 95 | 96 | 97 | /* 98 | *----------------------------------------------------------------------------- 99 | * 100 | * cleanup_module -- 101 | * Called at module unload time. Cleanup module local data structures. 102 | * 103 | * Results: 104 | * None. 105 | * 106 | * 107 | * Side effects: 108 | * None. 109 | * 110 | *----------------------------------------------------------------------------- 111 | */ 112 | 113 | static void 114 | CleanupLogFS() 115 | { 116 | DEBUG_ONLY(Log("cleanup!\n")); 117 | Dom_Cleanup(); 118 | LogFS_CommonCleanup(); 119 | } 120 | 121 | 122 | #ifdef STATIC_LINK_VMLOGFS 123 | 124 | /* Statically linked, expose unique init/cleanup functions */ 125 | int 126 | LogFS_init_module(void) 127 | { 128 | return InitLogFS(); 129 | } 130 | 131 | void 132 | LogFS_cleanup_module(void) 133 | { 134 | CleanupLogFS(); 135 | } 136 | 137 | #else 138 | 139 | /* Compiled as part of module expose standard init/cleanup functions */ 140 | int 141 | init_module(void) 142 | { 143 | return InitLogFS(); 144 | } 145 | void cleanup_module(void) 146 | { 147 | CleanupLogFS(); 148 | } 149 | VMK_VERSION_INFO("Version" VMK_STRINGIFY(CBRCFILTER_MAJOR_VERSION) "." 150 | VMK_STRINGIFY(CBRCFILTER_MINOR_VERSION) 151 | ", Built on: " __DATE__); 152 | VMK_LICENSE_INFO(VMK_MODULE_LICENSE_VMWARE); 153 | #endif 154 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/obsoleted.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #include "obsoleted.h" 35 | #include "metaLog.h" 36 | #include "binHeap.h" 37 | 38 | void LogFS_ObsoletedSegmentsInit(LogFS_ObsoletedSegments *os) 39 | { 40 | 41 | LogFS_BinHeapInit(&os->heap, MAX_NUM_SEGMENTS); 42 | os->numCandidateSegments = 0; 43 | 44 | SP_InitLock("obslock", &os->lock, SP_RANK_OBSOLETED); 45 | LogFS_ObsoletedSegmentsClearRemaps(os); 46 | } 47 | 48 | void LogFS_ObsoletedSegmentsCleanup(LogFS_ObsoletedSegments *os) 49 | { 50 | LogFS_BinHeapCleanup(&os->heap); 51 | SP_CleanupLock(&os->lock); 52 | } 53 | 54 | void LogFS_ObsoletedSegmentsClearRemaps(LogFS_ObsoletedSegments *os) 55 | { 56 | int i; 57 | 58 | for (i = 0; i < LOGFS_OBS_MAX_REMAPS; i++) { 59 | os->remaps[i].from = INVALID_SEGMENT; 60 | } 61 | os->numRemaps = 0; 62 | } 63 | 64 | void LogFS_ObsoletedSegmentsRemapSegment(LogFS_ObsoletedSegments *os, 65 | log_segment_id_t from, 66 | log_segment_id_t to) 67 | { 68 | int i; 69 | SP_Lock(&os->lock); 70 | 71 | for (i = 0; i < os->numRemaps; i++) { 72 | if (os->remaps[i].from == from) 73 | break; 74 | } 75 | 76 | ASSERT(i < LOGFS_OBS_MAX_REMAPS); 77 | 78 | os->numRemaps = i; 79 | os->remaps[i].from = from; 80 | os->remaps[i].to = to; 81 | 82 | SP_Unlock(&os->lock); 83 | } 84 | 85 | void LogFS_ObsoletedSegmentsAdd(LogFS_ObsoletedSegments *os, 86 | log_segment_id_t segment, int howmany) 87 | { 88 | int i; 89 | int value; 90 | 91 | ASSERT(segment < MAX_NUM_SEGMENTS); 92 | 93 | /* XXX 94 | * integrate segment_list and obsoleted under a single lock. Don't add 95 | * any more to segments not in use, to prevent them cropping up at the 96 | * top of the heap -- a'la: 97 | if(!LogFS_SegmentListSegmentInUse(&ml->segment_list,segment)) return; 98 | 99 | in fact we could integrate the free/used bit into the heap ordering function, 100 | so that free segments were _|_ and always resides last in the heap. Then you 101 | could alloc a segment in log(N) time (set bit in last elem and percolate up) 102 | To ensure continuity of allocs, we could order free segments by segment number 103 | 104 | however, we still have to account for that slack, even though the segment is 105 | in the process of getting compacted. The new destination for the data actually 106 | needs to have its slack count set to the sum of the slack appearing in the old 107 | segment while they were getting cleaned. This also means that the old segments 108 | will drop back down as they are reset to zero again at the end of GC. 109 | */ 110 | 111 | SP_Lock(&os->lock); 112 | 113 | for (i = 0; i < os->numRemaps; i++) { 114 | if (os->remaps[i].from == segment) 115 | segment = os->remaps[i].to; 116 | } 117 | 118 | value = LogFS_BinHeapAdjustUp(&os->heap, segment, howmany); 119 | 120 | int limit = LOG_MAX_SEGMENT_BLOCKS / 5; 121 | /* Did we cross the threshold and become GC fodder? */ 122 | if ((value - howmany) < limit && value >= limit) { 123 | ++(os->numCandidateSegments); 124 | } 125 | 126 | SP_Unlock(&os->lock); 127 | } 128 | 129 | int LogFS_ObsoletedSegmentsGetCandidates(LogFS_ObsoletedSegments *os, 130 | LogFS_MetaLog *ml, 131 | void **candidates, int max_candidates) 132 | { 133 | /* Hack around a circular dep in include files by using void* for the metalog :-( XXX not needed */ 134 | 135 | if (os->numCandidateSegments < 6) 136 | return 0; 137 | 138 | int i; 139 | int howmany = 0; 140 | 141 | log_segment_id_t segments[max_candidates + 1]; 142 | int values[max_candidates + 1]; 143 | 144 | int n; 145 | 146 | SP_Lock(&os->lock); 147 | n = MIN(max_candidates, os->numCandidateSegments); 148 | 149 | for (i = 0; i < n; i++) { 150 | segments[i] = LogFS_BinHeapPopMax(&os->heap, &values[i]); 151 | } 152 | SP_Unlock(&os->lock); 153 | 154 | int j; 155 | for (i = j = 0; i < n; i++) { 156 | log_segment_id_t segment = segments[i]; 157 | int value; 158 | 159 | LogFS_Log *log = LogFS_MetaLogGetLog(ml, segment); 160 | 161 | /* We do not want to attempt cleaning active segments, or segments which 162 | * have already been freed */ 163 | 164 | value = (LogFS_SegmentListSegmentInUse(&ml->segment_list, segment)) ? 165 | values[i] : 0; 166 | 167 | if (value == 0 && values[i] != 0) 168 | zprintf("attempted to GC unalloced log segment %ld\n", segment); 169 | 170 | if (value > 0 && !LogFS_LogIsAppendLog(log)) { 171 | /* we reuse the values array in case we need to rollback below, 172 | * which works because j<=i */ 173 | 174 | candidates[j] = log; 175 | values[j] = value; 176 | howmany += value; 177 | 178 | ++j; 179 | } else { 180 | SP_Lock(&os->lock); 181 | LogFS_BinHeapAdjustUp(&os->heap, segment, value); 182 | SP_Unlock(&os->lock); 183 | 184 | LogFS_MetaLogPutLog(ml, log); 185 | } 186 | } 187 | 188 | /* If we ended up not having enough segments for GC to be meaningful (free 189 | * up at least one), we have to rollback our changes to the binary heap and 190 | * drop references to the rest of the candidate log segments */ 191 | 192 | if (j < 5) { 193 | for (i = 0; i < j; i++) { 194 | LogFS_Log *log = candidates[i]; 195 | 196 | SP_Lock(&os->lock); 197 | LogFS_BinHeapAdjustUp(&os->heap, LogFS_LogGetSegment(log), values[i]); 198 | SP_Unlock(&os->lock); 199 | 200 | LogFS_MetaLogPutLog(ml, log); 201 | } 202 | n = 0; 203 | } else { 204 | n = j; 205 | SP_Lock(&os->lock); 206 | os->numCandidateSegments -= n; 207 | SP_Unlock(&os->lock); 208 | printf("returning %d segments with %d blocks free\n", n, howmany); 209 | } 210 | return n; 211 | } 212 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/obsoleted.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef __OBSOLETED_H__ 35 | #define __OBSOLETED_H__ 36 | 37 | #include "logfsConstants.h" 38 | #include "logtypes.h" 39 | #include "binHeap.h" 40 | 41 | #define LOGFS_OBS_MAX_REMAPS 64 42 | 43 | typedef struct { 44 | SP_SpinLock lock; 45 | LogFS_BinHeap heap; 46 | int numCandidateSegments; 47 | 48 | struct { 49 | log_segment_id_t from, to; 50 | } remaps[LOGFS_OBS_MAX_REMAPS]; 51 | int numRemaps; 52 | 53 | } LogFS_ObsoletedSegments; 54 | 55 | struct LogFS_MetaLog; 56 | 57 | void LogFS_ObsoletedSegmentsInit(LogFS_ObsoletedSegments *os); 58 | void LogFS_ObsoletedSegmentsCleanup(LogFS_ObsoletedSegments *os); 59 | void LogFS_ObsoletedSegmentsAdd(LogFS_ObsoletedSegments *os, 60 | log_segment_id_t segment, int howmany); 61 | int LogFS_ObsoletedSegmentsGetCandidates(LogFS_ObsoletedSegments *os, 62 | struct LogFS_MetaLog *ml, 63 | void **candidates, int max_candidates); 64 | void LogFS_ObsoletedSegmentsRemapSegment(LogFS_ObsoletedSegments *os, 65 | log_segment_id_t from, 66 | log_segment_id_t to); 67 | void LogFS_ObsoletedSegmentsClearRemaps(LogFS_ObsoletedSegments *os); 68 | #endif 69 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/pagedTree.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef _PAGEDTREE_H_ 35 | #define _PAGEDTREE_H_ 36 | 37 | 38 | #define LOGLINES 11 /* 2**1 * 32kB == 64MB of cache */ 39 | #define LINES (1< NodeInfo* */ 66 | 67 | typedef struct LogFS_PagedTreeCache { 68 | SP_SpinLock lock; 69 | 70 | /* the bits array is a binary tree with LINES-1 inner nodes */ 71 | char bits[INNER_NODES]; 72 | NodeInfo *lines[LINES]; 73 | int nodeMap[LINES]; /* dict mapping NodeInfo* to cache lines */ 74 | 75 | } LogFS_PagedTreeCache ; 76 | 77 | VMK_ReturnStatus LogFS_PagedTreeDiskReopen(struct LogFS_MetaLog *ml, disk_block_t superTreeRoot); 78 | 79 | void LogFS_PagedTreeCleanupGlobalState(struct LogFS_MetaLog *ml); 80 | 81 | struct btree; 82 | struct LogFS_MetaLog; 83 | 84 | struct btree *LogFS_PagedTreeReOpen(struct LogFS_MetaLog *ml, disk_block_t root); 85 | struct btree *LogFS_PagedTreeCreate(struct LogFS_MetaLog *ml); 86 | 87 | void LogFS_PagedTreeCleanup(btree_t* t); 88 | 89 | VMK_ReturnStatus LogFS_PagedTreeSync(btree_t *t, struct LogFS_MetaLog *ml, List_Links *movedNodes); 90 | 91 | VMK_ReturnStatus LogFS_PagedTreeRescan(struct LogFS_MetaLog *); 92 | 93 | static inline void LogFS_PagedTreeChecksum(node_t *n) 94 | { 95 | const int hdr = 20 + sizeof(void *); 96 | 97 | const uint8_t *nn = (const uint8_t *)n; 98 | struct sha1_ctx ctx; 99 | sha1_init(&ctx); 100 | sha1_update(&ctx, TREE_BLOCK_SIZE - hdr, nn + hdr); 101 | sha1_digest(&ctx, SHA1_DIGEST_SIZE, n->chk); 102 | } 103 | 104 | static inline int LogFS_PagedTreeVerifyChecksum(node_t *n) 105 | { 106 | const int hdr = 20 + sizeof(void *); 107 | 108 | unsigned char sum[SHA1_DIGEST_SIZE]; 109 | const uint8_t *nn = (const uint8_t *)n; 110 | struct sha1_ctx ctx; 111 | sha1_init(&ctx); 112 | sha1_update(&ctx, TREE_BLOCK_SIZE - hdr, nn + hdr); 113 | sha1_digest(&ctx, SHA1_DIGEST_SIZE, sum); 114 | return (memcmp(sum, n->chk, SHA1_DIGEST_SIZE) == 0); 115 | } 116 | 117 | #endif //_PAGEDTREE_H_ 118 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/rangemap.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef __RANGEMAP_H__ 35 | #define __RANGEMAP_H__ 36 | 37 | #include "btree.h" 38 | 39 | /* Internal representation of a range within the B-tree. */ 40 | 41 | struct range { 42 | /* key */ 43 | uint64_t to; 44 | 45 | /* value */ 46 | uint16_t length; 47 | uint64_t version; 48 | 49 | } __attribute__ ((__packed__)) ; 50 | 51 | /* Struct used when returning lookup results. */ 52 | 53 | typedef struct { 54 | /* key */ 55 | uint64_t from; 56 | uint64_t version; 57 | 58 | } range_t ; 59 | 60 | #define RANGEMAP_KEY_SIZE (sizeof(uint64_t)) 61 | #define RANGEMAP_VALUE_SIZE (sizeof(struct range)-sizeof(uint64_t)) 62 | 63 | void rangemap_meminit(btree_t*, void *); 64 | int rangemap_insert(btree_t *tree, uint64_t from, 65 | uint64_t to, uint64_t version); 66 | void rangemap_merge(btree_t*, btree_t*); 67 | void rangemap_show(btree_t*); 68 | void rangemap_clear(btree_t*); 69 | uint64_t rangemap_get(btree_t*, uint64_t, uint64_t *); 70 | tree_result_t __rangemap_get(btree_t *tree, uint64_t block, 71 | range_t *result, 72 | uint64_t * endsat, void *context); 73 | void rangemap_replace(btree_t*, uint64_t, uint64_t, uint64_t, 74 | uint64_t); 75 | int rangemap_check(btree_t*); 76 | 77 | #endif 78 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/remoteLog.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef __REMOTELOG_H__ 35 | #define __REMOTELOG_H__ 36 | 37 | #include "logfsNet.h" 38 | #include "logfsHash.h" 39 | 40 | /* RemoteLog state is protected by the VDisk lock */ 41 | 42 | struct LogFS_VDisk; 43 | 44 | typedef struct LogFS_RemoteLog { 45 | Atomic_uint32 refCount; 46 | Bool isSocketOpen; 47 | int shouldClose; 48 | LogFS_MetaLog *ml; 49 | vmk_Worldlet worldlet; 50 | uint64 lsn; 51 | LogFS_Hash hostId; 52 | World_ID serverWorld; 53 | List_Links outstandingWrites; 54 | uint32 numBuffered; 55 | struct LogFS_VDisk *vd; 56 | 57 | List_Links nextLog; 58 | List_Links next; 59 | SP_SpinLock lock; 60 | 61 | /* Position in the log, used when streaming asyncly */ 62 | log_id_t position; 63 | 64 | } LogFS_RemoteLog; 65 | 66 | void LogFS_RemoteLogPreInit(void); 67 | void LogFS_RemoteLogInit(LogFS_RemoteLog *rl, 68 | LogFS_MetaLog *ml, 69 | vmk_Worldlet worldlet, 70 | struct LogFS_VDisk *vd, 71 | uint64 lsn); 72 | void LogFS_RemoteLogRef(LogFS_RemoteLog *rl); 73 | void LogFS_RemoteLogRelease(LogFS_RemoteLog *rl); 74 | void LogFS_RemoteLogClose(LogFS_RemoteLog *rl); 75 | void LogFS_RemoteLogExit(LogFS_RemoteLog *rl); 76 | 77 | VMK_ReturnStatus LogFS_RemoteLogAppend(LogFS_RemoteLog *rl, SG_Array *sgArr, 78 | Async_Token * token); 79 | 80 | VMK_ReturnStatus LogFS_RemoteLogPopUpdate(LogFS_RemoteLog *rl, SG_Array ** 81 | sgArr, Async_Token **); 82 | 83 | void LogFS_RemoteLogSetCurrent(LogFS_RemoteLog *rl, Hash id); 84 | 85 | static inline Bool LogFS_RemoteLogIsOpen(LogFS_RemoteLog *rl) 86 | { 87 | return (rl->isSocketOpen && !rl->shouldClose); 88 | } 89 | 90 | VMK_ReturnStatus LogFS_RemoteLogSpoolFromRevision(LogFS_RemoteLog *rl); 91 | 92 | static inline Bool LogFS_RemoteLogBufferFull(LogFS_RemoteLog *rl) 93 | { 94 | return (rl->numBuffered > 0x400000); 95 | } 96 | 97 | #endif /* __REMOTELOG_H__ */ 98 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/segmentlist.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | 35 | #ifndef __SEGMENTLIST_H__ 36 | #define __SEGMENTLIST_H__ 37 | 38 | #include "logfsConstants.h" 39 | #include "logtypes.h" 40 | #include "bitOps.h" 41 | 42 | /** XXX locking makes no sense **/ 43 | 44 | typedef struct { 45 | SP_SpinLock lock; 46 | uint8 bitmap[MAX_NUM_SEGMENTS / 8 + 1]; 47 | } LogFS_SegmentList; 48 | 49 | static inline void LogFS_SegmentListInit(LogFS_SegmentList *sl) 50 | { 51 | memset(sl->bitmap, 0, sizeof(sl->bitmap)); 52 | SP_InitLock("seglistlock", &sl->lock, SP_RANK_SEGMENTLIST); 53 | } 54 | 55 | static inline void LogFS_SegmentListFreeSegment(LogFS_SegmentList *sl, 56 | log_segment_id_t segment) 57 | { 58 | SP_Lock(&sl->lock); 59 | BitClear(sl->bitmap,segment); 60 | SP_Unlock(&sl->lock); 61 | } 62 | 63 | static inline void LogFS_SegmentListStealSegment(LogFS_SegmentList *sl, 64 | log_segment_id_t segment) 65 | { 66 | SP_Lock(&sl->lock); 67 | BitSet(sl->bitmap,segment); 68 | SP_Unlock(&sl->lock); 69 | } 70 | 71 | static inline int LogFS_SegmentListSegmentInUse(LogFS_SegmentList *sl, 72 | log_segment_id_t segment) 73 | { 74 | SP_Lock(&sl->lock); 75 | int r = BitTest(sl->bitmap,segment); 76 | SP_Unlock(&sl->lock); 77 | return r; 78 | } 79 | 80 | static inline log_segment_id_t LogFS_SegmentListAllocSegment(LogFS_SegmentList 81 | *sl) 82 | { 83 | SP_Lock(&sl->lock); 84 | 85 | log_segment_id_t r = INVALID_SEGMENT; 86 | 87 | int i; 88 | for (i = 0; i < MAX_NUM_SEGMENTS; i++) { 89 | if (!BitTest(sl->bitmap,i)) { 90 | BitSet(sl->bitmap,i); 91 | r = i; 92 | break; 93 | } 94 | } 95 | 96 | //log_segment_id_t r = find_first_zero_bit(sl->bitmap,LogFS_SegmentList_MAX_SEGMENTS/sizeof(unsigned long)); 97 | SP_Unlock(&sl->lock); 98 | return r; 99 | } 100 | 101 | #endif /* __SEGMENTLIST_H__ */ 102 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/shalib/Jamfile: -------------------------------------------------------------------------------- 1 | if ( ! $(shalib) ) { shalib = 1 ; 2 | 3 | LOCATION = bora modules vmkernel cloudfs ; 4 | 5 | SubDir TOP $(LOCATION) shalib ; 6 | SubDirHdrs $(TOP) $(LOCATION) ; 7 | 8 | SHA1FILES = 9 | sha1.c 10 | sha1-compress.c 11 | sha1-meta.c 12 | ; 13 | 14 | VMKLibrary libvmksha : $(SHA1FILES) ; 15 | UWLibrary libsha : $(SHA1FILES) ; 16 | 17 | 18 | } 19 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/shalib/macros.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware-archive/CloudFS/c4364e839abf99c7f17cd2ecc62e38c23d9e01e6/modules/vmkernel/cloudfs/shalib/macros.h -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/shalib/nettle-meta.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware-archive/CloudFS/c4364e839abf99c7f17cd2ecc62e38c23d9e01e6/modules/vmkernel/cloudfs/shalib/nettle-meta.h -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/shalib/nettle-types.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware-archive/CloudFS/c4364e839abf99c7f17cd2ecc62e38c23d9e01e6/modules/vmkernel/cloudfs/shalib/nettle-types.h -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/shalib/sha.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware-archive/CloudFS/c4364e839abf99c7f17cd2ecc62e38c23d9e01e6/modules/vmkernel/cloudfs/shalib/sha.h -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/shalib/sha1-compress.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware-archive/CloudFS/c4364e839abf99c7f17cd2ecc62e38c23d9e01e6/modules/vmkernel/cloudfs/shalib/sha1-compress.c -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/shalib/sha1-meta.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware-archive/CloudFS/c4364e839abf99c7f17cd2ecc62e38c23d9e01e6/modules/vmkernel/cloudfs/shalib/sha1-meta.c -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/shalib/sha1.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware-archive/CloudFS/c4364e839abf99c7f17cd2ecc62e38c23d9e01e6/modules/vmkernel/cloudfs/shalib/sha1.c -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/showlog.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #include 35 | #include 36 | #include 37 | #include 38 | 39 | #include "logtypes.h" 40 | #include "logfsHash.h" 41 | 42 | int main(int argc, char **argv) 43 | { 44 | 45 | char *entry = malloc(LOG_HEAD_SIZE + LOG_HEAD_MAX_BLOCKS * BLKSIZE); 46 | struct log_head *head = (struct log_head *)entry; 47 | char *body = entry + BLKSIZE; 48 | 49 | int f = open(argv[1], O_RDONLY); 50 | int g = -1; 51 | 52 | log_offset_t e; 53 | LogFS_Hash last; 54 | int line; 55 | 56 | if (f < 0) { 57 | printf("could not open %s\n", argv[1]); 58 | exit(1); 59 | } 60 | 61 | if (argc > 3) { 62 | open(argv[2], O_CREAT | O_RDWR); 63 | if (g < 0) { 64 | printf("could not open %s\n", argv[1]); 65 | exit(1); 66 | } 67 | } 68 | 69 | for (e = 0, line = 0;; line++) { 70 | char s_id[SHA1_HEXED_SIZE]; 71 | char checksum[SHA1_DIGEST_SIZE]; 72 | LogFS_Hash id, parent; 73 | 74 | int r = pread(f, head, LOG_HEAD_SIZE, e); 75 | if (r <= 0) 76 | break; 77 | if (log_body_size(head) > 0) { 78 | r = pread(f, body, log_body_size(head), e + LOG_HEAD_SIZE); 79 | if (r <= 0) 80 | break; 81 | } 82 | 83 | LogFS_HashSetRaw(&id, head->id); 84 | LogFS_HashSetRaw(&parent, head->parent); 85 | LogFS_HashPrint(s_id, &id); 86 | printf("%05d lsn: %llu %d: %s parent %s\n", line, head->update.lsn, head->tag, s_id, 87 | LogFS_HashShow2(parent)); 88 | printf("%lld+%d : %s\n", head->update.blkno, head->update.num_blocks, 89 | s_id); 90 | 91 | log_entry_checksum(checksum, head, ((char *)head) + BLKSIZE, 92 | log_body_size(head)); 93 | 94 | if (head->tag == log_entry_type) { 95 | if (memcmp(checksum, head->update.checksum, SHA1_DIGEST_SIZE) != 0) { 96 | Hash a, b; 97 | LogFS_HashSetRaw(&a, checksum); 98 | LogFS_HashSetRaw(&b, head->update.checksum); 99 | printf("bad checksum %s vs %s\n", 100 | LogFS_HashShow(&a), LogFS_HashShow(&b)); 101 | //exit(1); 102 | } 103 | } 104 | 105 | 106 | if (head->tag == log_entry_type) { 107 | if (head->update.lsn > 1 && !LogFS_HashEquals(LogFS_HashApply(parent), last)) { 108 | printf("Chain invalid\n"); 109 | //break; 110 | } 111 | } 112 | if (g >= 0) 113 | pwrite(g, head, log_entry_size(head), e); 114 | 115 | last = id; 116 | e += log_entry_size(head); 117 | } 118 | 119 | return 0; 120 | } 121 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/system.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef __SYSTEM_H__ 35 | #define __SYSTEM_H__ 36 | 37 | #ifndef VMKERNEL 38 | 39 | #ifndef _XOPEN_SOURCE 40 | #define _XOPEN_SOURCE 500 41 | #endif 42 | 43 | #ifndef MIN 44 | #define MIN(__a,__b) ( (__a)<(__b) ? (__a) : (__b )) 45 | #endif 46 | 47 | #ifndef MAX 48 | #define MAX(__a,__b) ( (__a)>(__b) ? (__a) : (__b )) 49 | #endif 50 | 51 | #include 52 | 53 | #include 54 | #include 55 | #include 56 | #include 57 | #include 58 | #include 59 | #include 60 | #include 61 | #include 62 | #include 63 | #include 64 | #include 65 | 66 | #define O_DIRECT 040000 67 | 68 | #ifndef ASSERT 69 | #define ASSERT(_a) assert(_a) 70 | #endif 71 | 72 | #ifndef USERLEVEL 73 | #define Panic(_a) abort() 74 | //#define exit(_a) ASSERT(0) 75 | //#define abort() ASSERT(0) 76 | //#define NOT_REACHED() abort() 77 | 78 | typedef int VMK_ReturnStatus; 79 | #define VMK_OK 0 80 | #define VMK_BAD_PARAM (-EINVAL) 81 | #define VMK_WRITE_ERROR (-EIO) 82 | #define VMK_WOULD_BLOCK (-EWOULDBLOCK) 83 | 84 | #define FALSE (0) 85 | #define TRUE (1) 86 | 87 | #define FMT64 "L" 88 | #endif 89 | 90 | #define zprintf(fmt,args...) printf(fmt , ## args) 91 | 92 | #else 93 | 94 | #include "vm_types.h" 95 | #include "vm_libc.h" 96 | #include "vmkernel.h" 97 | #include "splock.h" 98 | #include "world.h" 99 | #include "fss_int.h" 100 | #include "logfs_int.h" 101 | #include "fs_common.h" 102 | #include "fsSwitch.h" 103 | #include "scattergather.h" 104 | #include "scsi_ext.h" 105 | #include "scsi_vmware.h" 106 | #include "timer.h" 107 | #include "helper_ext.h" 108 | #include "semaphore_ext.h" 109 | #include "util.h" 110 | #include "config.h" 111 | #include "libc.h" 112 | #include "volumeCache.h" 113 | #include "objectCache.h" 114 | //#include "reservation.h" 115 | #include "vcfs.h" 116 | #include "stopwatch.h" 117 | #include "kseg_dist.h" 118 | #include "srm_ext.h" 119 | #include "fsvcb_public.h" 120 | #include "volumeCache.h" 121 | #include "world_user.h" 122 | 123 | /* fake for VMKERNEL */ 124 | 125 | #ifndef _INT8_T_DECLARED 126 | 127 | typedef uint64 uint64_t; 128 | typedef uint32 uint32_t; 129 | typedef uint16 uint16_t; 130 | typedef uint8 uint8_t; 131 | #define _UINT8_T_DECLARED 132 | #define _UINT16_T_DECLARED 133 | #define _UINT32_T_DECLARED 134 | #define _UINT64_T_DECLARED 135 | 136 | typedef int64 int64_t; 137 | typedef int32 int32_t; 138 | typedef int16 int16_t; 139 | typedef int8 int8_t; 140 | #define _INT8_T_DECLARED 141 | #define _INT16_T_DECLARED 142 | #define _INT32_T_DECLARED 143 | #define _INT64_T_DECLARED 144 | 145 | #endif 146 | 147 | #define LOGLEVEL_MODULE FS3 148 | #define LOGLEVEL_MODULE_LEN 3 149 | #include "log.h" 150 | 151 | #define LOGFS_FSTYPENUM 0x10f2 152 | 153 | #define printf(fmt,args...) //Log(fmt , ## args) 154 | #define zprintf(fmt,args...) Log(fmt , ## args) 155 | 156 | void qsort(void *base, size_t nmemb, size_t size, 157 | int (*compar) (const void *, const void *)); 158 | 159 | void *LogFS_Alloc(size_t, const char *, int); 160 | void LogFS_Free(void *); 161 | #define malloc(_a) LogFS_Alloc(_a,__FUNCTION__,__LINE__) 162 | #define free(_a) LogFS_Free(_a) 163 | 164 | #define aligned_malloc(_a) LogFS_Alloc(_a,__FUNCTION__,__LINE__) 165 | #define aligned_free(_a) LogFS_Free(_a) 166 | 167 | #ifndef DVMX86_DEBUG 168 | #undef ASSERT 169 | #define ASSERT(_a) {if(!(_a)) Panic("Botched assert file %s line %u",__FILE__,__LINE__);} 170 | #endif 171 | 172 | #define assert(_a) ASSERT(_a) 173 | 174 | #endif 175 | 176 | #endif /* __SYSTEM_H__ */ 177 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/vDisk.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef __VDISK_H__ 35 | #define __VDISK_H__ 36 | 37 | #include "metaLog.h" 38 | #include "bTreeRange.h" 39 | #include "logfsHash.h" 40 | // #include "remoteLog.h" 41 | 42 | typedef struct LogFS_VDisk { 43 | 44 | Hash disk; 45 | Hash secret_parent; /* when the vd is writable, we know this value */ 46 | Hash parent; /* if not the primary copy of this vd, only the public parent is known */ 47 | uint64 generation; 48 | Hash secretView; /* random seed used when generating secret ids */ 49 | 50 | Hash entropy; /* Entropy resulting from last applied update, used for regenerating 51 | secret on a forced view change */ 52 | 53 | uint64 lsn; 54 | 55 | LogFS_MetaLog *log; 56 | List_Links remoteLogs; 57 | 58 | Bool stopRequested; 59 | Bool haveReservation; 60 | 61 | Bool isImmutable; 62 | 63 | Hash parentBaseId; 64 | struct LogFS_VDisk *parentDisk; 65 | 66 | LogFS_BTreeRangeMap *bt; 67 | 68 | SP_SpinLock lock; 69 | List_Links closeWaiters; 70 | List_Links tokenWaiters; 71 | List_Links openWaiters; 72 | 73 | Atomic_uint32 refCount; 74 | 75 | 76 | } LogFS_VDisk; 77 | 78 | void LogFS_VDiskInit(LogFS_VDisk *vd, LogFS_VDisk *parentDisk, Hash diskId, 79 | LogFS_MetaLog *log); 80 | void LogFS_VDiskCleanup(LogFS_VDisk *vd); 81 | 82 | static inline void LogFS_VDiskRef(LogFS_VDisk *vd) 83 | { 84 | Atomic_Inc(&vd->refCount); 85 | } 86 | 87 | VMK_ReturnStatus LogFS_VDiskReserve(LogFS_VDisk *vd); 88 | 89 | Bool LogFS_VDiskIsWritable(LogFS_VDisk *vd); 90 | 91 | VMK_ReturnStatus LogFS_VDiskBranch(LogFS_VDisk *vd, LogFS_Hash childId, 92 | LogFS_VDisk **result); 93 | 94 | VMK_ReturnStatus LogFS_VDiskSnapshot(LogFS_VDisk *vd, LogFS_VDisk **snapshot); 95 | 96 | void LogFS_VDiskRelease(LogFS_VDisk *vd); 97 | 98 | Hash LogFS_VDiskGetBaseId(LogFS_VDisk *vd); 99 | Hash LogFS_VDiskGetCurrentId(LogFS_VDisk *vd); 100 | Hash LogFS_VDiskGetStable(LogFS_VDisk *vd); 101 | Hash LogFS_VDiskGetParentDiskId(void *vd); 102 | VMK_ReturnStatus LogFS_VDiskWrite(LogFS_VDisk *vd, Async_Token *, 103 | const char *buf, log_block_t blkno, 104 | size_t num_blocks, int flags); 105 | VMK_ReturnStatus LogFS_VDiskSetSecret(LogFS_VDisk *vd, Hash secret, 106 | Hash secretView); 107 | VMK_ReturnStatus LogFS_VDiskGetSecret(LogFS_VDisk *vd, Hash * secret, 108 | Bool failIfBusy); 109 | VMK_ReturnStatus LogFS_VDiskAppend(LogFS_VDisk *vd, Async_Token *, 110 | struct LogFS_RefCountedBuffer *buf); 111 | VMK_ReturnStatus LogFS_VDiskRead(LogFS_VDisk *vd, Async_Token * token, 112 | char *buf, log_block_t blkno, 113 | size_t num_blocks, int flags); 114 | VMK_ReturnStatus LogFS_VDiskContinueRead(LogFS_VDisk *vd, Async_Token * token, 115 | const char *buf, log_block_t blkno, 116 | size_t num_blocks, int flags); 117 | VMK_ReturnStatus LogFS_VDiskContinueWrite(LogFS_VDisk *vd, Async_Token * token, 118 | const char *buf, log_block_t blkno, 119 | size_t num_blocks, int flags); 120 | LogFS_BTreeRangeMap *LogFS_VDiskGetVersionsMap(LogFS_VDisk *vd); 121 | 122 | static inline void 123 | LogFS_VDiskUpdateFromHead(LogFS_VDisk *vd, struct log_head *head) 124 | { 125 | if (head->tag == log_entry_type) { 126 | LogFS_HashSetRaw(&vd->entropy, head->entropy); 127 | LogFS_HashSetRaw(&vd->parent, head->id); 128 | } 129 | } 130 | 131 | static inline unsigned long long LogFS_VDiskGetCapacity(LogFS_VDisk *vd) 132 | { 133 | //static const unsigned long long FILE_SIZE=0x800000000ULL; 134 | //static const unsigned long long FILE_SIZE=0x400000000ULL; 135 | static const unsigned long long FILE_SIZE = 0x800000000ULL; 136 | return FILE_SIZE; 137 | } 138 | 139 | static inline int LogFS_VDiskIsOrphaned(LogFS_VDisk *vd) 140 | { 141 | return (vd->parentDisk == NULL && LogFS_HashIsValid(vd->parentBaseId)); 142 | } 143 | 144 | struct LogFS_RemoteLog; 145 | int LogFS_VDiskAddRemoteLogIfSameVersion(LogFS_VDisk *vd, struct LogFS_RemoteLog *rl); 146 | 147 | #endif 148 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/vDiskMap.c: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #include "logtypes.h" 35 | #include "logfsHash.h" 36 | #include "vDisk.h" 37 | #include "vDiskMap.h" 38 | 39 | LogFS_VDisk *vdisks[1024]; 40 | 41 | /* XXX we need locking for all of this, and need to adhere to code conventions */ 42 | 43 | Atomic_uint32 numDisks; 44 | 45 | void LogFS_DiskMapInit(void) 46 | { 47 | Atomic_Write(&numDisks, 0); 48 | } 49 | 50 | LogFS_VDisk *LogFS_DiskMapLookupDisk(Hash disk) 51 | { 52 | int i; 53 | for (i = 0; i < Atomic_Read(&numDisks); i++) { 54 | LogFS_VDisk *vd = vdisks[i]; 55 | if (LogFS_HashEquals(disk, LogFS_VDiskGetBaseId(vd))) { 56 | return vd; 57 | } 58 | } 59 | return NULL; 60 | } 61 | 62 | LogFS_VDisk *LogFS_DiskMapLookupDiskForVersion(Hash id) 63 | { 64 | int i; 65 | for (i = 0; i < Atomic_Read(&numDisks); i++) { 66 | LogFS_VDisk *vd = vdisks[i]; 67 | Hash cur = LogFS_VDiskGetCurrentId(vd); 68 | if (LogFS_HashEquals(id, cur)) { 69 | return vd; 70 | } 71 | } 72 | return NULL; 73 | } 74 | 75 | /* XXX we should allow multiple orphans with same parent */ 76 | 77 | LogFS_VDisk *LogFS_DiskMapLookupOrphan(Hash id) 78 | { 79 | int i; 80 | 81 | for (i = 0; i < Atomic_Read(&numDisks); i++) { 82 | LogFS_VDisk *vd = vdisks[i]; 83 | if (LogFS_VDiskIsOrphaned(vd) 84 | && LogFS_HashEquals(id, LogFS_VDiskGetParentDiskId(vd))) ; 85 | { 86 | return vd; 87 | } 88 | } 89 | return NULL; 90 | } 91 | 92 | void LogFS_DiskMapInsert(LogFS_VDisk *vd) 93 | { 94 | vdisks[Atomic_FetchAndInc(&numDisks)] = vd; 95 | } 96 | 97 | size_t LogFS_DiskMapListDisks(char *s) 98 | { 99 | int i; 100 | char *s0 = s; 101 | 102 | for (i = 0; i < Atomic_Read(&numDisks); i++) { 103 | LogFS_VDisk *vd = vdisks[i]; 104 | LogFS_Hash disk = LogFS_VDiskGetBaseId(vd); 105 | LogFS_Hash id = LogFS_VDiskGetCurrentId(vd); 106 | 107 | #if 0 108 | LogFS_Hash id; 109 | if (VDISK_INTERFACE(vd)->type == VDI_VDISK) 110 | id = ((LogFS_VDisk *)vd)->view; 111 | else 112 | id = VDISK_INTERFACE(vd)->GetCurrentId(vd); 113 | #endif 114 | 115 | LogFS_HashPrint(s, &disk); 116 | s += SHA1_HEXED_SIZE - 1; 117 | *s++ = ':'; 118 | 119 | LogFS_HashPrint(s, &id); 120 | s += SHA1_HEXED_SIZE - 1; 121 | 122 | *s++ = '\n'; 123 | } 124 | 125 | return s - s0; 126 | } 127 | 128 | #if 0 129 | #include "btree.h" 130 | #include "vdisk.h" 131 | 132 | /* B-tree listing all virtual disks */ 133 | 134 | btree_t disk_tree; 135 | 136 | int disk_cmp(LogFS_VDisk *a, LogFS_VDisk *b) 137 | { 138 | Hash *ha = (Hash *) a; 139 | Hash *hb = (Hash *) b; 140 | return memcmp(ha->raw, hb->raw, SHA1_DIGEST_SIZE); 141 | } 142 | 143 | static disk_block_t alloc_node_mem(btree_t *t) 144 | { 145 | return (disk_block_t) malloc(t->real_node_size); 146 | } 147 | 148 | static const node_t *get_node_mem(btree_t *t, disk_block_t block) 149 | { 150 | return (const node_t *)block; 151 | } 152 | 153 | static node_t *edit_node_mem(btree_t *t, disk_block_t block) 154 | { 155 | return (node_t *)block; 156 | } 157 | 158 | struct disk_tree_elem { 159 | char hash[20]; 160 | LogFS_VDiskInterface *vd; 161 | } __attribute__ ((__packed__)); 162 | 163 | LogFS_VDiskInterface *find_vdisk(Hash disk) 164 | { 165 | struct disk_tree_elem e; 166 | memcpy(e.hash, disk.raw, SHA1_DIGEST_SIZE); 167 | 168 | if (tree_find(&disk_tree, (elem_t *) & disk)) { 169 | printf("disk found %p\n", e.vd); 170 | return e.vd; 171 | 172 | } 173 | } 174 | 175 | void insert_vdisk(Hash disk, LogFS_VDiskInterface * vd) 176 | { 177 | struct disk_tree_elem e; 178 | memcpy(e.hash, disk.raw, SHA1_DIGEST_SIZE); 179 | e.vd = vd; 180 | 181 | tree_insert(&disk_tree, (elem_t *) & e); 182 | } 183 | 184 | void init_vdisk_tree(void) 185 | { 186 | btree_callbacks_t callbacks; 187 | callbacks.cmp = disk_cmp; 188 | callbacks.alloc_node = alloc_node_mem; 189 | callbacks.edit_node = edit_node_mem; 190 | callbacks.get_node = get_node_mem; 191 | 192 | LogFS_VDisk *base = malloc(0x4000); // XXX fixed sized memory of vdisk tree 193 | 194 | tree_create(&disk_tree, &callbacks, SHA1_DIGEST_SIZE, 195 | sizeof(LogFS_VDiskInterface *), 0x40, base); 196 | } 197 | 198 | int main() 199 | { 200 | init_vdisk_tree(); 201 | 202 | Hash disk; 203 | LogFS_HashRandomize(&disk); 204 | 205 | LogFS_VDiskInterface vd; 206 | LogFS_VDiskInterfaceInit(&vd, disk, NULL); 207 | 208 | } 209 | #endif 210 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/vDiskMap.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | 35 | struct LogFS_VDisk; 36 | extern struct LogFS_VDisk *vdisks[1024]; 37 | extern Atomic_uint32 numDisks; 38 | 39 | void LogFS_DiskMapInit(void); 40 | struct LogFS_VDisk *LogFS_DiskMapLookupDisk(Hash disk); 41 | struct LogFS_VDisk *LogFS_DiskMapLookupDiskForVersion(Hash id); 42 | struct LogFS_VDisk *LogFS_DiskMapLookupOrphan(Hash id); 43 | void LogFS_DiskMapInsert(struct LogFS_VDisk *vd); 44 | size_t LogFS_DiskMapListDisks(char *s); 45 | -------------------------------------------------------------------------------- /modules/vmkernel/cloudfs/vebTree.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2007-2011 VMware, Inc. All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted (subject to the limitations in the 6 | disclaimer below) provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of VMware nor the names of its 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE 21 | GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT 22 | HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED 23 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 24 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 31 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 32 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | #ifndef __VEBTREE_H__ 35 | #define __VEBTREE_H__ 36 | 37 | #include "system.h" 38 | #include "btree.h" 39 | 40 | #ifndef VMKERNEL 41 | 42 | typedef void LogFS_Device; 43 | 44 | #endif 45 | 46 | 47 | typedef struct LogFS_VebTree { 48 | struct LogFS_Device* device; 49 | size_t nodeSz; 50 | size_t totalSz; 51 | size_t treeSz; 52 | btree_t* t; 53 | void* mem; 54 | 55 | int activeBank; 56 | int bankDirty; 57 | btree_t tree; 58 | } LogFS_VebTree; 59 | 60 | 61 | 62 | struct LogFS_Device; 63 | struct LogFS_ObsoletedSegments; 64 | struct Graph; 65 | 66 | void LogFS_VebTreeInit( LogFS_VebTree* vt, struct LogFS_Device* device, size_t nodeSz, size_t treeSz); 67 | void LogFS_VebTreeCleanup(LogFS_VebTree *vt); 68 | void LogFS_VebTreeFlush( LogFS_VebTree* vt, struct LogFS_Device* device ); 69 | void LogFS_VebTreeClear( LogFS_VebTree* vt); 70 | void LogFS_VebTreeSetBank( LogFS_VebTree* vt, int bank); 71 | void LogFS_VebTreeInsert( LogFS_VebTree* vt, uint8_t bank, uint32_t key, uint32_t value); 72 | 73 | void LogFS_VebTreeMerge( 74 | LogFS_VebTree* base, 75 | LogFS_VebTree* overlay, 76 | struct Graph *g); 77 | 78 | void LogFS_VebTreePrint( LogFS_VebTree* vt); 79 | 80 | #endif /* __VEBTREE_H__ */ 81 | -------------------------------------------------------------------------------- /papers/osr.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware-archive/CloudFS/c4364e839abf99c7f17cd2ecc62e38c23d9e01e6/papers/osr.pdf --------------------------------------------------------------------------------