├── README.md ├── ping_reporter.py ├── ping_fuse.py ├── ping.py ├── ping_disk.py ├── ping_server.py └── ping_filesystem.py /README.md: -------------------------------------------------------------------------------- 1 | # PingFS: Living in the Rain 2 | PingFS is a set of python scripts which, in Linux, provide virtual disk storage on the network. Each file is broken up into 64-1024 byte blocks, sent over the wire in an ICMP echo request, and promptly erased from memory. Each time the server sends back your bundle of joy, PingFS recognizes its data and sends another gift [the data]. 3 | 4 | ## Usage 5 | 6 | - mkdir mount_dir/ 7 | - python ping_fuse.py mount_dir 8 | - ls mount_dir 9 | - echo cats > mount_dir/reddit 10 | - cat mount_dir/reddit 11 | 12 | ## Requirements 13 | 14 | - Linux 15 | - python 2.7+ 16 | - python-fuse 17 | -------------------------------------------------------------------------------- /ping_reporter.py: -------------------------------------------------------------------------------- 1 | import threading, logging, time, locale 2 | 3 | logging.TRACE = 5 4 | logging.NOTICE = 15 5 | logging.addLevelName(logging.TRACE, 'TRACE') 6 | logging.addLevelName(logging.NOTICE, 'NOTICE') 7 | 8 | def setup_log(name,level=logging.ERROR): 9 | log = logging.getLogger(name) 10 | setattr(log.__class__, 'notice', log_notice) 11 | setattr(log.__class__, 'trace', log_trace) 12 | log.setLevel(level) 13 | return log 14 | 15 | def start_log(logger,stream=logging.NOTICE,logs=logging.TRACE): 16 | logger.setLevel(min(stream,logs)) 17 | addStreamHandler(logger,stream) 18 | addFileHandler(logger,logs) 19 | 20 | def addFileHandler(log,level=None): 21 | formatter = logging.Formatter('[%(levelname)-6s] '+'[%s]'%log.name+' %(message)s') 22 | handler = logging.FileHandler('pingfs.log') 23 | if level: handler.setLevel(level) 24 | handler.setFormatter(formatter) 25 | log.addHandler(handler) 26 | 27 | def addStreamHandler(log,level=None): 28 | formatter = logging.Formatter('[%(levelname)s] %(message)s') 29 | handler = logging.StreamHandler() 30 | if level: handler.setLevel(level) 31 | handler.setFormatter(formatter) 32 | log.addHandler(handler) 33 | 34 | def log_generic(self, level, msg, *args, **kwargs): 35 | if self.manager.disable >= level: return 36 | if level >= self.getEffectiveLevel(): 37 | apply(self._log, (level,msg,args), kwargs) 38 | 39 | def log_notice(self, msg, *args, **kwargs): 40 | log_generic(self,logging.NOTICE,msg,*args,**kwargs) 41 | 42 | def log_trace(self, msg, *args, **kwargs): 43 | log_generic(self,logging.TRACE,msg,*args,**kwargs) 44 | 45 | def enableAllLogs(screen=logging.INFO,logs=logging.TRACE): 46 | import ping_filesystem, ping_fuse 47 | import ping, ping_disk, ping_server 48 | #start_log(ping.log, screen,logs) 49 | start_log(ping_server.log, screen,logs) 50 | start_log(ping_disk.log, screen,logs) 51 | start_log(ping_filesystem.log,screen,logs) 52 | start_log(ping_fuse.log, screen,logs) 53 | 54 | import ping 55 | def humanize_bytes(bytes, precision=2): 56 | # by Doug Latornell 57 | # http://code.activestate.com/recipes/577081-humanized-representation-of-a-number-of-bytes/ 58 | abbrevs = ( 59 | (1<<50L, 'PB'), 60 | (1<<40L, 'TB'), 61 | (1<<30L, 'GB'), 62 | (1<<20L, 'MB'), 63 | (1<<10L, 'kB'), 64 | (1, 'bytes') 65 | ) 66 | if bytes == 1: return '1 byte' 67 | for factor, suffix in abbrevs: 68 | if bytes >= factor: break 69 | return '%.*f %s' % (precision, float(bytes) / factor, suffix) 70 | 71 | class PingReporter(threading.Thread): 72 | def __init__(self, log, server, interval=90): 73 | locale.setlocale(locale.LC_ALL,'') 74 | threading.Thread.__init__(self) 75 | self.interval = interval 76 | self.server = server 77 | self.running = 1 78 | self.log = log 79 | 80 | def stop(self): 81 | log.info('reporter terminated') 82 | self.running = 0 83 | 84 | def run(self): 85 | start = time.time() 86 | self.log.info('reporter started at %s'%time.ctime()) 87 | while self.running: 88 | time.sleep(self.interval) 89 | bw = humanize_bytes(ping.ping_bandwidth) 90 | num = locale.format('%d', ping.ping_count, True) 91 | tstr = time.strftime('%H:%M:%S',time.gmtime(time.time()-start)) 92 | self.log.info('%s (%s pings) -> %s elapsed'%(bw,num,tstr)) 93 | -------------------------------------------------------------------------------- /ping_fuse.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import os, sys, stat, errno, posix, logging, time, fuse 4 | import ping, ping_reporter, ping_filesystem 5 | from time import time 6 | 7 | fuse.fuse_python_api = (0,2) 8 | 9 | log = ping_reporter.setup_log('PingFuse') 10 | 11 | class PingFuse(fuse.Fuse): 12 | def __init__(self, server): 13 | self.FS = ping_filesystem.PingFS(server) 14 | #ping.drop_privileges() 15 | fuse.Fuse.__init__(self) 16 | log.notice('ping::fuse: initialized (%d-byte blocks)'%self.FS.disk.block_size()) 17 | 18 | def fsinit(self): 19 | self.reporter = ping_reporter.PingReporter(log,'',90) 20 | self.reporter.start() 21 | 22 | def getattr(self, path): 23 | """ 24 | - st_mode (protection bits) 25 | - st_ino (inode number) 26 | - st_dev (device) 27 | - st_nlink (number of hard links) 28 | - st_uid (user ID of owner) 29 | - st_gid (group ID of owner) 30 | - st_size (size of file, in bytes) 31 | - st_atime (time of most recent access) 32 | - st_mtime (time of most recent content modification) 33 | - st_ctime (platform dependent; time of most recent metadata change on Unix, 34 | or the time of creation on Windows). 35 | """ 36 | 37 | log.info('getattr: %s' % path) 38 | 39 | pFile = self.FS.get(path) 40 | if not pFile: return -errno.ENOENT 41 | 42 | st = fuse.Stat() 43 | st.st_mode = pFile.type | pFile.mode 44 | st.st_ino = pFile.inode 45 | st.st_nlink = pFile.links() 46 | st.st_uid = 1000 #pFile.uid 47 | st.st_gid = 1000 #pFile.gid 48 | st.st_size = pFile.size() 49 | #st.st_atime = time() 50 | #st.st_mtime = time() 51 | #st.st_ctime = time() 52 | #st.st_dev = 2050L 53 | return st 54 | 55 | def readdir(self, path, offset): 56 | log.info('readdir: %s'%path) 57 | pDir = self.FS.get(path) 58 | if not pDir: return -errno.ENOENT 59 | if pDir.type != stat.S_IFDIR: 60 | return [fuse.Direntry(pDir.name)] 61 | 62 | files = [fuse.Direntry('.'),fuse.Direntry('..')] 63 | for e in pDir.entries: 64 | files.append(fuse.Direntry(e.name)) 65 | return files 66 | 67 | def mkdir(self, path, mode): 68 | log.info('mkdir: %s mode=%04o'%(path,mode)) 69 | if path == '/' or path == '': return -errno.EACCESS 70 | if self.FS.get(path): return -errno.EEXIST 71 | rPath,rName = path.rsplit('/',1) 72 | pDir = self.FS.get(rPath) 73 | if not pDir: return -errno.ENOENT 74 | 75 | nDir = ping_filesystem.PingDirectory(rName) 76 | self.FS.add(nDir) # acquire inode 77 | pDir.add_node(nDir) # add dirent 78 | self.FS.update(pDir) # save 79 | return 0 80 | 81 | def open(self, path, flags): 82 | log.info('open: %s flags=%x'%(path,flags)) 83 | pFile = self.FS.get(path) 84 | if not pFile: return -errno.ENOENT 85 | return 0 86 | 87 | def read(self, path, length, offset): 88 | log.info('read: %s region=%d,%d'%(path,offset,length)) 89 | pFile = self.FS.get(path) 90 | if not pFile: return -errno.ENOENT 91 | if offset > len(pFile.data): return -errno.EINVAL 92 | if pFile.type == stat.S_IFDIR: return -errno.EISDIR 93 | return pFile.data[offset:offset+length] 94 | 95 | def chmod(self, path, mode): 96 | log.info('chmod: %s mode=%04o'%(path,mode)) 97 | pFile = self.FS.get(path) 98 | if not pFile: return -errno.ENOENT 99 | pFile.mode = mode 100 | self.FS.update(pFile) 101 | return 0 102 | 103 | def chown(self, path, uid, gid): 104 | log.info('chown: %s uid=%d gid=%d)'%(path,uid,gid)) 105 | pFile = self.FS.get(path) 106 | if not pFile: return -errno.ENOENT 107 | pFile.uid = uid 108 | pFile.gid = gid 109 | self.FS.update(pFile) 110 | return 0 111 | 112 | def rmdir(self, path): 113 | log.info('rmdir: %s'%path) 114 | pFile = self.FS.get(path) 115 | if not pFile: return -errno.ENOENT 116 | if pFile.type != stat.S_IFDIR: return -errno.ENOTDIR 117 | if self.FS.unlink(path,pFile): return 0 118 | return -errno.EINVAL 119 | 120 | def unlink(self, path): 121 | log.info('unlink: %s'%path) 122 | pFile = self.FS.get(path) 123 | if not pFile: return -errno.ENOENT 124 | if pFile.type != stat.S_IFREG: return -errno.ENOTDIR 125 | if self.FS.unlink(path,pFile): return 0 126 | return -errno.EINVAL 127 | 128 | def write(self, path, buf, offset): 129 | log.info('write: %s region=%d,%d'%(path,offset,offset+len(buf))) 130 | pFile = self.FS.get(path) 131 | if not pFile: return -errno.ENOENT 132 | pDir = self.FS.get_parent(path,pFile) 133 | if not pDir: raise Exception('write failed to find parent after filding child!') 134 | if offset: 135 | fill = '\0' * min(0,offset - len(pFile.data)) # technically don't need to bound negatives 136 | offset = pFile.data[:offset] + fill 137 | else: offset = '' 138 | 139 | pFile.data = offset + buf 140 | if not self.FS.update(pFile,pDir): 141 | return -errno.EINVAL 142 | return len(buf) 143 | 144 | def truncate(self, path, size): 145 | log.info('truncate: %s size=%d'%(path, size)) 146 | pFile = self.FS.get(path) 147 | if not pFile: return -errno.ENOENT 148 | if size > len(pFile.data): return -errno.EINVAL 149 | if pFile.type != stat.S_IFREG: return -errno.EINVAL 150 | pFile.data = pFile.data[:size] 151 | self.FS.update(pFile) 152 | return 0 153 | 154 | def mknod(self, path, mode, dev): 155 | log.info('mknod: %s mode=%04o dev=%d)'%(path,mode,dev)) 156 | if not mode & stat.S_IFREG: return -errno.ENOSYS 157 | pFile = self.FS.get(path) 158 | if pFile: return -errno.EEXIST 159 | pFile = self.FS.create(path) 160 | if not pFile: return -errno.EINVAL 161 | pFile.mode = mode & 0777 162 | self.FS.update(pFile) 163 | return 0 164 | 165 | def rename(self, old_path, new_path): 166 | log.info('rename: %s -> %s'%(old_path,new_path)) 167 | 168 | (oDir,oFile) = self.FS.get_both(old_path) 169 | (nDir,nFile) = self.FS.get_both(new_path) 170 | new_name = new_path.rsplit('/',1)[1] 171 | 172 | if not oFile: return -errno.ENOENT 173 | if not oDir or not nDir: return -errno.ENOENT 174 | if nFile: return -errno.EEXIST 175 | 176 | oDir.del_node(oFile.name,oFile) 177 | oFile.name = new_name 178 | nDir.add_node(oFile) 179 | 180 | # better to be in both than neither 181 | self.FS.update(nDir) 182 | self.FS.update(oFile) 183 | self.FS.update(oDir) 184 | return 0 185 | 186 | def link(self, targetPath, linkPath): 187 | log.info('link: %s <- %s)'%(targetPath, linkPath)) 188 | return -errno.ENOSYS 189 | 190 | def readlink(self, path): 191 | log.info('readlink: %s'%path) 192 | return -errno.ENOSYS 193 | 194 | def symlink(self, targetPath, linkPath): 195 | log.info('symlink: %s <- %s'%(targetPath, linkPath)) 196 | return -errno.ENOSYS 197 | 198 | # def mythread ( self ): 199 | # log.info('mythread') 200 | # return -errno.ENOSYS 201 | 202 | def release(self, path, flags): 203 | log.info('release: %s flags=%x'%(path,flags)) 204 | return -errno.ENOSYS 205 | 206 | def statf(self): 207 | log.info('statfs') 208 | return -errno.ENOSYS 209 | 210 | def utime(self, path, times): 211 | log.info('utime: %s times=%s'%(path,times)) 212 | return -errno.ENOSYS 213 | 214 | def fsync(self, path, isFsyncFile): 215 | log.info('fsync: %s fsyncFile? %s'%(path,isFsyncFile)) 216 | return -errno.ENOSYS 217 | 218 | 219 | 220 | if __name__ == "__main__": 221 | import ping_disk 222 | #ping_reporter.enableAllLogs(logging.DEBUG,logging.TRACE) 223 | ping_reporter.start_log(log,logging.NOTICE) 224 | #ping_reporter.start_log(ping_filesystem.log,logging.DEBUG) 225 | #ping_reporter.start_log(ping_disk.log,logging.DEBUG) 226 | server = ping.select_server(log) 227 | if len(sys.argv) < 2: 228 | print 'usage: %s ' % sys.argv[0] 229 | sys.exit(1) 230 | sys.argv.append('-f') 231 | fs = PingFuse(server) 232 | #fs.parser.add_option(mountopt="root",metavar="PATH", default='/') 233 | #fs.parse(values=fs, errex=1) 234 | fs.parse(errex=1) 235 | 236 | fs.flags = 0 237 | #fs.multithreaded = 0 238 | ping_filesystem.init_fs(fs.FS) 239 | #ping_filesystem.test_fs(fs.FS) 240 | 241 | 242 | if 0: 243 | log.info('file system testing begun') 244 | fs.mknod('/test_file2',stat.S_IFREG | 0777,0) 245 | fs.mknod('/test_file1',stat.S_IFREG | 0777,0) 246 | wData = 'A'*fs.FS.disk.region_size()*fs.FS.disk.block_size() 247 | 248 | fs.write('/test_file1',wData,0) 249 | log.info('completed writing testfile1: %d bytes'%len(wData)) 250 | rData = fs.read('/test_file1',len(wData),0) 251 | if rData == wData: log.info( 'read verified successfully') 252 | elif len(wData) != len(rData): log.error('read failed (%d of %d bytes)'%(len(rData),len(wData))) 253 | else: log.error('read failed (bytes corrupted)') 254 | 255 | fs.write('/test_file1',wData,len(wData)) 256 | wData = wData + wData 257 | log.info('completed writing testfile1: %d bytes'%len(wData)) 258 | rData = fs.read('/test_file1',len(wData),0) 259 | if rData == wData: log.info( 'read verified successfully') 260 | elif len(wData) != len(rData): log.error('read failed (%d of %d bytes)'%(len(rData),len(wData))) 261 | else: log.error('read failed (bytes corrupted)') 262 | 263 | fs.write('/test_file2',wData,0) 264 | log.info('completed writing testfile2: %d bytes'%len(wData)) 265 | rData = fs.read('/test_file2',len(wData),0) 266 | if rData == wData: log.info( 'read verified successfully') 267 | elif len(wData) != len(rData): log.error('read failed (%d of %d bytes)'%(len(rData),len(wData))) 268 | else: log.error('read failed (bytes corrupted)') 269 | 270 | log.info('file system testing complete') 271 | log.info('file system up and running') 272 | try: 273 | fs.main() 274 | except KeyboardInterrupt: 275 | log.info('fs stopping') 276 | sys.exit(1) 277 | 278 | 279 | -------------------------------------------------------------------------------- /ping.py: -------------------------------------------------------------------------------- 1 | import os,sys,socket,struct,select,time,binascii,logging 2 | import ping_reporter 3 | 4 | ping_count = 0 5 | ping_bandwidth = 0 6 | log = ping_reporter.setup_log('Ping') 7 | server_list = ['www.google.com','172.16.2.1','10.44.0.1'] 8 | 9 | def select_server(log,max_timeout=1): 10 | server = '' 11 | log.notice('selecting server') 12 | maxw = len(max(server_list, key=len)) 13 | min_delay = max_timeout * 1000 # seconds -> ms 14 | for x in server_list: 15 | delay = min_delay + 1 16 | try: delay = single_ping(x,max_timeout) 17 | finally: 18 | if delay == None: log.notice('%-*s: timed out'%(maxw,x)) 19 | else: log.notice('%-*s: %05.02fms'%(maxw,x,delay*1000)) 20 | if delay != None and delay < min_delay: 21 | min_delay = delay 22 | server = x 23 | log.info('selected server: %s (%.02fms)'%(server,min_delay*1000)) 24 | return server 25 | 26 | def carry_add(a, b): 27 | c = a + b 28 | return (c & 0xFFFF) + (c >> 16) 29 | 30 | def checksum(msg): 31 | s = 0 32 | if len(msg)%2: # pad with NULL 33 | msg = msg + '%c'%0 34 | for i in range(0, len(msg)/2*2, 2): 35 | w = ord(msg[i]) + (ord(msg[i+1]) << 8) 36 | s = carry_add(s, w) 37 | return ~s & 0xFFFF 38 | 39 | def build_ping(ID, data): 40 | log.trace('ping::build_ping: ID=%d, bytes=%d'%(ID,len(data))) 41 | if ID == 0: raise Exception('Invalid BlockID (0): many servers will corrupt ID=0 ICMP messages') 42 | 43 | data = str(data) # string type, like the packed result 44 | 45 | # Header is type (8), code (8), checksum (16), id (16), sequence (16) 46 | icmp_type = 8 # ICMP_ECHO_REQUEST 47 | icmp_code = 0 # Can be anything, but reply MUST be 0 48 | icmp_checksum = 0 # 0 for initial checksum calculation 49 | # icmp_id = (ID >> 16) & 0xFFFF 50 | # icmp_sequence = (ID << 0) & 0xFFFF 51 | block_id = ID # append id & seq for 4-byte identifier 52 | 53 | header = struct.pack("bbHL", icmp_type, icmp_code, icmp_checksum, block_id) 54 | icmp_checksum = checksum(header+data) 55 | header = struct.pack("bbHL", icmp_type, icmp_code, icmp_checksum, block_id) 56 | 57 | # Return built ICMP message 58 | return header+data 59 | 60 | def build_socket(RCVBUF=1024*1024): 61 | # By default, SO_RCVBUF is ~50k (kernel doubles to 114688), which only supports 62 | # ~1k blocks with <1ms timing. Raising this to 1m supports >16k blocks. Unfortunately, 63 | # raising it more does little because we can't read/process the events fast enough, so 64 | # the buffer pretty quickly fills, and then start dropping packets again. 65 | log.trace('ping::build_socket') 66 | icmp = socket.getprotobyname("icmp") 67 | try: 68 | icmp_socket = socket.socket(socket.AF_INET, socket.SOCK_RAW, icmp) 69 | except socket.error, (errno, msg): 70 | if errno == 1: # Operation not permitted 71 | msg = msg + (" (ICMP messages can only be sent from processes running as root)") 72 | raise socket.error(msg) 73 | raise # raise the original error 74 | socket.SO_RCVBUFFORCE = 33 75 | icmp_socket.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUFFORCE, RCVBUF) 76 | return icmp_socket 77 | 78 | def time_ping(d_socket, d_addr, ID=1): 79 | log.trace('ping::time_ping: server=%s ID=%d'%(d_addr,ID)) 80 | data = struct.pack("d",time.time()) 81 | return data_ping(d_socket, d_addr, ID, data) 82 | 83 | def data_ping(d_socket, d_addr, ID, data): 84 | log.trace('ping::time_ping: server=%s ID=%d bytes=%d'%(d_addr,ID,len(data))) 85 | global ping_count, ping_bandwidth 86 | packet = build_ping(ID,data) 87 | d_addr = socket.gethostbyname(d_addr) 88 | d_socket.sendto(packet, (d_addr, 1)) 89 | if 1: 90 | ping_count = ping_count + 1 91 | ping_bandwidth = ping_bandwidth + len(packet) 92 | 93 | def parse_ip(packet): 94 | log.trace('ping::parse_ip: bytes=%d'%(len(packet))) 95 | if len(packet) < 20: return None 96 | (verlen,ID,flags,frag,ttl,protocol,csum,src,dst) = struct.unpack('!B3xH4BHLL',packet[:20]) 97 | ip = dict( version= verlen >> 4, 98 | length= 4*(verlen & 0xF), 99 | ID= ID, 100 | flags= flags >> 5, 101 | fragment=((flags & 0x1F)+frag), 102 | ttl= ttl, 103 | protocol=protocol, 104 | checksum=csum, 105 | src= src, 106 | dst= dst) 107 | return ip 108 | 109 | def parse_icmp(packet,validate): 110 | log.trace('ping::parse_icmp: bytes=%d'%(len(packet))) 111 | if len(packet) < 8: return None 112 | (type, code, csum, block_id) = struct.unpack('bbHL', packet[:8]) 113 | log.debug('ping::parse_icmp: type=%d code=%d csum=%x ID=%d'%(type,code,csum,block_id)) 114 | icmp = dict(type=type, 115 | code=code, 116 | checksum=csum, # calculated big-endian 117 | block_id=block_id) 118 | 119 | if validate: 120 | t_header = struct.pack('bbHL',type,code,0,block_id) 121 | t_csum = checksum(t_header+packet[8:]) 122 | icmp['valid'] = (t_csum == csum) 123 | 124 | return icmp 125 | 126 | def parse_ping(packet,validate=False): 127 | log.trace('ping::parse_ping: bytes=%d validate=%s'%(len(packet),validate)) 128 | if len(packet) < 20+8+1: return None # require 1 block of data 129 | ip = parse_ip(packet) 130 | if not ip: return None 131 | if ip['protocol'] != socket.IPPROTO_ICMP: return None # ICMP 132 | if ip['version'] != socket.IPPROTO_IPIP: return None # IPv4 133 | if ip['length']+8+1 > len(packet): return None # invalid ICMP header 134 | 135 | packet = packet[ip['length']:] 136 | icmp = parse_icmp(packet,validate) 137 | if not icmp: return None 138 | if icmp['type'] != 0: return None # not an Echo Reply packet 139 | if icmp['code'] != 0: return None # not a valid Echo Reply packet 140 | if validate and icmp['valid'] != True: return None # invalid ICMP checksum 141 | 142 | payload = packet[8:] 143 | log.debug('ping::parse_ping: valid echo reply w/ ID=%d (%d bytes)'%(icmp['block_id'],len(payload))) 144 | return dict(ip=ip,icmp=icmp,payload=payload) 145 | 146 | 147 | def recv_ping(d_socket, timeout, validate=False): 148 | d_socket.settimeout(timeout) 149 | try: 150 | data,addr = d_socket.recvfrom(2048) 151 | except socket.timeout: 152 | return None 153 | parsed = parse_ping(data,validate) 154 | if not parsed: return None 155 | parsed['ID']=parsed['icmp']['block_id'] 156 | parsed['address']=addr 157 | parsed['raw']=data 158 | log.debug('ping::recv_ping: ID=%d address=%s bytes=%d'%(parsed['ID'],addr,len(data))) 159 | return parsed 160 | 161 | def read_ping(d_socket, timeout): 162 | start = time.time() 163 | while time.time() - start < timeout: 164 | msg = recv_ping(d_socket,timeout) 165 | if msg: return msg 166 | return None 167 | 168 | def receive_ping(my_socket, ID, timeout): 169 | timeLeft = timeout 170 | while True: 171 | startedSelect = time.time() 172 | whatReady = select.select([my_socket], [], [], timeLeft) 173 | if whatReady[0] == []: # Timeout 174 | return 175 | 176 | timeReceived = time.time() 177 | howLongInSelect = (timeReceived - startedSelect) 178 | recPacket, addr = my_socket.recvfrom(1024) 179 | icmpHeader = recPacket[20:28] 180 | type, code, checksum, packetID = struct.unpack("bbHL", icmpHeader) 181 | if packetID == ID: 182 | bytesInDouble = struct.calcsize("d") 183 | timeSent = struct.unpack("d", recPacket[28:28 + bytesInDouble])[0] 184 | return timeReceived - timeSent 185 | 186 | timeLeft = timeLeft - howLongInSelect 187 | if timeLeft <= 0: 188 | return 0 189 | 190 | def single_ping(dest_addr, timeout): 191 | my_socket = build_socket() 192 | my_ID = os.getpid() & 0xFFFF 193 | 194 | time_ping(my_socket, dest_addr, my_ID) 195 | delay = receive_ping(my_socket, my_ID, timeout) 196 | my_socket.close() 197 | return delay 198 | 199 | def verbose_ping(dest_addr, timeout = 2, count = 4): 200 | for i in xrange(count): 201 | log.info("ping %s..." % dest_addr,) 202 | try: 203 | delay = single_ping(dest_addr, timeout) 204 | except socket.gaierror, e: 205 | log.error("failed. (socket error: '%s')" % e[1]) 206 | break 207 | 208 | if delay == None: 209 | log.info("failed. (timeout within %ssec.)" % timeout) 210 | else: 211 | delay = delay * 1000 212 | log.info("get ping in %0.4fms" % delay) 213 | print 214 | 215 | 216 | import os, pwd, grp 217 | 218 | def drop_privileges(uid_name='nobody', gid_name='nogroup'): 219 | # by Tamas 220 | # http://stackoverflow.com/questions/2699907/dropping-root-permissions-in-python 221 | if os.getuid() != 0: return 222 | log.notice('ping::drop_privileges: uid=%s gid=%s'%(uid_name,gid_name)) 223 | 224 | try: 225 | # Get the uid/gid from the name 226 | running_uid = 1000#pwd.getpwnam(uid_name).pw_uid 227 | running_gid = 1000#grp.getgrnam(gid_name).gr_gid 228 | 229 | # Remove group privileges 230 | os.setgroups([]) 231 | 232 | # Try setting the new uid/gid 233 | os.setgid(running_gid) 234 | os.setuid(running_uid) 235 | 236 | # Ensure a very conservative umask 237 | old_umask = os.umask(077) 238 | print 'dropped permissions' 239 | except: 240 | raise OSError('ping::drop_privileges: failed to drop root privs') 241 | 242 | if __name__ == '__main__': 243 | ping_reporter.start_log(log) 244 | server = select_server(log,2) 245 | 246 | if 1: 247 | verbose_ping(server) 248 | else: 249 | s = build_socket() 250 | drop_privileges() 251 | print 'sending 100 pings...' 252 | for x in range(1,100): 253 | data_ping(s,server,x,struct.pack('d',x)) 254 | print 'ping cycled...' 255 | recv_ping(s,1) 256 | print '100 pings sent' 257 | 258 | -------------------------------------------------------------------------------- /ping_disk.py: -------------------------------------------------------------------------------- 1 | import ping, threading, time, socket, select, sys, struct, logging 2 | import binascii, threading, collections, math, random 3 | import ping, ping_server, ping_reporter 4 | 5 | log = ping_reporter.setup_log('PingDisk') 6 | 7 | class PingDisk(): 8 | def __init__(self, d_addr, block_size=1024, timeout=2): 9 | self.server = ping_server.PingServer(d_addr,block_size,timeout) 10 | self.server.setup() 11 | self.server.start() 12 | 13 | def stop(self): 14 | self.server.stop() 15 | 16 | def size(self): 17 | return self.server.block_size * (1<<28) 18 | 19 | def block_size(self): 20 | return self.server.block_size 21 | 22 | def region_size(self): 23 | return max(2,4096/self.block_size()) 24 | 25 | def read_block(self, ID, datastore, blocking=False): 26 | event = self.server.read_block(ID, self.__read_callback, datastore, False) 27 | if not blocking: return event 28 | event.wait() 29 | 30 | def read_block_sync(self, ID): 31 | data = {} 32 | self.read_block(ID,[data],True) 33 | return data[ID] 34 | 35 | def read_blocks(self, init_block, fini_block): 36 | data = {} 37 | events = [] 38 | result = '' 39 | blocks = range(init_block,fini_block+1) 40 | log.debug('PingDisk::read_blocks: blocks %d-%d'%(init_block,fini_block)) 41 | for x in blocks: events.append(self.read_block(x,[data])) 42 | for x in events: x.wait() 43 | for x in blocks: result = result + data[x] 44 | return result 45 | 46 | def __read_callback(self, ID, data, data_store): 47 | log.trace('PingDisk::read::callback: ID=%d bytes=%d'%(ID,len(data))) 48 | data_store[ID] = data 49 | 50 | def read(self, index, length): 51 | endex = index + length 52 | init_index = (index % self.server.block_size) 53 | fini_index = init_index + length 54 | init_block = (index / self.server.block_size) + 1 55 | fini_block = (endex / self.server.block_size) + 1 56 | 57 | if 0 == endex % self.server.block_size: 58 | fini_block = max(init_block,fini_block-1) 59 | data = self.read_blocks(init_block,fini_block) 60 | return data[init_index:fini_index] 61 | 62 | def __block_merge(self, old_data, new_data, index = 0): 63 | if index >= self.server.block_size: raise Exception('block_merge: invalid index ('+str(index)+')') 64 | old_data = old_data[:self.server.block_size] 65 | new_data = new_data[:self.server.block_size-index] 66 | data = old_data[:index] + new_data + old_data[index+len(new_data):] 67 | return data 68 | 69 | def write_block(self, ID, data, blocking=False): 70 | log.trace('PingDisk::write_block: ID=%d bytes=%d'%(ID,len(data))) 71 | return self.server.write_block(ID,data,blocking) 72 | 73 | def write_blocks(self, index, data): 74 | endex = index + len(data) 75 | block_size = self.server.block_size 76 | init_index = (index % self.server.block_size) 77 | fini_index = (endex % self.server.block_size) 78 | init_block = (index / self.server.block_size) + 1 # byte 0 is in block 1 79 | fini_block = (endex / self.server.block_size) + 1 80 | log.debug('PingDisk::write_blocks: blocks %d-%d'%(init_block,fini_block)) 81 | 82 | events = [] 83 | if init_index == 0: 84 | start_block = data[:block_size] 85 | else: 86 | start_block = self.read_block_sync(init_block) 87 | start_block = self.__block_merge(start_block,data,init_index) 88 | events.append(self.write_block(init_block,start_block)) 89 | if init_block == fini_block: return events 90 | 91 | data = data[self.server.block_size - init_index:] 92 | for x in range(init_block+1,fini_block): 93 | events.append(self.write_block(x,data[:block_size])) 94 | data = data[block_size:] 95 | 96 | if fini_index != 0: 97 | end_block = self.read_block_sync(fini_block) 98 | end_block = self.__block_merge(end_block,data,0) 99 | events.append(self.write_block(fini_block,end_block)) 100 | return events 101 | 102 | def write(self, index, data, blocking=True): 103 | events = self.write_blocks(index,data) 104 | if not blocking: return events 105 | for x in events: x.wait() 106 | 107 | def delete_blocks(self, index, length): 108 | endex = index + length 109 | init_block = (index / self.server.block_size) + 1 # byte 0 is in block 1 110 | fini_block = (endex / self.server.block_size) + 1 111 | log.debug('PingDisk::delete_blocks: blocks %d-%d'%(init_block,fini_block)) 112 | 113 | events = [] 114 | for x in range(init_block,fini_block+1): 115 | events.append(self.server.delete_block(x)) 116 | return events 117 | 118 | # delete operates at block-level boundaries 119 | def delete(self, index, length, blocking=False): 120 | log.debug('PingDisk::delete: index=%d for %d bytes'%(index,length)) 121 | events = self.delete_blocks(index,length) 122 | if not blocking: return events 123 | for x in events: x.wait() 124 | 125 | def free_blocks(self, timeout=None): 126 | blocks = ping_server.live_blocks(self.server,timeout) 127 | log.debug('live_blocks: %s'%blocks) 128 | return ping_server.free_blocks(blocks) 129 | 130 | def used_blocks(self, timeout=None): 131 | blocks = ping_server.live_blocks(self.server,timeout) 132 | log.debug('used_blocks: %s'%blocks) 133 | if blocks: return ping_server.used_blocks(blocks) 134 | return {} 135 | 136 | def get_block_region(self, blocks=1, timeout=None): 137 | free = self.free_blocks(timeout) 138 | log.debug('get_block_region: %d blocks'%(blocks)) 139 | log.debug('frees -> %s'%free) 140 | if not free: return None 141 | max_blockid = (1<<28) 142 | 143 | # 1) allocate an encompassing span of regions 144 | top_node = max(free.keys()) 145 | reg_size = self.region_size() * int(1 + blocks/self.region_size()) 146 | top_node = reg_size * int(1 + top_node/reg_size) 147 | if max_blockid - top_node > reg_size: return top_node 148 | 149 | # 2)try minimal sufficiently large region 150 | region = [(v,k) for (k,v) in free.iteritems() if v >= blocks] 151 | if region: return min(region)[1] 152 | return None 153 | 154 | def timeout(self): return self.server.timeout() 155 | def safe_timeout(self): return self.server.safe_timeout() 156 | def byte_to_block(self, byte): return int(math.ceil(1.0*byte/self.block_size())) 157 | def block_to_byte(self, block): return block * self.block_size() 158 | 159 | def get_region(self, bytes, timeout=None, target=None): 160 | log.debug('get_region: %d bytes'%(bytes)) 161 | if not timeout: timeout = self.safe_timeout() 162 | if target: return self.test_region(target,bytes,timeout) 163 | block = self.byte_to_block(bytes) # to blocks 164 | region = self.get_block_region(block,timeout) # <------> 165 | if region: region = self.block_to_byte(region) # to bytes 166 | log.debug('get_region: allocated region %d (%d bytes)'%(region,bytes)) 167 | return region 168 | 169 | def test_region(self, start, end, length, timeout=None): 170 | if not timeout: timeout = self.safe_timeout() 171 | log.debug('test_region: region=%d-%d length=%d'%(start,end,length)) 172 | if length < end: return start # smaller block 173 | collision = [start+end-1,start+length-1] 174 | collision2 = [collision[0],collision[1]] 175 | collision[0] = self.byte_to_block(collision[0]) 176 | collision[1] = self.byte_to_block(collision[1]) 177 | if collision[0] == collision[1]: return start # same block 178 | used = self.used_blocks(timeout) 179 | if not used: # 0 used blocks implies no root directory... 180 | log.exception('test_region: used blocks returned nil') 181 | raise Exception('test_region: used blocks returned nil') 182 | 183 | for x in used: 184 | if x <= collision[0]: continue # used block before test region (no collision) 185 | if x > collision[1]: continue # used block begins after collision space 186 | log.debug('test_region: collision at node %d'%x) 187 | return False 188 | return start 189 | 190 | 191 | if __name__ == "__main__": 192 | Disk = None 193 | try: 194 | #ping_reporter.start_log(log,logging.DEBUG) 195 | ping_reporter.enableAllLogs(logging.DEBUG) 196 | server = ping.select_server(log) 197 | Disk = PingDisk(server,4) 198 | #ping.drop_privileges() 199 | 200 | if 1: 201 | data = "1234567890123456789_123456789012345" 202 | log.info('blind disk read') 203 | rData = Disk.read(0,50) 204 | log.info('1-length: 50 requested -> %d'%(len(rData))) 205 | if rData != '\0'*50: log.exception('invalid rData: %s'%rData) 206 | else: log.info('success') 207 | time.sleep(5) 208 | 209 | log.info('writing %d bytes'%len(data)) 210 | Disk.write(0,data) 211 | time.sleep(5) 212 | rData = Disk.read(0,len(data)) 213 | log.info('2-length: %d vs %d'%(len(data),len(rData))) 214 | if rData != data: log.exception('invalid rData: %s'%rData) 215 | else: log.info('success') 216 | 217 | wData = 'abcdefghijk' 218 | Disk.write(10,wData) 219 | time.sleep(2) 220 | rData = Disk.read(0,len(data)) 221 | data = data[0:10] + wData + data[10+len(wData):] 222 | log.info('3-length: %d vs %d'%(len(data),len(rData))) 223 | if rData != data: log.exception('invalid rData: %s'%rData) 224 | else: log.info('success') 225 | 226 | time.sleep(2) 227 | data = data[2:] + '\0\0' 228 | rData = Disk.read(2,len(data)) 229 | log.info('4-length: %d vs %d'%(len(data),len(rData))) 230 | if rData != data: log.exception('invalid rData: %s'%rData) 231 | else: log.info('success') 232 | 233 | time.sleep(2) 234 | data = data[0:10] 235 | rData = Disk.read(2,10) 236 | log.info('5-length: %d vs %d'%(len(data),len(rData))) 237 | if rData != data: log.exception('invalid rData: %s'%rData) 238 | else: log.info('success') 239 | 240 | if 1: 241 | #free_node = Disk.get_region(1500) 242 | #log.info('get_region = %s'%free_node) 243 | 244 | strA = 'A' 245 | strB = 'B'*4096*4 246 | Disk.write(0,strA) 247 | Disk.write(5000,strB) 248 | 249 | time.sleep(3) 250 | #log.info('region A [%d] and B [%d] written'%(len(strA),len(strB))) 251 | readA = Disk.read(0,len(strA)) 252 | readB = Disk.read(5000,len(strB)) 253 | log.info('region A and B read') 254 | if readA != strA: log.error('corruption in region A (%d bytes)'%len(readA)) 255 | else: log.debug('region A read successfully') 256 | if readB != strB: log.error('corruption in region B (%d bytes)'%len(readB)) 257 | else: log.debug('region B read successfully') 258 | 259 | time.sleep(10) 260 | print 'terminate' 261 | except KeyboardInterrupt: 262 | print "Keyboard Interrupt" 263 | except Exception: 264 | print 'General Exception' 265 | from traceback import print_exc 266 | print_exc() 267 | finally: 268 | if Disk: Disk.stop() 269 | log.info('traffic: %d pings (%s)' 270 | %(ping.ping_count,ping_reporter.humanize_bytes(ping.ping_bandwidth))) 271 | sys.exit(1) 272 | 273 | -------------------------------------------------------------------------------- /ping_server.py: -------------------------------------------------------------------------------- 1 | import ping, threading, time, socket, select, sys, struct, Queue 2 | import binascii, collections, math, random, logging 3 | import ping_reporter 4 | 5 | log = ping_reporter.setup_log('PingServer') 6 | 7 | class PingTimer(threading.Thread): # helper class for PingServer to manage timeouts 8 | def __init__(self, event): 9 | self.queue = Queue.PriorityQueue() 10 | threading.Thread.__init__(self) 11 | self.running = False 12 | self.event = event 13 | 14 | def stop(self): 15 | log.debug('PingTimeout terminating') 16 | self.running = False 17 | self.event.set() 18 | 19 | def run(self): 20 | self.running = True 21 | log.debug('PingTimeout starting') 22 | while self.running: 23 | timeout = None 24 | self.event.clear() 25 | timeout = self.process() 26 | self.event.wait(timeout) 27 | 28 | def process(self): 29 | while self.queue.qsize() > 0: 30 | try: expire,event,callback,cb_args = item = self.queue.get_nowait() 31 | except Queue.Empty: break # our qsize check isn't guaranteed to prevent this 32 | if event.is_set(): continue # event was completed; ignore it 33 | if expire > time.time(): 34 | self.queue.put(item) 35 | return expire - time.time() 36 | callback(*cb_args) 37 | event.set() # make sure no one executes it 38 | return None 39 | 40 | def add_callback(self, timeout, handler, args): 41 | event = threading.Event() 42 | item = (time.time()+timeout,event,handler,args) 43 | self.queue.put(item) 44 | self.event.set() 45 | return event 46 | 47 | 48 | class PingServer(threading.Thread): 49 | def __init__(self, d_addr, block_size=1024, initial_timeout=2): 50 | self.block_size = block_size # default; use setup for exact 51 | self.server = d_addr,socket.gethostbyname(d_addr) 52 | self.running_timeout = initial_timeout 53 | threading.Thread.__init__(self) 54 | self.listeners = [] 55 | self.debug = 0 56 | 57 | # timeout events are queued and executed in a seperate thread 58 | self.timer_event = threading.Event() 59 | self.timer = PingTimer(self.timer_event) 60 | 61 | 62 | self.blocks = 0 63 | self.running = False 64 | self.socket = ping.build_socket() 65 | self.empty_block = self.null_block() 66 | self.queued_events = collections.defaultdict(collections.deque) 67 | 68 | def timeout(self): return 2.0/5.0 # self.running_timeout 69 | def safe_timeout(self): return 3 * self.timeout() 70 | 71 | def setup_timeout(self, ID=0): 72 | Time = time.time() 73 | Times = struct.pack('d',Time) 74 | if ID == 0: ID = random.getrandbits(32) # ID size in bits 75 | 76 | ping.data_ping(self.socket,self.server[1],ID,Times) 77 | msg = ping.read_ping(self.socket,self.timeout()) 78 | if not msg: raise Exception('PingServer::setup_timeout: no valid response from '+self.server[0]) 79 | addr,rID,data = msg['address'],msg['ID'],msg['payload'] 80 | log.debug("Addr=%s rID=%d Data=%d bytes"%(addr[0],rID,len(data))) 81 | if len(data) == 0: raise Exception('PingServer::setup_timeout: null response from '+self.server[0]) 82 | if rID != ID: raise Exception('PingServer::setup_timeout: invalid response id from '+self.server[0]) 83 | if data != Times: raise Exception('PingServer::setup_timeout: invalid response data from '+self.server[0]) 84 | if addr[0] != self.server[1]: raise Exception('PingServer::setup_timeout: invalid response server from '+self.server[0]) 85 | delay = time.time() - Time 86 | log.notice('echo delay: %.02fms'%(1000*delay)) 87 | 88 | def setup_block(self, ID = 0): 89 | if ID == 0: ID = random.getrandbits(32) # ID size in bits 90 | Fill = chr(random.getrandbits(8)) # repeated data 91 | Filler = self.block_size * Fill 92 | 93 | ping.data_ping(self.socket,self.server[1],ID,Filler) 94 | msg = ping.read_ping(self.socket,self.timeout()) 95 | if not msg: raise Exception('PingServer::setup_block: no valid response from '+self.server[0]) 96 | addr,rID,data = msg['address'],msg['ID'],msg['payload'] 97 | log.debug("Addr=%s rID=%d Data=%d bytes"%(addr[0],rID,len(data))) 98 | if len(data) == 0: raise Exception('PingServer::setup_block: null response from '+self.server[0]) 99 | if rID != ID: raise Exception('PingServer::setup_block: invalid response id from '+self.server[0]) 100 | if data != len(data)*Fill: raise Exception('PingServer::setup_block: invalid response data from '+self.server[0]) 101 | if addr[0] != self.server[1]: raise Exception('PingServer::setup_block: invalid response server from '+self.server[0]) 102 | self.block_size = len(data) 103 | self.empty_block = self.null_block() 104 | log.notice('echo length: %d bytes'%self.block_size) 105 | 106 | def setup(self): 107 | log.trace('PingServer::setup: testing server "%s"'%self.server[0]) 108 | ID = random.getrandbits(32) 109 | self.setup_timeout(ID) 110 | self.setup_block(ID) 111 | 112 | def stop(self): 113 | self.running = False 114 | log.info('PingServer terminating') 115 | self.timer.stop() 116 | 117 | def run(self): 118 | self.running = True 119 | log.notice('PingServer starting') 120 | self.timer.start() 121 | while self.running: 122 | start_blocks = self.blocks # updated asynchronously 123 | ready = select.select([self.socket], [], [], self.timeout()) 124 | if ready[0] == []: # timeout 125 | if start_blocks != 0 and self.blocks != 0: 126 | log.error('%s timed out'%self.server[0]) 127 | try: 128 | msg = ping.recv_ping(self.socket,self.timeout()) 129 | if not msg: continue 130 | except: 131 | continue 132 | addr,block_id,data = msg['address'],msg['ID'],msg['payload'] 133 | if block_id == 0: 134 | import binascii 135 | raise Exception('received packet w/ ID 0 packet: '+binascii.hexlify(msg['raw'])) 136 | self.process_block(addr[0],block_id,data) 137 | 138 | def process_block(self, addr, ID, data): 139 | if ID == 0: raise Exception('server responded with ID 0 packet') 140 | 141 | while len(self.queued_events[ID]): 142 | handler,event,args = self.queued_events[ID].popleft() 143 | if event.is_set(): continue 144 | 145 | if handler == self.write_block_timeout: 146 | if self.debug: log.trace('%s (block %d) updated'%(self.server[0],ID)) 147 | data = args[1] 148 | elif handler == self.read_block_timeout: 149 | if self.debug: log.trace('%s (block %d) read'%(self.server[0],ID)) 150 | callback,cb_args = args[1],args[2] 151 | if len(data) > 0: callback(ID,data,*cb_args) 152 | else: callback(ID,self.null_block(),*cb_args) 153 | elif handler == self.delete_block_timeout: 154 | if self.debug: log.trace('%s (block %d) deleted'%(self.server[0],ID)) 155 | data = '' 156 | event.set() 157 | 158 | if len(data) == 0: 159 | self.blocks = self.blocks - 1 160 | else: 161 | if len(self.listeners): self.process_listeners(addr, ID, data) 162 | #log.trace('%s: sending %d bytes from block %d'%(self.server[0],len(data),ID)) 163 | ping.data_ping(self.socket, addr, ID, data) 164 | 165 | def process_listeners(self, addr, ID, data): 166 | if not self.listeners: raise Exception('process_listeners invoked without valid listeners on ID=%d'%ID) 167 | self.listeners = [l for l in self.listeners if l[0] >= time.time()] # clean the listeners 168 | for x in self.listeners: 169 | expire,handler,cb_args = x 170 | handler(ID, addr, data, *cb_args) 171 | 172 | def add_listener(self, handler, timeout, args): 173 | log.debug('add_listener: timeout=%d handler=%s'%(timeout,handler)) 174 | expire = time.time() + timeout 175 | self.listeners.append((expire,handler,args)) 176 | 177 | def null_block(self): 178 | return self.block_size * struct.pack('B',0) 179 | 180 | def event_insert(self, ID, handler, args): 181 | event = self.timer.add_callback(self.timeout(), handler, args) 182 | self.queued_events[ID].append((handler,event,args)) 183 | return event 184 | 185 | # read / write / delete a single block 186 | def write_block(self, ID, data, blocking = False): 187 | # add a block to the queue (or delete if equivalent) 188 | log.trace('PingServer::write_block: ID=%d bytes=%d blocking=%s'%(ID,len(data),blocking)) 189 | if ID == 0: raise Exception('write_block: invalid block ID (0)') 190 | if data == '%c'%0 * len(data): return self.delete_block(ID,blocking) 191 | event = self.event_insert(ID,self.write_block_timeout,[ID,data[:self.block_size]]) 192 | if blocking: event.wait() 193 | return event 194 | 195 | def delete_block(self, ID, blocking = False): 196 | log.trace('PingServer::delete_block: ID=%d blocking=%s'%(ID,blocking)) 197 | if ID == 0: raise Exception('delete_block: invalid block ID (0)') 198 | t = self.event_insert(ID,self.delete_block_timeout,[ID]) 199 | if blocking: t.wait() 200 | return t 201 | 202 | def read_block(self, ID, callback, cb_args = [], blocking = False): 203 | log.trace('PingServer::read_block: ID=%d blocking=%s'%(ID,blocking)) 204 | if ID == 0: raise Exception('read_block: invalid block ID (0)') 205 | t = self.event_insert(ID,self.read_block_timeout,[ID,callback,cb_args]) 206 | if blocking: t.wait() 207 | return t 208 | 209 | def read_block_timeout(self, ID, callback, cb_args): 210 | log.debug('PingServer::read_block_timeout: ID=%d callback=%s'%(ID,callback.__name__)) 211 | callback(ID,self.null_block(),*cb_args) 212 | 213 | def delete_block_timeout(self, ID): 214 | log.debug('PingServer::delete_block_timeout: ID=%d'%ID) 215 | # do nothing; we're marked invalid anyhow 216 | pass 217 | 218 | def write_block_timeout(self, ID, data): 219 | log.trace('PingServer::write_block_timeout: ID=%d bytes=%d'%(ID,len(data))) 220 | self.blocks = self.blocks + 1 221 | # force update queue (as if packet arrived) 222 | if ID == 0: raise Exception('write_block_timeout: ID == 0') 223 | self.process_block(self.server[1], ID, data) 224 | 225 | def print_block(ID, data): 226 | print '----- print block -----' 227 | print 'block',ID,'bytes',len(data) 228 | print data 229 | print '----- print block -----' 230 | 231 | def __live_blocks(ID, addr, data, datastore): 232 | datastore[ID] = 1 233 | 234 | def live_blocks(PServer, timeout=None): 235 | store = {} 236 | if not timeout: timeout = PServer.safe_timeout() 237 | PServer.add_listener(__live_blocks,timeout,[store]) 238 | time.sleep(timeout) 239 | return store 240 | 241 | def used_blocks(blocks): 242 | result,lookup = {},{} 243 | for x in blocks: 244 | if x-1 in lookup: 245 | lookup[x] = lookup[x-1] 246 | result[lookup[x]] += 1 247 | else: 248 | lookup[x] = x 249 | result[x] = 1 250 | return result 251 | 252 | 253 | def free_blocks(blocks): 254 | result = {} 255 | if 1 not in blocks: 256 | if not blocks: result[1] = 0 257 | elif len(blocks) == 0: result[1] = 0 258 | else: result[1] = min(blocks.keys())-1 259 | for x in blocks: 260 | if not x+1 in blocks: result[x+1] = 0 261 | if not x-1 in blocks: 262 | if not len(result): continue 263 | block = max(result.keys()) 264 | result[block] = x-block 265 | return result 266 | 267 | if __name__ == "__main__": 268 | ping_reporter.start_log(log,logging.DEBUG) 269 | server = ping.select_server(log,1) 270 | 271 | from ping_reporter import humanize_bytes 272 | try: 273 | PS = PingServer(server) 274 | PS.debug = 1 275 | PS.setup() 276 | PS.start() 277 | print 'traffic:',ping.ping_count,'pings ('+humanize_bytes(ping.ping_bandwidth)+')' 278 | PS.read_block(2,print_block) 279 | time.sleep(2) 280 | PS.write_block(2,'coconut') 281 | time.sleep(1) 282 | print 'traffic:',ping.ping_count,'pings ('+humanize_bytes(ping.ping_bandwidth)+')' 283 | 284 | PS.write_block(1,'apples') 285 | PS.read_block(1,print_block) 286 | PS.read_block(1,print_block) 287 | time.sleep(2) 288 | print 'traffic:',ping.ping_count,'pings ('+humanize_bytes(ping.ping_bandwidth)+')' 289 | 290 | log.info('testing block metrics') 291 | blocks = live_blocks(PS) 292 | log.debug('blocks: %s'%blocks) 293 | log.debug('--used: %s'%used_blocks(blocks)) 294 | log.debug('--free: %s'%free_blocks(blocks)) 295 | 296 | PS.delete_block(1) 297 | time.sleep(2) 298 | print 'traffic:',ping.ping_count,'pings ('+humanize_bytes(ping.ping_bandwidth)+')' 299 | PS.write_block(1,'apples') 300 | time.sleep(2) 301 | PS.read_block(1,print_block) 302 | time.sleep(4) 303 | PS.read_block(1,print_block) 304 | time.sleep(1) 305 | PS.write_block(1,'bananas') 306 | time.sleep(1) 307 | PS.read_block(1,print_block) 308 | time.sleep(1) 309 | PS.read_block(1,print_block) 310 | PS.read_block(1,print_block) 311 | time.sleep(1) 312 | PS.delete_block(1) 313 | print 'traffic:',ping.ping_count,'pings ('+humanize_bytes(ping.ping_bandwidth)+')' 314 | while True: 315 | time.sleep(1) 316 | print 'terminate' 317 | except KeyboardInterrupt: 318 | print "Keyboard Interrupt" 319 | except Exception: 320 | print 'General Exception' 321 | from traceback import print_exc 322 | print_exc() 323 | finally: 324 | PS.stop() 325 | print 'traffic:',ping.ping_count,'pings ('+humanize_bytes(ping.ping_bandwidth)+')' 326 | sys.exit(1) 327 | 328 | -------------------------------------------------------------------------------- /ping_filesystem.py: -------------------------------------------------------------------------------- 1 | import time, struct, sys, stat, logging 2 | import ping, ping_disk, ping_reporter 3 | 4 | log = ping_reporter.setup_log('PingFileSystem') 5 | 6 | """ 7 | PingFS_File 8 | [00: 4] size 9 | [04: 4] type 10 | [08: 2] uid 11 | [0a: 2] gid 12 | [0c: 2] mode 13 | [0e: 2] reserved 14 | [10:__] data 15 | 16 | PingFS_Directory(PingFS_File) 17 | [10: 4] entry count 18 | [14:__] entries 19 | 20 | PingFS_DirEntry 21 | [00: 2] name length 22 | [02:__] name 23 | """ 24 | 25 | def makePingNode(data): 26 | pnode = PingNode() 27 | pnode.deserialize(data) 28 | return pnode 29 | 30 | def makePingFile(data): 31 | pfile = PingFile() 32 | pfile.deserialize(data) 33 | return pfile 34 | 35 | def makePingDirent(data): 36 | pdir = PingDirent() 37 | pdir.deserialize(data) 38 | return pdir 39 | 40 | def makePingDirectory(data): 41 | pdir = PingDirectory() 42 | pdir.deserialize(data) 43 | return pdir 44 | 45 | def interpretFile(data): 46 | pf = makePingFile(data) 47 | if pf.type == stat.S_IFDIR: return makePingDirectory(data) 48 | return pf 49 | 50 | def interpretSize(data): 51 | inode,size = struct.unpack('2L',data[:struct.calcsize('2L')]) 52 | return size 53 | 54 | class PingNode(): 55 | layout = 'L' 56 | overhead = struct.calcsize(layout) 57 | 58 | def __init__(self,inode=0): 59 | self.parent = None 60 | self.inode = inode 61 | 62 | def get_parts(self,data,size): 63 | if len(data) < size: return '' 64 | return data[:size],data[size:] 65 | 66 | def serialize(self): 67 | log.trace('%s::serialize'%self.__class__.__name__) 68 | return struct.pack(PingNode.layout,self.inode) 69 | 70 | def deserialize(self,data): 71 | log.trace('%s::deserialize'%self.__class__.__name__) 72 | layout,overhead = PingNode.layout,PingNode.overhead 73 | if len(data) < overhead: raise Exception('PingFS::node: invalid deserialize data') 74 | self.inode = struct.unpack(layout,data[:overhead])[0] 75 | return data[overhead:] 76 | 77 | class PingFile(PingNode): 78 | layout = '2L3H2x' 79 | overhead = struct.calcsize(layout) 80 | file_header = overhead + PingNode.overhead 81 | 82 | def __init__(self,name='',inode=0): 83 | PingNode.__init__(self,inode) 84 | self.type = stat.S_IFREG 85 | self.mode = 0666 86 | self.name = name 87 | self.data = '' 88 | self.uid = 0 89 | self.gid = 0 90 | 91 | def get_attr(self): 92 | return self.attrs 93 | 94 | def size(self): 95 | return PingFile.file_header + len(self.data) 96 | 97 | def links(self): 98 | return 1 99 | 100 | def serialize(self): 101 | self.disk_size = self.size() 102 | node_hdr = PingNode.serialize(self) 103 | layout,overhead = PingFile.layout,PingFile.overhead 104 | file_hdr = struct.pack(layout,len(self.data),self.type,self.uid,self.gid,self.mode) 105 | return node_hdr + file_hdr + self.data 106 | 107 | def deserialize(self,data): 108 | data = PingNode.deserialize(self,data) 109 | layout,overhead = PingFile.layout,PingFile.overhead 110 | if len(data) < overhead: raise Exception('PingFS::file: invalid deserialize data') 111 | size,self.type,self.uid,self.gid,self.mode = struct.unpack(layout,data[:overhead]) 112 | self.data = data[overhead:overhead+size] 113 | self.disk_size = self.size() 114 | #print 'PingFile::name(',self.name,'),size,type,attr:',size,self.type,self.attr 115 | return data[overhead+size:] 116 | 117 | class PingDirent(PingNode): 118 | layout = 'H' 119 | overhead = struct.calcsize(layout) 120 | 121 | def __init__(self): 122 | PingNode.__init__(self,None) 123 | 124 | def size(self): 125 | return PingNode.overhead + PingDirent.overhead + len(self.name) 126 | 127 | def serialize(self): 128 | node_hdr = PingNode.serialize(self) 129 | layout,overhead = PingDirent.layout,PingDirent.overhead 130 | header = struct.pack(layout,len(self.name)) 131 | return node_hdr + header + self.name 132 | 133 | def deserialize(self,data): 134 | data = PingNode.deserialize(self,data) 135 | layout,overhead = PingDirent.layout,PingDirent.overhead 136 | if len(data) < overhead: raise Exception('PingFS::dirent: invalid deserialize') 137 | size = struct.unpack(layout,data[:overhead])[0] 138 | data = data[overhead:] 139 | if len(data) < size: raise Exception('PingFS::dirent: invalid directory object (%d,%d)' 140 | %(len(data),size)) 141 | self.name = data[:size] 142 | #print 'PingDirent::inode,len,name',self.inode,len(self.name),self.name 143 | return data[size:] 144 | 145 | class PingDirectory(PingFile): 146 | layout = 'L' 147 | overhead = struct.calcsize(layout) 148 | 149 | def __init__(self,name='',inode=0): 150 | PingFile.__init__(self,name,inode) 151 | self.type = stat.S_IFDIR 152 | self.entries = [] 153 | self.mode = 0766 154 | 155 | def size(self): 156 | size = PingFile.overhead + PingDirectory.overhead 157 | for x in self.entries: 158 | size = size + x.size() 159 | return size 160 | 161 | def links(self): 162 | return len(self.entries) + 1 163 | 164 | def add_node(self,node): 165 | if node.parent: node.parent.del_node(node.name,node) 166 | self.del_node(node.name) 167 | dirent = PingDirent() 168 | dirent.name = node.name 169 | dirent.inode = node.inode 170 | self.entries.append(dirent) 171 | node.parent = self 172 | 173 | def del_node(self,name,node=None): 174 | self.entries = [x for x in self.entries if x.name != name] 175 | if node: node.parent = None 176 | 177 | def get_dirent(self,name,node=None): 178 | for x in self.entries: 179 | if x.name == name: 180 | return x 181 | return None 182 | 183 | def serialize(self): 184 | file_hdr = PingFile.serialize(self) 185 | layout,overhead = PingDirectory.layout,PingDirectory.overhead 186 | header = struct.pack(layout, len(self.entries)) 187 | 188 | data = '' 189 | for x in self.entries: data = data + x.serialize() 190 | return file_hdr + header + data 191 | 192 | def deserialize(self,data): 193 | self.entries = [] 194 | data = PingFile.deserialize(self,data) 195 | layout,overhead = PingDirectory.layout,PingDirectory.overhead 196 | if len(data) < overhead: raise Exception('PingFS::dir: invalid deserialize') 197 | count = struct.unpack(layout,data[:overhead])[0] 198 | data = data[overhead:] 199 | for x in range(0,count): 200 | dirent = PingDirent() 201 | data = dirent.deserialize(data) 202 | self.add_node(dirent) 203 | return data 204 | 205 | class PingFS: 206 | def __init__(self,server): 207 | try: 208 | self.disk = ping_disk.PingDisk(server) 209 | self.cache = PingDirectory('/') # create root 210 | self.add(self.cache,0) # and cache it 211 | 212 | except: 213 | print 'General Exception' 214 | from traceback import print_exc 215 | print_exc() 216 | 217 | def read_inode(self,inode,length=0): 218 | log.debug('PingFS::read_inode: inode=%d length=%d'%(inode,length)) 219 | if length == 0: block_size = max(self.disk.block_size(),PingFile.file_header) 220 | data = self.disk.read(inode,block_size) 221 | size = PingFile.file_header + interpretSize(data) 222 | 223 | if size > len(data): 224 | data = self.disk.read(inode,size) 225 | return data 226 | 227 | def read_as_file(self,inode): 228 | log.debug('PingFS::read_as_file: inode=%d'%inode) 229 | data = self.read_inode(inode) 230 | pfile = make 231 | pfile = makePingFile(data) 232 | return pfile 233 | 234 | def read_as_dir(self,inode): 235 | log.debug('PingFS::read_as_dir: inode=%d'%inode) 236 | data = self.read_inode(inode) 237 | pdir = makePingDirectory(data) 238 | if not (pdir.type & stat.S_IFDIR): 239 | raise Exception('read_as_dir: %s (%d,%d) -> %x %d'%(pdir.name,inode,len(data),pdir.type,len(pdir.entries))) 240 | return pdir 241 | 242 | def cache_hit(self,name,pFile=None): 243 | if not self.cache: return False 244 | if self.cache.name != name: return False 245 | if pFile and self.cache.inode != pFile.inode: return False 246 | return True 247 | 248 | def get(self, path): 249 | log.notice('PingFS::get %s'%path) 250 | if self.cache_hit(path): return self.cache 251 | if path == '/' or path == '': 252 | if self.cache.inode == 0: return self.cache 253 | return self.read_as_dir(0) 254 | parts = path.rsplit('/',1) 255 | if len(parts) != 2: raise Exception('PingFS::get_file: invalid path: %s'%path) 256 | rPath,fName = parts[0],parts[1] 257 | pDir = self.get(rPath) 258 | if pDir and pDir.type == stat.S_IFDIR: 259 | pEntry = pDir.get_dirent(fName) 260 | self.cache = pDir # cache the directory 261 | self.cache.name = rPath 262 | if pEntry: 263 | data = self.read_inode(pEntry.inode) 264 | pFile = interpretFile(data) 265 | pFile.name = pEntry.name 266 | return pFile 267 | return None 268 | 269 | def get_both(self, path): 270 | log.notice('PingFS::get_both %s'%path) 271 | if self.cache_hit(path): 272 | if self.cache.parent: 273 | return (self.cache.parent,self.cache) 274 | if path == '/' or path == '': 275 | if self.cache.inode == 0: 276 | return (self.cache,self.cache) 277 | return self.read_as_dir(0) 278 | parts = path.rsplit('/',1) 279 | if len(parts) != 2: raise Exception('PingFS::get_both: invalid path: %s'%path) 280 | sPath,sName = parts[0],parts[1] 281 | pDir = self.get(sPath) 282 | if not pDir: return (None,None) 283 | if not pDir.type == stat.S_IFDIR: return (None,None) 284 | pEntry = pDir.get_dirent(sName) 285 | self.cache = pDir # cache the directory 286 | self.cache.name = sPath 287 | if not pEntry: return (pDir,None) 288 | data = self.read_inode(pEntry.inode) 289 | pFile = interpretFile(data) 290 | pFile.name = pEntry.name 291 | return (pDir,pFile) 292 | 293 | def get_parent(self, path, pFile=None): 294 | if path == '/' or path == '': return self.read_as_dir(0) 295 | parts = path.rsplit('/',1) 296 | if len(parts) != 2: 297 | log.exception('PingFS::get_parent: invalid path: %s'%path) 298 | return None 299 | pDir = self.get(parts[0]) 300 | if pDir.type != stat.S_IFDIR: return None 301 | return pDir 302 | 303 | def root_node(self, node): 304 | if node.inode == 0: return True 305 | return False 306 | 307 | def unlink(self, path, pFile=None, pDir=None): 308 | log.notice('PingFS::unlink %s'%path) 309 | if not pFile: pFile = self.get(path) 310 | if not pFile: return False 311 | if self.root_node(pFile): return False # don't delete the root 312 | if not pDir: pDir = self.get_parent(path,pFile) 313 | if pDir: self.disconnect(path,pFile,pDir) 314 | self.delete(path,pFile) 315 | return True 316 | 317 | def disconnect(self, path, pFile=None, pDir=None): 318 | log.notice('PingFS::disconnect %s'%path) 319 | if path == '/' or path == '': return False 320 | if not pFile: pFile = self.get(path) 321 | if not pFile: return False 322 | if not pDir: pDir = self.get_parent(path,pFile) 323 | if not pDir: return True # we're technically disconnected 324 | pDir.del_node(pFile.name,pFile) 325 | self.update(pDir) 326 | return True 327 | 328 | def delete(self, path, pFile=None): # assumes node disconnected from dir tree 329 | log.notice('PingFS::delete %s'%path) 330 | if not pFile: pFile = self.get(path) 331 | if not pFile: return False 332 | if self.cache_hit(path,pFile): self.cache = None 333 | self.disk.delete(pFile.inode,pFile.size()) 334 | 335 | def move_blocks(self, path, pFile, dest, pDir=None): 336 | log.debug('move_blocks: %s (%d->%d)'%(pFile.name,pFile.inode,dest)) 337 | if self.root_node(pFile): return False # don't move the root 338 | if not pDir: pDir = self.get_parent(path,pFile) 339 | if not pDir: return False 340 | self.delete(pFile.name,pFile) 341 | self.add(pFile,dest) 342 | dirent = pDir.get_dirent(pFile.name,pFile) 343 | dirent.inode = dest 344 | self.update(pDir) 345 | return True 346 | 347 | def move_links(self, pFile, oDir, nDir): 348 | log.notice('move_links: %s (%s -> %s)'%(pFile.name,oDir.name,nDir.name)) 349 | if self.root_node(pFile): raise Exception('move_link on root!') 350 | if not oDir.get_dirent(pFile.name,pFile): return False 351 | oDir.del_node(pFile.name,pFile); self.update(oDir) 352 | nDir.add_node(pFile.name,pFile); self.update(nDir) 353 | return True 354 | 355 | def cache_update(self,node): 356 | if not self.cache: return 357 | if self.cache.inode != node.inode: return 358 | self.cache = node 359 | 360 | def add(self,node,force_inode=None): 361 | if force_inode != None: 362 | node.inode = force_inode 363 | else: 364 | node.inode = self.disk.get_region(node.size()) 365 | if not node.inode: return None 366 | log.notice('PingFS::add %s at %d'%(node.name,node.inode)) 367 | self.disk.write(node.inode,node.serialize()) 368 | self.cache_update(node) 369 | return node.inode 370 | 371 | def relocate(self,pFile,pDir=None): 372 | log.notice('relocating %s to larger region'%pFile) 373 | region = self.disk.get_region(pFile.size()) 374 | if not region: raise Exception('PingFS::update %s at %d: collision correction fail'%(pFile.name,pFile.inode)) 375 | if not pFile.parent: pFile.parent = pDir 376 | if not pFile.parent: raise Exception('PingFS::update %s at %d: collision parent not found'%(pFile.name,pFile.inode)) 377 | if not self.move_blocks(None,pFile,region,pFile.parent): 378 | raise Exception('PingFS::update %s at %d: collision correction failed'%(pFile.name,pFile.inode)) 379 | log.notice('relocated %d:%s to region %d'%(pFile.inode,pFile.name,region)) 380 | return True 381 | 382 | def update(self,pFile,pDir=None): 383 | log.debug('PingFS::update %s at %d [%d -> %d]'%(pFile.name,pFile.inode,pFile.disk_size,pFile.size())) 384 | if pFile.size() > pFile.disk_size: 385 | region = self.disk.test_region(pFile.inode,pFile.disk_size,pFile.size()) 386 | if region != pFile.inode: return self.relocate(pFile,pDir) # continuing would cause collision 387 | self.disk.write(pFile.inode,pFile.serialize()) 388 | self.cache_update(pFile) 389 | return True 390 | 391 | def create(self,path,buf='',offset=0): 392 | log.debug('PingFS::create %s (offset=%d len=%d)'%(path,offset,len(buf))) 393 | parts = path.rsplit('/',1) 394 | if len(parts) != 2: 395 | log.exception('PingFS::create: invalid path: %s'%path) 396 | return False 397 | rPath,rName = parts[0],parts[1] 398 | pDir = self.get(rPath) 399 | if not pDir: 400 | log.error('PingFS::create invalid parent dir: %s'%path) 401 | return False 402 | pFile = PingFile(rName) 403 | if not offset: offset = '' 404 | else: offset = '\0'*offset 405 | pFile.data = offset + buf 406 | inode = self.add(pFile) 407 | pDir.add_node(pFile) 408 | self.update(pDir) 409 | return pFile 410 | 411 | def stop(self): 412 | log.info('PingFS: stopping') 413 | self.disk.stop() 414 | 415 | def init_fs(FS): 416 | log.notice('building nodes') 417 | d1 = PingDirectory('/') 418 | d2 = PingDirectory('l1') 419 | f1 = PingFile('apples') 420 | f2 = PingFile('banana') 421 | f1.mode = 0700 422 | f1.uid = 1000 423 | f1.gid = 1000 424 | 425 | log.notice('adding nodes to system') 426 | FS.add(d1,0) 427 | FS.add(d2) 428 | FS.add(f1) 429 | FS.add(f2) 430 | 431 | log.notice('connecting nodes') 432 | d1.add_node(d2) 433 | d1.add_node(f1) 434 | d2.add_node(f2) 435 | 436 | log.notice('fleshing out nodes') 437 | f1.data = 'delicious apples\n' 438 | f2.data = 'ripe yellow bananas\n' 439 | 440 | log.notice('updating nodes in system') 441 | FS.update(d1) 442 | FS.update(d2) 443 | FS.update(f1) 444 | FS.update(f2) 445 | 446 | FS.create('/l1/bonus','contenttttttt',0) 447 | 448 | log.notice('test filesystem initialized') 449 | 450 | def test_fs(FS): 451 | root = FS.read_as_dir(0) 452 | log.info('- read as dir / -----------------------------') 453 | log.info('%d' % root.type) 454 | log.info('---------------------------------------------') 455 | 456 | root = FS.get('') 457 | log.info('- get "" ------------------------------------') 458 | log.info('%d' % root.type) 459 | log.info('---------------------------------------------') 460 | 461 | root = FS.get('/') 462 | log.info('- get / -------------------------------------') 463 | if not root: log.info('missed /') 464 | else: log.info('%s %d'%(root.name,root.inode)) 465 | log.info('---------------------------------------------') 466 | 467 | sfile = FS.get('/apples') 468 | log.info('- get /apples -------------------------------') 469 | if not sfile: log.info('missed /apples') 470 | else: log.info('%s %d'%(sfile.name,sfile.inode)) 471 | log.info('---------------------------------------------') 472 | 473 | sub = FS.get('/l1') 474 | log.info('- get /l1 -----------------------------------') 475 | if not sub: log.info('missed /l1') 476 | else: log.info('%s %d'%(sub.name,sub.inode)) 477 | log.info('---------------------------------------------') 478 | 479 | sfile = FS.get('/l1/banana') 480 | log.info('- get /l1/banana ----------------------------') 481 | if not sfile: log.info('missed /l1/banana') 482 | else: log.info('%s %d'%(sfile.name,sfile.inode)) 483 | log.info('---------------------------------------------') 484 | 485 | if __name__ == '__main__': 486 | FS = None 487 | try: 488 | ping_reporter.start_log(log,logging.DEBUG) 489 | server = ping.select_server(log) 490 | FS = PingFS(server) 491 | init_fs(FS) 492 | test_fs(FS) 493 | 494 | except KeyboardInterrupt: 495 | print "Keyboard Interrupt" 496 | except Exception: 497 | print 'General Exception' 498 | from traceback import print_exc 499 | print_exc() 500 | finally: 501 | if FS: FS.stop() 502 | sys.exit(1) 503 | --------------------------------------------------------------------------------