├── conf
    └── gmailfs.conf
└── gmailfs.py


/conf/gmailfs.conf:
--------------------------------------------------------------------------------
 1 | # this goes in /etc/gmailfs/gmailfs.conf
 2 | [connection]
 3 | # The proxy URL
 4 | #proxy = http://user:pass@proxyhost:port
 5 | # or just 
 6 | #proxy = http://proxyhost:port
 7 | 
 8 | # The number or retries for the proxy connection.
 9 | #retries = 3
10 | 
11 | [account]
12 | username = gmailfsuser12322634@gmail.com
13 | password = s33kr1t
14 | 
15 | [filesystem]
16 | fsname = linux_fs_4
17 | 
18 | [references]
19 | # reference = filesystem:username:password
20 | 
21 | [logs]
22 | # Change this to DEBUG for verbose output (useful for debugging)
23 | level = INFO
24 | 
25 | # if you'd like logs to go to stdout, comment out this variable.
26 | # For logging to, say, stderr, use /dev/stderr of your system's
27 | # equivalent for it
28 | logfile = ~/gmailfs.log
29 | 


--------------------------------------------------------------------------------
/gmailfs.py:
--------------------------------------------------------------------------------
   1 | #!/usr/bin/python
   2 | #@+leo-ver=4
   3 | #@+node:@file gmailfs.py
   4 | #@@first
   5 | #
   6 | #    Copyright (C) 2004  Richard Jones  <richard followed by funny at sign then jones then a dot then name>
   7 | #    Copyright (C) 2010  Dave Hansen <dave@sr71.net>
   8 | #
   9 | #    GmailFS - Gmail Filesystem Version 0.8.6
  10 | #    This program can be distributed under the terms of the GNU GPL.
  11 | #    See the file COPYING.
  12 | #
  13 | # TODO:
  14 | # Problem: a simple write ends up costing at least 3 server writes:
  15 | # 	1. create directory entry
  16 | # 	2. create inode
  17 | # 	3. create first data block
  18 | # It would be greate if files below a certain size (say 64k or something)
  19 | # could be inlined and just stuck as an attachment inside the inode.
  20 | # It should not be too big or else it will end up making things like
  21 | # stat() or getattr() much more expensive
  22 | #
  23 | # It would also be nice to be able to defer actual inode creation for
  24 | # a time.  dirents are going to be harder because we look them up more,
  25 | # but inodes should be easier to keep consistent
  26 | #
  27 | # Wrap all of the imap access functions up better so that we
  28 | # can catch the places to invalidate the caches better.
  29 | #
  30 | # Are there any other options for storing messages than in base64-encoded
  31 | # attachments?  I'm worried about the waste of space and bandwidth.  It
  32 | # appears to be about a 30% penalty.
  33 | #
  34 | # Be more selective about clearing the rsp cache.  It is a bit heavy-handed
  35 | # right now.  Do we really even need the rsp cache that much?  We do our own
  36 | # caching for blocks and inodes.  I guess it helps for constructing readdir
  37 | # responses.
  38 | #
  39 | # CATENATE
  40 | # See if anybody supports this: http://www.faqs.org/rfcs/rfc4469.html
  41 | # It would be wonderful when only writing parts of small files, or even
  42 | # when updating inodes.
  43 | #
  44 | # MUTIAPPEND
  45 | # With this: http://www.faqs.org/rfcs/rfc3502.html
  46 | # we could keep track of modified inodes and submit them in batches back
  47 | # to the server
  48 | #
  49 | # Could support "mount -o ro" or "mount -o remount,ro" with a read-only
  50 | # selection of the target mailbox
  51 | #
  52 | # There some tangling up here of inodes only having a single path
  53 | #
  54 | 
  55 | """
  56 | GmailFS provides a filesystem using a Google Gmail account as its storage medium
  57 | """
  58 | 
  59 | #@+others
  60 | #@+node:imports
  61 | 
  62 | import pprint
  63 | 
  64 | try:
  65 | 	import fuse
  66 | except ImportError, e:
  67 | 	print e
  68 | 	print "Are you sure you sure fuse is built into the kernel or loaded as a module?"
  69 | 	print "In a linux shell type \"lsmod | grep fuse\" to find out."
  70 | 
  71 | import imaplib
  72 | import email
  73 | import random
  74 | from email import encoders
  75 | from email.mime.multipart import MIMEMultipart
  76 | from email.MIMEText import MIMEText
  77 | from email.mime.base import MIMEBase
  78 | 
  79 | 
  80 | import Queue
  81 | from fuse import Fuse
  82 | import os
  83 | from threading import Thread
  84 | import threading
  85 | import thread
  86 | from errno import * # NOTE: wildcard star imports considered evil, namespace pollution
  87 | from stat import *
  88 | from os.path import abspath, expanduser, isfile
  89 | 
  90 | fuse.fuse_python_api = (0, 2)
  91 | 
  92 | 
  93 | import thread
  94 | import quopri
  95 | 
  96 | import sys,traceback,re,string,time,tempfile,array,logging,logging.handlers
  97 | 
  98 | #@-node:imports
  99 | 
 100 | # Globals
 101 | DefaultUsername = 'defaultUser'
 102 | DefaultPassword = 'defaultPassword'
 103 | DefaultFsname = 'gmailfs'
 104 | References={}
 105 | 
 106 | IMAPBlockSize = 1024
 107 | 
 108 | # this isn't used yet
 109 | InlineInodeMax = 32 * 1024
 110 | 
 111 | # I tried 64MB for this, but the base64-encoded
 112 | # blocks end up about 90MB per message, which is
 113 | # a bit too much, and gmail rejects them.
 114 | DefaultBlockSize = 512 * 1024
 115 | 
 116 | # How many blocks can we cache at once
 117 | BlockCacheSize = 100
 118 | 
 119 | SystemConfigFile = "/etc/gmailfs/gmailfs.conf"
 120 | UserConfigFile = abspath(expanduser("~/.gmailfs.conf"))
 121 | 
 122 | GMAILFS_VERSION = '5'
 123 | 
 124 | PathStartDelim  = '__a__'
 125 | PathEndDelim    = '__b__'
 126 | FileStartDelim  = '__c__'
 127 | FileEndDelim    = '__d__'
 128 | LinkStartDelim  = '__e__'
 129 | LinkEndDelim    = '__f__'
 130 | MagicStartDelim = '__g__'
 131 | MagicEndDelim   = '__h__'
 132 | InodeSubjectPrefix = 'inode_msg'
 133 | DirentSubjectPrefix = 'dirent_msg'
 134 | 
 135 | InodeTag ='i'
 136 | DevTag = 'd'
 137 | NumberLinksTag = 'k'
 138 | FsNameTag = 'q'
 139 | ModeTag = 'e'
 140 | UidTag = 'u'
 141 | GidTag = 'g'
 142 | SizeTag = 's'
 143 | AtimeTag = 'a'
 144 | MtimeTag = 'm'
 145 | CtimeTag = 'c'
 146 | BSizeTag = 'z'
 147 | VersionTag = 'v'
 148 | SymlinkTag = 'l'
 149 | 
 150 | RefInodeTag = 'r'
 151 | FileNameTag = 'n'
 152 | PathNameTag = 'p'
 153 | 
 154 | NumberQueryRetries = 1
 155 | 
 156 | regexObjectTrailingMB = re.compile(r'\s?MB$')
 157 | 
 158 | rsp_cache_hits = 0
 159 | rsp_cache_misses = 0
 160 | rsp_cache = {}
 161 | 
 162 | debug = 1
 163 | if debug >= 3:
 164 | 	imaplib.Debug = 3
 165 | #imaplib.Debug = 4
 166 | 
 167 | writeout_threads = {}
 168 | def abort():
 169 | 	global do_writeout
 170 | 	do_writeout = 0
 171 | 	#for t in writeout_threads:
 172 | 	#	print "abort joining thread..."
 173 | 	#	t.join()
 174 | 	#	print "done joining thread"
 175 | 	exit(0)
 176 | 
 177 | sem_msg = {}
 178 | 
 179 | def semget(sem):
 180 | 	tries = 0
 181 | 	while not sem.acquire(0):
 182 | 		tries = tries + 1
 183 | 		time.sleep(1)
 184 | 		if tries % 60 == 0:
 185 | 			print("[%d] hung on lock for %d seconds (holder: %s)" % (thread.get_ident(), tries, sem_msg[sem]))
 186 | 			traceback.print_stack()
 187 | 	if tries >= 60:
 188 | 		print("[%d] unhung on lock after %d seconds (last holder: %s)" % (thread.get_ident(), tries, sem_msg[sem]))
 189 | 	sem_msg[sem] = "acquired semget"
 190 | 	return "OK"
 191 | 
 192 | def log_error(str):
 193 | 	log.debug(str)
 194 | 	log.error(str)
 195 | 	sys.stdout.write(str+"\n")
 196 | 	sys.stderr.write(str+"\n")
 197 | 	return
 198 | 
 199 | def log_debug(str):
 200 | 	log_debug3(str)
 201 | 	#str += "\n"
 202 | 	#sys.stderr.write(str)
 203 | 	return
 204 | 
 205 | def log_entry(str):
 206 | 	#print str
 207 | 	log_debug1(str)
 208 | 
 209 | def am_lead_thread():
 210 | 	if writeout_threads.has_key(thread.get_ident()):
 211 | 		return 0
 212 | 	return 1
 213 | 
 214 | def log_debug1(str):
 215 | 	log_info(str)
 216 | 	#str += "\n"
 217 | 	#sys.stderr.write(str)
 218 | 	return
 219 | 
 220 | def log_debug2(str):
 221 | 	if debug >= 2:
 222 | 		log_info(str)
 223 | 	return
 224 | 
 225 | def log_debug3(str):
 226 | 	if debug >= 3:
 227 | 		log_info(str)
 228 | 	return
 229 | 
 230 | def log_debug4(str):
 231 | 	if debug >= 4:
 232 | 		log_info(str)
 233 | 	return
 234 | 
 235 | def log_imap(str):
 236 | 	log_debug2("IMAP: " + str)
 237 | 
 238 | def log_imap2(str):
 239 | 	log_debug3("IMAP: " + str)
 240 | 
 241 | def log_info(s):
 242 | 	if not am_lead_thread():
 243 | 		return
 244 | 	log.info("[%.2f] %s" % (time.time(), s))
 245 | 	#print str
 246 | 	#str += "\n"
 247 | 	#sys.stderr.write(str)
 248 | 	return
 249 | 
 250 | def log_warning(str):
 251 | 	log.warning(str)
 252 | 	#str += "\n"
 253 | 	#sys.stderr.write(str)
 254 | 	return
 255 | 
 256 | def parse_path(path):
 257 | 	# should we check that there's always a / in the path??
 258 | 	ind = string.rindex(path, '/')
 259 | 	parent_dir = path[:ind]
 260 |         filename = path[ind+1:]
 261 | 	if len(parent_dir) == 0:
 262 | 		parent_dir = "/"
 263 | 	return parent_dir, filename
 264 | 
 265 | 
 266 | def msg_add_payload(msg, payload, filename=None):
 267 | 	attach_part = MIMEBase('file', 'attach')
 268 | 	attach_part.set_payload(payload)
 269 | 	if filename != None:
 270 | 		attach_part.add_header('Content-Disposition', 'attachment; filename="%s"' % filename)
 271 | 	encoders.encode_base64(attach_part)
 272 | 	msg.attach(attach_part)
 273 | 
 274 | # This probably doesn't need to be handed the fsNameVar
 275 | # and the username
 276 | def mkmsg(subject, preamble, attach = ""):
 277 | 	global username
 278 | 	global fsNameVar
 279 | 	msg = MIMEMultipart()
 280 | 	log_debug2("mkmsg('%s', '%s', '%s', '%s',...)" % (username, fsNameVar, subject, preamble))
 281 | 	msg['Subject'] = subject
 282 | 	msg['To'] = username
 283 | 	msg['From'] = username
 284 | 	msg.preamble = preamble
 285 | 	if len(attach):
 286 | 		log_debug("attaching %d byte file contents" % len(attach))
 287 | 		msg_add_payload(msg, attach)
 288 | 	log_debug3("mkmsg() after subject: '%s'" % (msg['Subject']))
 289 | 	msg.uid = -1
 290 | 	return msg
 291 | 
 292 | imap_times = {}
 293 | imap_times_last_print = 0
 294 | def log_imap_time(cmd, start_time):
 295 | 	global imap_times
 296 | 	global imap_times_last_print
 297 | 	if not imap_times.has_key(cmd):
 298 | 		imap_times[cmd] = 0.0
 299 | 
 300 | 	now = time.time()
 301 | 	end_time = now
 302 | 	duration = end_time - start_time
 303 | 	imap_times[cmd] += duration
 304 | 	imap_times_print()
 305 | 
 306 | def imap_times_print(force=0):
 307 | 	global imap_times
 308 | 	global imap_times_last_print
 309 | 	now = time.time()
 310 | 	if force or (now - imap_times_last_print > 10):
 311 | 		for key, total in imap_times.items():
 312 | 			log_info("imap_times[%s]: %d" % (key, total))
 313 | 		imap_times_last_print = now
 314 | 
 315 | # this is intended to be a drop-in for imap.uid(), while
 316 | # also allowing the imap object to reconnect in the event
 317 | # of failures
 318 | #
 319 | # This hopefully just means that one of the connections
 320 | # died.  This will try to reestablish it.
 321 | def imap_uid(imap, cmd, arg1, arg2 = None, arg3 = None, arg4 = None):
 322 | 	tries = 3
 323 | 	ret = None
 324 | 	while ret == None:
 325 | 		tries = tries - 1
 326 | 		try:
 327 | 		        ret = imap.uid(cmd, arg1, arg2, arg3)
 328 | 		except Exception, e:
 329 | 			log_error("imap.uid() error: %s (tries left: %d)" % (str(e), tries))
 330 | 			imap.fs.kick_imap(imap)
 331 | 			if tries <= 0:
 332 | 				raise
 333 | 		except:
 334 | 			log_error("imap.uid() unknown error: (tries left: %d)" % (tries))
 335 | 			imap.fs.kick_imap(imap)
 336 | 			if tries <= 0:
 337 | 				raise
 338 | 	return ret
 339 | 
 340 | def __imap_append(imap, fsNameVar, flags, now, msg):
 341 | 	tries = 3
 342 | 	rsp = None
 343 | 	data = None
 344 | 	while rsp == None:
 345 | 		tries = tries - 1
 346 | 		try:
 347 | 		        rsp, data = imap.append(fsNameVar, flags, now, msg)
 348 | 			log_debug2("__imap_append() try: %d rsp: '%s'" % (tries, rsp))
 349 | 			if rsp == "NO":
 350 | 				time.sleep(1)
 351 | 				rsp = None
 352 | 				continue
 353 | 		except RuntimeError, e:
 354 | 			log_error("imap.append() error: %s" % (str(e)))
 355 | 			imap.fs.kick_imap(imap)
 356 | 			if tries <= 0:
 357 | 				raise
 358 | 	return rsp, data
 359 | 
 360 | def imap_getquotaroot(imap, fsNameVar):
 361 | 	tries = 2
 362 | 	ret = None
 363 | 	while ret == None:
 364 | 		try:
 365 | 		        ret = imap.getquotaroot(fsNameVar)
 366 | 		except RuntimeError, e:
 367 | 			log_error("imap.getquotaroot() error: %s" % (str(e)))
 368 | 			imap.fs.kick_imap(imap)
 369 | 			if tries <= 0:
 370 | 				raise
 371 | 		tries = tries - 1
 372 | 	return ret
 373 | 
 374 | # The IMAP uid commands can take multiple uids and return
 375 | # multiple results
 376 | #
 377 | # uid here is intended to be an array of uids, and this
 378 | # returns a dictionary of results indexed by uid
 379 | #
 380 | # does python have a ... operator like c preprocessor?
 381 | def uid_cmd(imap, cmd, uids, arg1, arg2 = None, arg3 = None):
 382 | 	semget(imap.lock)
 383 | 	ret = __uid_cmd(imap, cmd, uids, arg1, arg2, arg3)
 384 | 	imap.lock.release()
 385 | 	return ret
 386 | 
 387 | def __uid_cmd(imap, cmd, uids, arg1, arg2 = None, arg3 = None):
 388 | 	uids_str = string.join(uids, ",")
 389 | 	start = time.time()
 390 | 	log_info("__uid_cmd(%s,...) %d uids" % (cmd, len(uids)))
 391 |     	rsp, rsp_data = imap_uid(imap, cmd, uids_str, arg1, arg2, arg3)
 392 | 	log_imap_time(cmd, start);
 393 | 	log_info("__uid_cmd(%s, [%s]) ret: '%s'" % (cmd, uids_str, rsp))
 394 | 	if rsp != "OK":
 395 | 		log_error("IMAP uid cmd (%s, [%s]) error: %s" % (cmd, uids_str, rsp))
 396 | 		return None
 397 | 	ret = {}
 398 | 	uid_index = 0
 399 | 	for one_rsp_data in rsp_data:
 400 | 		log_debug3("rsp_data[%d]: ->%s<-" % (uid_index, one_rsp_data))
 401 | 		uid_index += 1
 402 | 	uid_index = 0
 403 | 	for rsp_nr in range(len(rsp_data)):
 404 | 		data = rsp_data[rsp_nr]
 405 | 		# I don't know if this is expected or
 406 | 		# not, but every other response is just
 407 | 		# a plain ')' char.  Skip them
 408 | 		log_debug3("about to lookup uids[%d] data class: '%s'" % (uid_index, data.__class__.__name__))
 409 | 		if isinstance(data, tuple):
 410 | 			log_debug4("is tuple")
 411 | 			for tval in data:
 412 | 				log_debug4("tval: ->%s<- class: '%s'" % (str(tval), tval.__class__.__name__))
 413 | 		if isinstance(data, str):
 414 | 			continue
 415 | 		uid = uids[uid_index]
 416 | 		uid_index += 1
 417 | 		if data == None:
 418 | 			log_info("uid_cmd(%s) got strange result %s/%s" %
 419 | 					(cmd, rsp_nr, range(len(rsp_data))))
 420 | 			continue
 421 | 		desc = data[0]
 422 | 		result = data[1]
 423 | 		ret[uid] = result
 424 | 	return ret
 425 | 
 426 | def clear_rsp_cache():
 427 | 	global rsp_cache
 428 | 	log_debug2("clearing rsp cache with %d entries" % (len(rsp_cache)))
 429 | 	rsp_cache = {}
 430 | 
 431 | def imap_trash_uids(imap, raw_uids):
 432 | 	clear_rsp_cache()
 433 | 	checked_uids = []
 434 | 	# there have been a few cases where a -1
 435 | 	# creeps in here because we're trying to
 436 | 	# delete a message that has not yet been
 437 | 	# uploaded to the server.  Filter those
 438 | 	# out.
 439 | 	for uid in raw_uids:
 440 | 		if int(uid) <= 0:
 441 | 			continue
 442 | 		checked_uids.append(uid)
 443 | 
 444 | 	if len(checked_uids) == 0:
 445 | 		return
 446 | 	log_imap("imap_trash_uids(%s)" % (string.join(checked_uids,",")))
 447 | 	ret = uid_cmd(imap, "STORE", checked_uids, '+FLAGS', '\\Deleted')
 448 | 	global msg_cache
 449 | 	for uid in checked_uids:
 450 | 		try:
 451 | 			del msg_cache[uid]
 452 | 		except:
 453 | 			foo = 1
 454 | 			# this is OK because the msg may neve have
 455 | 			# been cached
 456 | 	return ret
 457 | 
 458 | def imap_trash_msg(imap, msg):
 459 | 	if msg.uid <= 0:
 460 | 		return
 461 | 	imap_trash_uids(imap, [str(msg.uid)])
 462 | 
 463 | def imap_append(info, imap, msg):
 464 | 	#gmsg = libgmail.GmailComposedMessage(username, subject, body)
 465 | 	log_imap("imap_append(%s)" % (info))
 466 | 	log_debug2("append Subject: ->%s<-" % (msg['Subject']))
 467 | 	log_debug3("entire message: ->%s<-" % str(msg))
 468 | 
 469 | 	now = imaplib.Time2Internaldate(time.time())
 470 | 	clear_rsp_cache()
 471 | 	start = time.time()
 472 | 	semget(imap.lock)
 473 |     	rsp, data = __imap_append(imap, fsNameVar, "", now, str(msg))
 474 | 	imap.lock.release()
 475 | 	log_imap_time("APPEND", start);
 476 | 	log_imap2("append for '%s': rsp,data: '%s' '%s'" % (info, rsp, data))
 477 | 	if rsp != "OK":
 478 | 		return -1
 479 | 	# data looks like this: '['[APPENDUID 631933985 286] (Success)']'
 480 | 	msgid = int((data[0].split()[2]).replace("]",""))
 481 | 	msg.uid = msgid
 482 | 	log_debug("imap msgid: '%d'" % msgid)
 483 | 	return msgid
 484 | 
 485 | def _addLoggingHandlerHelper(handler):
 486 |     """ Sets our default formatter on the log handler before adding it to
 487 |         the log object. """
 488 |     handler.setFormatter(defaultLogFormatter)
 489 |     log.addHandler(handler)
 490 | 
 491 | def GmailConfig(fname):
 492 |     import ConfigParser
 493 |     cp = ConfigParser.ConfigParser()
 494 |     global References
 495 |     global DefaultUsername, DefaultPassword, DefaultFsname
 496 |     global NumberQueryRetries
 497 |     if cp.read(fname) == []:
 498 |       log_warning("Unable to read configuration file: " + str(fname))
 499 |       return
 500 | 
 501 |     sections = cp.sections()
 502 |     if "account" in sections:
 503 |       options = cp.options("account")
 504 |       if "username" in options:
 505 |           DefaultUsername = cp.get("account", "username")
 506 |       if "password" in options:
 507 |           DefaultPassword = cp.get("account", "password")
 508 |     else:
 509 |       log.error("Unable to find GMail account configuration")
 510 | 
 511 |     if "filesystem" in sections:
 512 |       options = cp.options("filesystem")
 513 |       if "fsname" in options:
 514 |           DefaultFsname = cp.get("filesystem", "fsname")
 515 |     else:
 516 |       log_warning("Using default file system (Dangerous!)")
 517 | 
 518 |     if "logs" in sections:
 519 |       options = cp.options("logs")
 520 |       if "level" in options:
 521 |         level = cp.get("logs", "level")
 522 |         log.setLevel(logging._levelNames[level])
 523 |       if "logfile" in options:
 524 |         logfile = abspath(expanduser(cp.get("logs", "logfile")))
 525 | 	log.removeHandler(defaultLoggingHandler)
 526 |         _addLoggingHandlerHelper(logging.handlers.RotatingFileHandler(logfile, "a", 5242880, 3))
 527 | 
 528 |     if "references" in sections:
 529 |       options = cp.options("references")
 530 |       for option in options:
 531 |           record = cp.get("references",option)
 532 |           fields = record.split(':')
 533 |           if len(fields)<1 or len(fields)>3:
 534 |               log_warning("Invalid reference '%s' in configuration." % (record))
 535 |               continue
 536 |           reference = reference_class(*fields)
 537 |           References[option] = reference
 538 | 
 539 | do_writeout = 1
 540 | #@+node:mythread
 541 | class testthread(Thread):
 542 | 	def __init__ (self, fs, nr):
 543 | 		Thread.__init__(self)
 544 | 		self.fs = fs
 545 | 		self.nr = nr
 546 | 
 547 | 	def write_out_object(self):
 548 | 		try:
 549 | 			# block, and timeout after 1 second
 550 | 			object = self.fs.dirty_objects.get(1, 1)
 551 | 		except:
 552 | 			# effectively success if we timeout
 553 | 			return 0
 554 | 		# we do not want to sit here sleeping on objects
 555 | 		# so if we can not get the lock, move on to another
 556 | 		# object
 557 | 		got_lock = object.writeout_lock.acquire(0)
 558 | 		if not got_lock:
 559 | 			self.fs.dirty_objects.put(object)
 560 | 			return -1
 561 | 		sem_msg[object.writeout_lock] = "acquired write_out_object()"
 562 | 		reason = Dirtyable.dirty_reason(object)
 563 |     		start = time.time()
 564 | 		ret = write_out_nolock(object, "bdflushd")
 565 |     		end = time.time()
 566 | 		object.writeout_lock.release()
 567 | 		sem_msg[object.writeout_lock] += " released write_out_object()"
 568 | 		size = self.fs.dirty_objects.qsize()
 569 | 		# 0 means it got written out
 570 | 		# 1 means it was not dirty
 571 | 		took = end - start
 572 | 		msg =  "[%d] (%2d sec), %%s %s because '%s' %d left" % (self.nr, took, object.to_str(), reason, size)
 573 | 		if ret == 0:
 574 | 			print(msg % ("wrote out"));
 575 | 		else:
 576 | 			print(msg % ("did not write"));
 577 | 		return 1
 578 | 
 579 | 	def run_writeout(self):
 580 | 		tries = 5
 581 | 		for try_nr in range(tries):
 582 | 			writeout_threads[thread.get_ident()] = "running"
 583 | 			ret = self.write_out_object()
 584 | 			#rint("writeout ret: '%s'" % (ret))
 585 | 			if ret == 0:
 586 | 				writeout_threads[thread.get_ident()] = "idle"
 587 | 				msg = "["
 588 | 				for t in range(self.fs.nr_imap_threads):
 589 | 					if t >= 1:
 590 | 						msg += " "
 591 | 					if writeout_threads[thread.get_ident()] == "idle":
 592 | 						msg += str(t)
 593 | 					else:
 594 | 						msg += " "
 595 | 				msg += "] idle\r"
 596 | 				sys.stderr.write(msg)
 597 | 				sys.stderr.flush()
 598 | 			if ret >= 0:
 599 | 				break
 600 | 			# this will happen when there are
 601 | 			# objects in the queue for which
 602 | 			# we can not get the lock.  Do
 603 | 			# not spin, sleep instead
 604 | 			if try_nr < tries-1:
 605 | 				continue
 606 | 
 607 | 			time.sleep(1)
 608 | 
 609 | 	def run(self):
 610 | 		global do_writeout
 611 | 		writeout_threads[thread.get_ident()] = 1
 612 | 		log_debug1("mythread: started pid: %d" % (os.getpid()))
 613 | 		print "connected[%d]" % (self.nr)
 614 | 		log_debug1("connected[%d]" % (self.nr))
 615 | 		while do_writeout:
 616 | 			self.run_writeout()
 617 | 	       	print "thread[%d] done" % (self.nr)
 618 | 
 619 |     #@-node:mythread
 620 | 
 621 | 
 622 | class reference_class:
 623 |     def __init__(self,fsname,username=None,password=None):
 624 |       self.fsname = fsname
 625 |       if username is None or username == '':
 626 |           self.username = DefaultUsername
 627 |       else:
 628 |           self.username = username
 629 |       if password is None or password == '':
 630 |           self.password = DefaultPassword
 631 |       else:
 632 |           self.password = password
 633 | 
 634 | # This ensures backwards compatability where
 635 | # old filesystems were stored with 7bit encodings
 636 | # but new ones are all quoted printable
 637 | def fixQuotedPrintable(body):
 638 |     # first remove headers
 639 |     newline = body.find("\r\n\r\n")
 640 |     if newline >= 0:
 641 | 	body = body[newline:]
 642 |     fixed = body
 643 |     if re.search("Content-Transfer-Encoding: quoted",body):
 644 |         fixed = quopri.decodestring(body)
 645 |     # Map unicode
 646 |     return fixed.replace('\u003d','=')
 647 | 
 648 | def psub(s):
 649 |     if len(s) == 0:
 650 | 	return "";
 651 |     return "SUBJECT \""+s+"\""
 652 | 
 653 | def _getMsguidsByQuery(about, imap, queries, or_query = 0):
 654 |     or_str = ""
 655 |     if or_query:
 656 | 	or_str = " OR"
 657 |     fsq = (str(FsNameTag + "=" + MagicStartDelim + fsNameVar + MagicEndDelim))
 658 |     # this is *REALLY* sensitive, at least on gmail
 659 |     # Don't put any extra space in it anywhere, or you
 660 |     # will be sorry
 661 |     #  53:12.12 > MGLK6 SEARCH (SUBJECT "foo=bar" SUBJECT "bar=__fo__o__")
 662 |     queryString  = '(SUBJECT "%s"' % (fsq)
 663 |     last_q = queries.pop()
 664 |     for q in queries:
 665 |     	queryString += or_str + ' SUBJECT "%s"' % (q)
 666 |     queryString += ' SUBJECT "%s")' % last_q
 667 | 
 668 |     global rsp_cache
 669 |     global rsp_cache_hits
 670 |     global rsp_cache_misses
 671 |     if rsp_cache_hits+rsp_cache_misses % 10 == 0:
 672 | 	    log_info("rsp_cache (size: %d hits: %d misses: %d)" % (len(rsp_cache), rsp_cache_hits, rsp_cache_misses))
 673 |     if rsp_cache.has_key(queryString):
 674 | 	    rsp_cache_hits += 1
 675 | 	    return rsp_cache[queryString]
 676 |     else:
 677 | 	    rsp_cache_misses += 1
 678 | 
 679 |     # make sure mailbox is selected
 680 |     log_imap("SEARCH query: '"+queryString+"'")
 681 |     start = time.time()
 682 |     semget(imap.lock)
 683 |     try:
 684 |         resp, msgids_list = imap_uid(imap, "SEARCH", None, queryString)
 685 |     except:
 686 | 	log_error("IMAP error on SEARCH")
 687 | 	log_error("queryString: ->%s<-" % (queryString))
 688 | 	print "\nIMAP exception ", sys.exc_info()[0]
 689 |     	exit(-1)
 690 |     finally:
 691 |         imap.lock.release()
 692 |     log_imap_time("SEARCH", start);
 693 |     msgids = msgids_list[0].split(" ")
 694 |     log_imap2("search resp: %s msgids len: %d" % (resp, len(msgids)))
 695 |     ret = []
 696 |     for msgid in msgids:
 697 |         log_debug2("IMAP search result msg_uid: '%s'" % str(msgid))
 698 | 	if len(str(msgid)) > 0:
 699 |             ret = msgids
 700 | 	    break
 701 |     if len(rsp_cache) > 1000:
 702 | 	clear_rsp_cache()
 703 |     rsp_cache[queryString] = ret
 704 |     return ret
 705 | 
 706 | def getSingleMsguidByQuery(imap, q):
 707 |         msgids = _getMsguidsByQuery("fillme1", imap, q)
 708 | 	nr = len(msgids)
 709 |         if nr != 1:
 710 | 	  qstr = string.join(q, " ")
 711 | 	  # this is debug because it's normal to have non-existent files
 712 |           log_debug2("could not find messages for query: '%s' (found %d)" % (qstr, nr))
 713 |           return -1;
 714 |  	log_debug2("getSingleMsguidByQuery('%s') ret: '%s' nr: %d" % (string.join(q," "), msgids[0], nr))
 715 | 	return int(msgids[0])
 716 | 
 717 | def __fetch_full_messages(imap, msgids):
 718 | 	if msgids == None or len(msgids) == 0:
 719 | 		return None
 720 | 	data = __uid_cmd(imap, "FETCH", msgids, '(RFC822)')
 721 | 	if data == None:
 722 | 		return None
 723 | 	log_imap("fetch(msgids=%s): got %d messages" % (string.join(msgids, ","), len(data)))
 724 | 	#log_debug2("fetch msgid: '%s' resp: '%s' data: %d bytes" % (str(msgid), resp, len(data)))
 725 | 	ret = {}
 726 | 	for uid, raw_str in data.items():
 727 | 		msg = email.message_from_string(raw_str)
 728 | 		msg.uid = uid
 729 | 		ret[str(uid)] = msg
 730 |         return ret
 731 | 
 732 | msg_cache = {}
 733 | def fetch_full_messages(imap, msgids):
 734 | 	global msg_cache
 735 | 	ret = {}
 736 | 	fetch_msgids = []
 737 | 	# if we do not hold the lock over this entire
 738 | 	# sequence, we can race and fetch messages
 739 | 	# twice.  It doesn't hurt, but it is inefficient
 740 | 	hits = 0
 741 | 	misses = 0
 742 | 	semget(imap.lock)
 743 | 	for msgid in msgids:
 744 | 		if msgid in msg_cache:
 745 | 			ret[msgid] = msg_cache[msgid]
 746 | 			hits += 1
 747 | 		else:
 748 | 			fetch_msgids.append(msgid)
 749 | 			misses += 1
 750 | 	log_debug3("fetch_full_messages() trying to fetch %d msgs" % (len(fetch_msgids)))
 751 | 	fetched = None
 752 | 	if len(fetch_msgids):
 753 | 		fetched = __fetch_full_messages(imap, fetch_msgids)
 754 | 	if fetched != None:
 755 | 		ret.update(fetched)
 756 | 		for uid, msg in fetched.items():
 757 | 			if msg_cache.has_key(uid):
 758 | 				print "uh oh, double-fetched uid: '%s'" % (uid)
 759 | 			log_debug2("filled msg_cache[%s]" % (str(uid)))
 760 | 			msg_cache[uid] = msg
 761 | 	if len(msg_cache) > 1000:
 762 | 		log_info("flushed message cache")
 763 | 		msg_cache = {}
 764 | 	imap.lock.release()
 765 | 	log_debug3("fetch_full_messages() hits: %d misses: %d" % (hits, misses))
 766 | 	return ret
 767 | 
 768 | def fetch_full_message(imap, msgid):
 769 | 	resp = fetch_full_messages(imap, [str(msgid)])
 770 | 	if resp == None:
 771 | 		return None
 772 | 	return resp[str(msgid)]
 773 | 
 774 | def getSingleMessageByQuery(desc, imap, q):
 775 | 	log_debug2("getSingleMessageByQuery(%s)" % (desc))
 776 | 	msgid = getSingleMsguidByQuery(imap, q)
 777 | 	if msgid == -1:
 778 | 		log_debug2("getSingleMessageByQuery() msgid: %s" % (str(msgid)))
 779 | 		return None
 780 | 	return fetch_full_message(imap, msgid)
 781 | 
 782 | def _pathSeparatorEncode(path):
 783 |     s1 = re.sub("/","__fs__",path)
 784 |     s2 = re.sub("-","__mi__",s1)
 785 |     return re.sub("\+","__pl__",s2)
 786 | 
 787 | def _pathSeparatorDecode(path):
 788 |     s1 = re.sub("__fs__","/",path)
 789 |     s2 = re.sub("__mi__","-",s1)
 790 |     return re.sub("__pl__","+",s2)
 791 | 
 792 | 
 793 | def _logException(msg):
 794 |     traceback.print_exc(file=sys.stderr)
 795 |     log.exception(msg)
 796 |     log.info(msg)
 797 | 
 798 | # Maybe I'm retarded, but I couldn't get this to work
 799 | # with python inheritance.  Oh, well.
 800 | def write_out_nolock(o, desc):
 801 | 	dirty_token = o.dirty()
 802 | 	if not dirty_token:
 803 | 		log_debug1("object is not dirty (%s), not writing out" % (str(dirty_token)))
 804 | 		print("object is not dirty (token: %s), not writing out" % (str(dirty_token)))
 805 | 		return 1
 806 | 	#clear_msg = "none"
 807 | 	clear_msg = o.clear_dirty(dirty_token)
 808 | 	if   isinstance(o, GmailInode):
 809 | 		ret = o.i_write_out(desc)
 810 | 	elif isinstance(o, GmailDirent):
 811 | 		ret = o.d_write_out(desc)
 812 | 	elif isinstance(o, GmailBlock):
 813 | 		ret = o.b_write_out(desc)
 814 | 	else:
 815 | 		print("unknown dirty object:"+o.to_str())
 816 | 	if ret != 0:
 817 | 		o.mark_dirty("failed writeout");
 818 | 	log_debug1("write_out() finished '%s' (cleared '%s')" % (desc, clear_msg))
 819 | 	return ret
 820 | 
 821 | def write_out(o, desc):
 822 | 	# I was seeing situations where a network error (SSL in this case)
 823 | 	# was raised.  It wasn't handled and the thread died while holding
 824 | 	# this lock.  This should at least make it release the lock before
 825 | 	# dying.
 826 | 	try:
 827 | 		semget(o.writeout_lock)
 828 | 		sem_msg[o.writeout_lock] = "acquired write_out()"
 829 | 		ret = write_out_nolock(o, desc)
 830 | 	finally:
 831 | 		o.writeout_lock.release()
 832 | 		sem_msg[o.writeout_lock] += " released write_out() in exception"
 833 | 	return ret
 834 | 
 835 | class Dirtyable(object):
 836 | 	def __init__(self):
 837 | 		log_debug3("Dirtyable.__init__() '%s'" % (self))
 838 | 		self.dirty_reasons = Queue.Queue(1<<20)
 839 | 		self.dirty_mark = Queue.Queue(1)
 840 | 		self.writeout_lock = thread.allocate_lock()
 841 | 		sem_msg[self.writeout_lock] = "brand spankin new"
 842 | 
 843 | 	def dirty(self):
 844 |     		return self.dirty_reasons.qsize()
 845 | 
 846 | 	def dirty_reason(self):
 847 |     		return "%s (%d more reasons hidden)" % (self.__dirty, self.dirty())
 848 | 
 849 | 	def clear_dirty(self, nr):
 850 | 		msgs = []
 851 | 		log_info("clearing %d dirty reasons" % (nr))
 852 | 		for msg_nr in range(nr):
 853 | 			d_msg = self.dirty_reasons.get_nowait()
 854 | 			log_info("dirty reason[%d]: %s" % (msg_nr, d_msg))
 855 | 			msgs.append(d_msg)
 856 | 		msg = "(%s)" % string.join(msgs, ", ")
 857 | 		# there's a race to do this twice
 858 | 		orig_reason = self.dirty_mark.get_nowait();
 859 | 		log_info("cleared original dirty reason: '%s'" % (orig_reason))
 860 | 		return msg
 861 | 
 862 | 	def mark_dirty(self, desc):
 863 | 		self.__dirty = desc
 864 | 		self.dirty_reasons.put(desc)
 865 | 		try:
 866 | 			self.dirty_mark.put_nowait(desc);
 867 | 			self.fs.dirty_objects.put(self)
 868 | 		except:
 869 | 			log_debug("mark_dirty('%s') skipped global list, already dirty" % (self.to_str()))
 870 | 		log_debug1("mark_dirty('%s') because '%s' (%d reasons)" %
 871 | 				(self.to_str(), desc, self.dirty_reasons.qsize()))
 872 | 
 873 | 	def to_str(self):
 874 | 		return "Dirtyable.to_str()"
 875 | # end class Dirtyable
 876 | 
 877 | #@+node:class GmailDirent
 878 | class GmailDirent(Dirtyable):
 879 | 	def __init__(self, dirent_msg, inode, fs):
 880 |         	Dirtyable.__init__(self)
 881 | 		self.dirent_msg = dirent_msg
 882 | 		self.inode = inode
 883 | 		self.fs = fs
 884 | 
 885 | 	def to_str(self):
 886 | 		return "dirent('%s' ino=%s)" % (self.path(), str(self.inode.ino))
 887 | 
 888 | 	def path(self):
 889 | 		d = self.fs.parse_dirent_msg(self.dirent_msg)
 890 | 		file = _pathSeparatorDecode(d[FileNameTag])
 891 | 		path = _pathSeparatorDecode(d[PathNameTag])
 892 | 		log_debug3("decoded path: '%s' file: '%s'" % (path, file))
 893 | 		log_debug3("subject was: ->%s<-" % (self.dirent_msg['Subject']))
 894 | 		# path doesn't have a trailing slash, but the root
 895 | 		# does have one.  Need to add one when we're dealing
 896 | 		# with the non-root dir
 897 | 		if path != "/":
 898 | 			path += "/"
 899 | 		return ("%s%s" % (path, file))
 900 | 
 901 | 	def d_write_out(self, desc):
 902 | 		log_info("writing out dirent '%s' for '%s' (dirty reason: '%s')"
 903 | 				% (self.path(), desc, Dirtyable.dirty_reason(self)))
 904 | 		imap = self.fs.get_imap()
 905 | 		msgid = imap_append("dirent writeout", imap, self.dirent_msg)
 906 | 		self.fs.put_imap(imap)
 907 | 		if msgid <= 0:
 908 | 		    e = OSError("Could not send mesg in write_out() for: '%s'" % (path))
 909 | 	            e.errno = ENOSPC
 910 |        		    raise e
 911 | 		return 0
 912 | 
 913 | 	def unlink(self):
 914 | 		# FIXME, don't allow directory unlinking when children
 915 | 		log_debug1("unlink path:"+self.path()+" with nlinks:"+str(self.inode.i_nlink))
 916 | 		if self.inode.mode & S_IFDIR:
 917 | 			log_debug("unlinking dir")
 918 | 			# guaranteed not to return any messages to
 919 | 			# trash since there are two links for dirs
 920 | 			self.inode.dec_nlink()
 921 | 		else:
 922 | 			log_debug("unlinking file")
 923 | 
 924 | 		to_trash = self.inode.dec_nlink()
 925 | 		to_trash.append(str(self.dirent_msg.uid))
 926 | 		if len(to_trash):
 927 | 			imap_trash_uids(self.fs.imap, to_trash)
 928 | 		deleted = self.fs.dirent_cache.pop(self.path())
 929 | 		if deleted != None and deleted != self:
 930 | 			log_error("removed wrong dirent from cache")
 931 | 
 932 | 
 933 | #@-node:class GmailDirent
 934 | 
 935 | last_ino = -1
 936 | 
 937 | # using time for ino is a bad idea FIXME
 938 | #
 939 | # This helps, but there's still a theoretical
 940 | # problem if we mount(), write(), unmount()
 941 | # and mount again all within a second.
 942 | #
 943 | # Should we store this persistently in the
 944 | # root inode perhaps?
 945 | #
 946 | def get_ino():
 947 |     global last_ino
 948 |     ret = int(time.time()) << 16
 949 |     if ret <= last_ino:
 950 | 	    ret = last_ino + 1
 951 |     return int(ret)
 952 | 
 953 | #@+node:class GmailInode
 954 | class GmailInode(Dirtyable):
 955 | 
 956 |     """
 957 |     Class used to store gmailfs inode details
 958 |     """
 959 |     #@+node:__init__
 960 |     def __init__(self, inode_msg, fs):
 961 |         Dirtyable.__init__(self)
 962 | 	# We can either make this inode from scratch, or
 963 | 	# use the inode_msg to fill in all these fields
 964 | 	self.fs = fs
 965 | 	self.xattr = {}
 966 | 	self.i_blocks = {}
 967 | 	self.inode_cache_lock = thread.allocate_lock()
 968 | 	# protected by fs.inode_cache_lock
 969 |         self.pinned = 0
 970 |         if inode_msg != None:
 971 |  	    self.inode_msg = inode_msg
 972 |             self.fill_from_inode_msg()
 973 | 	else:
 974 |             self.version = 2
 975 |             self.ino = get_ino()
 976 |             self.mode = 0
 977 |             self.dev = 0
 978 |             self.i_nlink = 0
 979 |             self.uid = 0
 980 |             self.gid = 0
 981 |             self.size = 0
 982 |             self.atime = 0
 983 |             self.mtime = 0
 984 |             self.ctime = 0
 985 |             self.symlink_tgt = ""
 986 |             self.block_size = DefaultBlockSize
 987 | 	    # there are a couple of spots that depend
 988 | 	    # on having one of these around
 989 | 	    self.inode_msg = self.mk_inode_msg()
 990 |     #@-node:__init__
 991 |     def to_str(self):
 992 | 	    return "inode(%s)" % (str(self.ino))
 993 | 
 994 |     def mark_dirty(self, desc):
 995 | 	log_debug2("inode mark_dirty(%s) size: '%s'" % (desc, str(self.size)))
 996 |         self.mtime = int(time.time())
 997 | 	Dirtyable.mark_dirty(self, desc)
 998 | 
 999 |     def i_write_out(self, desc):
1000 | 	log_debug2("i_write_out() self: '%s'" % (self))
1001 | 	log_info("writing out inode for '%s' (dirty reason: '%s')" % (desc, Dirtyable.dirty_reason(self)))
1002 | 	for attr in self.xattr:
1003 | 		value = self.xattr[attr]
1004 | 		payload_name = 'xattr-'+attr
1005 | 		log_debug1("adding xattr payload named '%s': '%s'" % (payload_name, value))
1006 | 		msg_add_payload(msg, value, payload_name)
1007 | 	# remember where this is in case we have to delete it
1008 | 	i_orig_uid = self.inode_msg.uid
1009 | 	# because this wipes it out
1010 | 	self.inode_msg = self.mk_inode_msg()
1011 | 	imap = self.fs.get_imap()
1012 | 	i_msgid = imap_append("inode writeout", imap, self.inode_msg)
1013 | 	self.fs.put_imap(imap)
1014 | 	if i_msgid > 0 and i_orig_uid > 0:
1015 | 		log_debug("trashing old inode uid: %s new is: %s" % (i_orig_uid, i_msgid))
1016 | 		imap_trash_uids(imap, [str(i_orig_uid)])
1017 |     	if i_msgid <= 0:
1018 |             msg = "Unable to write new inode message: '%s'" % (self.inode_msg['Subject'])
1019 |             e = OSError(msg)
1020 | 	    log_error(msg)
1021 |             e.errno = ENOSPC
1022 | 	    abort()
1023 |             raise e
1024 |     	# Uh oh.  Does this properly truncate data blocks that are no
1025 | 	# longer in use?
1026 | 	return 0
1027 | 
1028 |     def fill_xattrs(self):
1029 | 	log_debug3("fill_xattrs()")
1030 | 	for part in self.inode_msg.get_payload():
1031 | 		log_debug3("fill_xattrs() loop")
1032 | 		fname = part.get_filename(None)
1033 | 		log_debug3("fill_xattrs() fname: '%s'" % (str(fname)))
1034 | 		if fname == None:
1035 | 			continue
1036 | 		m = re.match('xattr-(.*)', fname)
1037 | 		if m == None:
1038 | 			continue
1039 | 		xattr_name = m.group(1)
1040 | 		log_debug3("fill_xattrs() xattr_name: '%s'" % (xattr_name))
1041 | 		self.xattr[xattr_name] = part.get_payload(decode=True)
1042 | 
1043 |     def mk_inode_msg(self):
1044 |    	dev = "11"
1045 |         subject = (InodeSubjectPrefix+ " " +
1046 | 	    	   VersionTag     + "=" + GMAILFS_VERSION+ " " +
1047 |                    InodeTag       + "=" + str(self.ino)+ " " +
1048 | 	           DevTag         + "=" + dev + " " +
1049 | 		   NumberLinksTag + "=" + str(self.i_nlink)+ " " +
1050 | 		   FsNameTag      + "=" + MagicStartDelim + fsNameVar +MagicEndDelim +
1051 | 		   "")
1052 |         timeString = str(self.mtime)
1053 | 	bsize = str(DefaultBlockSize)
1054 | 	symlink_str = ""
1055 | 	if self.symlink_tgt != None:
1056 | 		symlink_str = _pathSeparatorEncode(self.symlink_tgt)
1057 |         body = (ModeTag  + "=" + str(self.mode)   + " " +
1058 | 	        UidTag   + "=" + str(os.getuid()) + " " +
1059 | 		GidTag   + "=" + str(os.getgid()) + " " +
1060 | 		SizeTag  + "=" + str(self.size)   + " " +
1061 | 		AtimeTag + "=" + timeString 	  + " " +
1062 | 		MtimeTag + "=" + timeString 	  + " " +
1063 | 		CtimeTag + "=" + timeString 	  + " " +
1064 | 		BSizeTag + "=" + bsize            + " " +
1065 | 		SymlinkTag+"=" + LinkStartDelim  + symlink_str + LinkEndDelim +
1066 | 		"")
1067 | 	return mkmsg(subject, body)
1068 | 
1069 | #yy		  SymlinkTag  + "=" + LinkStartDelim  + str + LinkEndDelim + " " +
1070 | #		ret[LinkToTag]   =     m.group(4)
1071 | #	link_to  = src_msg_hash[LinkToTag]
1072 |     def dec_nlink(self):
1073 | 	self.i_nlink -= 1
1074 | 	if self.i_nlink >= 1:
1075 | 		self.mark_dirty("dec nlink")
1076 | 		return []
1077 | 	log_debug2("truncating inode")
1078 | 	subject = 'b='+str(self.ino)+''
1079 | 	block_uids = _getMsguidsByQuery("unlink blocks", self.fs.imap, [subject])
1080 | 	to_trash = []
1081 | 	to_trash.extend(block_uids)
1082 | 	to_trash.append(str(self.inode_msg.uid))
1083 | 	return to_trash
1084 | 
1085 |     def fill_from_inode_msg(self):
1086 |         """
1087 |         Setup the inode instances members from the gmail inode message
1088 |         """
1089 | 	log_debug2("filling inode")
1090 | 	if self.inode_msg.is_multipart():
1091 | 		body = self.inode_msg.preamble
1092 | 		log_debug2("message was multipart, reading body from preamble")
1093 | 	else:
1094 | 		# this is a bug
1095 | 		log_debug2("message was single part")
1096 |         log_debug2("body: ->%s<-" % body)
1097 | 	body = fixQuotedPrintable(body)
1098 | 	##
1099 | 	subj_hash = self.fs.parse_inode_msg_subj(self.inode_msg)
1100 |         self.version = subj_hash[VersionTag]
1101 |         self.ino = int(subj_hash[InodeTag])
1102 | 	log_debug2("set self.ino to: int: '%d' str: '%s'" % (self.ino, str(subj_hash[InodeTag])))
1103 |         self.dev =     subj_hash[DevTag]
1104 |         self.i_nlink =   subj_hash[NumberLinksTag]
1105 |         #quotedEquals = "=(?:3D)?(.*)"
1106 |         quotedEquals = "=(.*)"
1107 | 	restr = (	  ModeTag  + quotedEquals + ' ' +
1108 | 			  UidTag   + quotedEquals + ' ' +
1109 | 	  		  GidTag   + quotedEquals + ' ' +
1110 | 	                  SizeTag  + quotedEquals + ' ' +
1111 | 			  AtimeTag + quotedEquals + ' ' +
1112 | 			  MtimeTag + quotedEquals + ' ' +
1113 | 	                  CtimeTag + quotedEquals + ' ' +
1114 | 	                  BSizeTag + quotedEquals + ' ' +
1115 | 			  SymlinkTag + "=" + LinkStartDelim  + '(.*)' + LinkEndDelim)
1116 |         log_debug2("restr: ->%s<-" % (restr))
1117 | 	m = re.search(re.compile(restr, re.DOTALL), body)
1118 | 	self.mode  = int(m.group(1))
1119 |         self.uid   = int(m.group(2))
1120 |         self.gid   = int(m.group(3))
1121 |         self.size  = int(m.group(4))
1122 |         self.atime = int(m.group(5))
1123 |         self.mtime = int(m.group(6))
1124 |         self.ctime = int(m.group(7))
1125 |         self.block_size = int(m.group(8))
1126 | 	symlink_tmp    = m.group(9)
1127 | 	self.symlink_tgt = _pathSeparatorDecode(symlink_tmp)
1128 | 	log_debug2("filled inode size: %d" % self.size)
1129 | 	self.fill_xattrs()
1130 | 
1131 | #@-node:class GmailInode
1132 | 
1133 | #@+node:class OpenGmailFile
1134 | class OpenGmailFile():
1135 | 	def __init__(self, inode):
1136 | 		self.inode = inode
1137 | 		self.fs = self.inode.fs
1138 | 		self.users = 1
1139 |         	self.block_size = inode.block_size
1140 | 
1141 | 	def ts_cmp(self, a, b):
1142 | 	        return cmp(a.ts, b.ts) # compare as integers
1143 | 
1144 | 	def prune(self):
1145 | 		# This locking is a bit coarse.  We could lock
1146 | 		# just the inode or just OpenGmailFile
1147 | 		semget(self.inode.fs.inode_cache_lock)
1148 | 		for i in range(10):
1149 | 			# We do this so not to unfairly bias against
1150 | 			# blocks that keep hashing into the low buckets
1151 | 			skip = random.random() * len(gmail_blocks)
1152 | 			nr = 0
1153 | 			for block, g in gmail_blocks.items():
1154 | 				nr = nr + 1
1155 | 				if nr < skip:
1156 | 					continue
1157 | 				if len(gmail_blocks) > BlockCacheSize:
1158 | 					break
1159 | 				if block.dirty():
1160 | 					continue
1161 | 				del block.inode.i_blocks[block.block_nr]
1162 | 				del gmail_blocks[block]
1163 | 		self.inode.fs.inode_cache_lock.release()
1164 | 		#print("[%d] file now has %d blocks" % (time.time(), len(self.inode.blocks)))
1165 | 
1166 | 
1167 |     	def write(self, buf, off):
1168 | 		first_block = off / self.block_size
1169 | 		last_block = (off + len(buf)) / self.block_size
1170 | 
1171 | 		semget(self.inode.fs.inode_cache_lock)
1172 | 		for i in range(first_block, last_block+1):
1173 | 			if not self.inode.i_blocks.has_key(i):
1174 | 				self.inode.i_blocks[i] = GmailBlock(self.inode, i);
1175 | 			self.inode.i_blocks[i].write(buf, off)
1176 | 		self.inode.fs.inode_cache_lock.release()
1177 | 		self.prune()
1178 | 		return len(buf)
1179 | 
1180 |     	def read(self, readlen, off):
1181 | 		first_block = off / self.block_size
1182 | 		last_block = (off + readlen) / self.block_size
1183 | 
1184 | 		ret = []
1185 | 		semget(self.inode.fs.inode_cache_lock)
1186 | 		for i in range(first_block, last_block+1):
1187 | 			if not self.inode.i_blocks.has_key(i):
1188 | 				self.inode.i_blocks[i] = GmailBlock(self.inode, i);
1189 | 			ret += self.inode.i_blocks[i].read(readlen, off)
1190 | 		self.inode.fs.inode_cache_lock.release()
1191 | 		self.prune()
1192 | 		return ret
1193 | 
1194 | 	def close(self):
1195 | 	        """
1196 | 	        Closes this file by committing any changes to the users gmail account
1197 | 	        """
1198 | 		self.users -= 1
1199 | 		if self.users >= 1:
1200 | 			return self.users
1201 | 		return 0
1202 | 
1203 | 
1204 | gmail_blocks = {}
1205 | 
1206 | #@+node:class OpenGmailFile
1207 | class GmailBlock(Dirtyable):
1208 |     """
1209 |     Class holding any currently open files, includes cached instance of the last data block retrieved
1210 |     """
1211 | 
1212 |     def __init__(self, inode, block_nr):
1213 |         Dirtyable.__init__(self)
1214 |         self.inode = inode
1215 | 	self.fs = self.inode.fs
1216 | 
1217 |         self.block_size = inode.block_size
1218 |         self.buffer = []
1219 | 	self.buffer_lock = threading.Semaphore(1)
1220 | 	#list(" "*self.block_size)
1221 |         self.block_nr = block_nr
1222 |         self.start_offset = self.block_nr * self.block_size
1223 |         self.end_offset = self.start_offset + self.block_size
1224 | 	self.ts = time.time()
1225 | 	log_debug1("created new block: %d" % (self.block_nr))
1226 | 	gmail_blocks[self] = self
1227 | 
1228 |     def to_str(self):
1229 | 	return "block(%d)" % (self.block_nr)
1230 | 
1231 |     def covers(self, off, len):
1232 | 	# does this block cover the specified buffer?
1233 | 	if off+len <= self.start_offset:
1234 | 		return 0;
1235 | 	if off >= self.end_offset:
1236 | 		return 0;
1237 | 	return 1;
1238 | 
1239 |     def mypart(self, buf, off):
1240 | 	if not self.covers(off, len(buf)):
1241 | 	    return None, None;
1242 | 	if off >= self.end_offset:
1243 | 	    # strip off some of the beginning of the buffer
1244 | 	    to_chop = self.start_offset - off
1245 | 	    buf = buf[to_chop:]
1246 | 	    off = self.start_offset
1247 | 	if off + len(buf) > self.end_offset:
1248 | 	    new_len = self.block_size - offset
1249 | 	    buf = buf[:new_len]
1250 | 	return buf, off
1251 | 
1252 |     def write(self, buf, off):
1253 | 	buf_part, file_off = self.mypart(buf, off)
1254 | 	log_debug1("write block: %d" % (self.block_nr))
1255 | 	if buf_part == None or file_off == None:
1256 | 		return
1257 | 	log_debug1("my part of buffer: %d bytes, at offset: %d" % (len(buf_part), file_off))
1258 | 
1259 | 	if (len(buf_part) == self.block_size or
1260 |  	    off > self.inode.size):
1261 | 	    # If we're going to write the whole buffer, do
1262 | 	    # not bother fetching what we will write over
1263 | 	    # entirely anyway.
1264 | 	    semget(self.buffer_lock)
1265 | 	    self.buffer = list(" "*self.block_size)
1266 | 	    self.buffer_lock.release()
1267 |         else:
1268 | 	    self.populate_buffer(1)
1269 | 
1270 | 	buf_write_start = file_off - self.start_offset
1271 | 	buf_write_end = buf_write_start + len(buf_part)
1272 | 	if buf_write_start < 0:
1273 | 		print("bad block range: [%d:%d]" % (buf_write_start, buf_write_end))
1274 | 		print("bad block range: file_off: %d" % (file_off))
1275 | 		print("bad block range: start_offset: %d" % (self.start_offset))
1276 | 		print("bad block range: end_offset: %d" % (self.end_offset))
1277 | 		print("bad block range: buf_write_start: %d" % (buf_write_start))
1278 | 		print("bad block range: buf_write_end: %d" % (buf_write_end))
1279 | 		print("bad block range: len(buf_part): %d" % (len(buf_part)))
1280 | 		print("bad block orig: %d %d" % (len(buf), off))
1281 | 		abort()
1282 | 
1283 | 	semget(self.buffer_lock)
1284 | 	self.buffer[buf_write_start:buf_write_end] = buf_part;
1285 | 	self.buffer_lock.release()
1286 | 	log_debug1("wrote block range: [%d:%d]" % (buf_write_start, buf_write_end))
1287 | 
1288 | 	log_debug1("block write() setting dirty")
1289 | 	self.mark_dirty("file write")
1290 | 
1291 |         if file_off + len(buf_part) > self.inode.size:
1292 |             self.inode.size = file_off + len(buf_part)
1293 | 	    self.inode.mark_dirty("file write extend")
1294 | 	else:
1295 | 	    self.inode.mark_dirty("file write")
1296 | 	self.ts = time.time()
1297 |         return len(buf_part)
1298 | 
1299 |     def b_write_out(self, desc):
1300 |         log_debug1("b_write_out() self.dirty: '%s' desc: '%s'" % (Dirtyable.dirty_reason(self), desc))
1301 | 	#print("b_write_out() block %d self.dirty: '%s' desc: '%s'" % (self.block_nr, Dirtyable.dirty_reason(self), desc))
1302 | 
1303 |     	#a = self.inode.ga
1304 |         subject = ('b='+str(self.inode.ino)+
1305 | 	          ' x='+str(self.block_nr)+
1306 | 		  ' '+FsNameTag+'='+MagicStartDelim+ fsNameVar +MagicEndDelim )
1307 | 	tmpf = tempfile.NamedTemporaryFile()
1308 | 
1309 | 	semget(self.buffer_lock)
1310 | 	buf = self.buffer
1311 | 	self.buffer_lock.release()
1312 | 	if self.inode.size / self.block_size == self.block_nr:
1313 | 		part = self.inode.size % self.block_size
1314 | 		print("on last block, so only writing out %d/%d bytes of block" % (part, len(buf)))
1315 | 		buf = buf[:part]
1316 | 
1317 |         arr = array.array('c')
1318 |         arr.fromlist(buf)
1319 | 	log_debug("wrote contents to tmp file: ->"+arr.tostring()+"<-")
1320 | 
1321 | 	tmpf.write(arr.tostring())
1322 | 	tmpf.flush()
1323 | 
1324 | 	msg = mkmsg(subject, fsNameVar, arr.tostring())
1325 | 	imap = self.fs.get_imap()
1326 | 	msgid = imap_append("commit data blocks (%d bytes)" % len(str(msg)), self.inode.fs.imap, msg)
1327 | 	self.fs.put_imap(imap)
1328 | 	log_debug("b_write_out() finished, rsp: '%s'" % str(msgid))
1329 | 	if msgid > 0:
1330 |             log_debug("Sent write commit ok")
1331 |             self.inode.mark_dirty("commit data block")
1332 | 	    tmpf.close()
1333 |             ret = 0
1334 |         else:
1335 |             log.error("Sent write commit failed")
1336 | 	    tmpf.close()
1337 |             ret = -3
1338 | 	return ret
1339 | 
1340 |     def read(self, readlen, file_off):
1341 |         readlen = min(self.inode.size - file_off, readlen)
1342 | 	log_debug1("read block: %d" % (self.block_nr))
1343 | 
1344 |         self.populate_buffer(1)
1345 | 	start_offset = max(file_off, self.start_offset)
1346 | 	end_offset   = min(file_off + readlen, self.end_offset)
1347 | 	start_offset -= self.start_offset
1348 | 	end_offset   -= self.start_offset
1349 | 
1350 | 	self.ts = time.time()
1351 | 	return self.buffer[start_offset:end_offset]
1352 | 
1353 |     def populate_buffer(self, deleteAfter):
1354 |         """
1355 |         Read this data block with from gmail.  If 'deleteAfter' is
1356 |         true then the block will be removed from Gmail after reading
1357 |         """
1358 | 	semget(self.buffer_lock)
1359 | 	if len(self.buffer):
1360 | 	    self.buffer_lock.release()
1361 | 	    return
1362 | 	log_debug1("populate_buffer() filling block %d because len: %d" % (self.block_nr, len(self.buffer)))
1363 | 
1364 |         q1 = 'b='+str(self.inode.ino)
1365 | 	q2 = 'x='+str(self.block_nr)
1366 |         msg = getSingleMessageByQuery("block read", self.inode.fs.imap, [ q1, q2 ])
1367 | 	if msg == None:
1368 | 	    log_debug2("readFromGmail(): file has no blocks, returning empty contents (%s %s)" % (q1, q2))
1369 | 	    self.buffer = list(" "*self.block_size)
1370 | 	    self.buffer_lock.release()
1371 | 	    return
1372 |         log_debug2("got msg with subject:"+msg['Subject'])
1373 | 	for part in msg.walk():
1374 |             log_debug2("message part.get_content_maintype(): '%s'" % part.get_content_maintype())
1375 |             if part.get_content_maintype() == 'multipart':
1376 |                 continue
1377 | 	    #if part.get('Content-Disposition') is None:
1378 | 	    #    continue
1379 | 	    log_debug2("message is multipart")
1380 | 	    a = part.get_payload(decode = True)
1381 | 	    log_debug3("part payload has len: %d asstr: '%s'" % (len(a), str(a)))
1382 | 	log_debug3("after loop, a: '%s'" % str(a))
1383 | 	a = list(a)
1384 | 
1385 |         if deleteAfter:
1386 |             imap_trash_msg(self.inode.fs.imap, msg)
1387 |         contentList = list(" "*self.block_size)
1388 |         contentList[0:] = a
1389 |         self.buffer = contentList
1390 | 	print("populate_buffer() filled block %d with len: %d" % (self.block_nr, len(self.buffer)))
1391 | 	self.buffer_lock.release()
1392 | 
1393 | #@-node:class OpenGmailFile
1394 | 
1395 | #@+node:class Gmailfs
1396 | class Gmailfs(Fuse):
1397 | 
1398 |     def kick_imap(self, imap):
1399 | 	try:
1400 | 		self.disconnect_from_server(imap)
1401 | 	except:
1402 | 		pass
1403 | 	self.connect_to_server(imap)
1404 | 
1405 |     def disconnect_from_server(self, imap):
1406 | 	try:
1407 | 		imap.close()
1408 | 	except:
1409 | 		pass
1410 | 	try:
1411 | 		imap.logout()
1412 | 	except:
1413 | 		pass
1414 | 
1415 |     #@	@+others
1416 |     def connect_to_server(self, imap = None):
1417 |         global fsNameVar
1418 |         global password
1419 |         global username
1420 | 
1421 | 	fsNameVar = DefaultFsname
1422 |         password = DefaultPassword
1423 |         username = DefaultUsername
1424 | 	if imap == None:
1425 | 	        imap = imaplib.IMAP4_SSL("imap.gmail.com", 993)
1426 | 		imap.fs = self
1427 | 		imap.lock = threading.Semaphore(1)
1428 | 	else:
1429 | 		imap.open("imap.gmail.com", 993) #libgmail.GmailAccount(username, password)
1430 |         if username.find("@")<0:
1431 | 		username = username+"@gmail.com"
1432 |         imap.login(username, password)
1433 | 	resp, data = imap.select(fsNameVar)
1434 | 	log_debug1("folder select '%s' resp: '%s' data: '%s'" % (fsNameVar, resp, data))
1435 | 	if resp == "NO":
1436 | 		log_info("creating mailbox")
1437 | 		resp, data = imap.create(fsNameVar)
1438 | 		log_debug1("create '%s' resp: '%s' data: '%s'" % (fsNameVar, resp, data))
1439 | 		resp, data = imap.select(fsNameVar)
1440 | 		log_debug1("select2 '%s' resp: '%s' data: '%s'" % (fsNameVar, resp, data))
1441 | 		return
1442 | 	return imap
1443 | 
1444 |     def get_imap(self):
1445 | 	return self.imap_pool.get()
1446 | 
1447 |     def put_imap(self, imap):
1448 | 	self.imap_pool.put(imap)
1449 | 
1450 |     #@+node:__init__
1451 |     def __init__(self, extraOpts, mountpoint, *args, **kw):
1452 |         Fuse.__init__(self, *args, **kw)
1453 | 
1454 |     	self.nr_imap_threads = 4
1455 | 	self.imap_pool = Queue.Queue(self.nr_imap_threads)
1456 | 	for i in range(self.nr_imap_threads):
1457 | 		self.imap_pool.put(self.connect_to_server())
1458 | 
1459 | 	self.dirty_objects = Queue.Queue(50)
1460 | 	self.lookup_lock = threading.Semaphore(1)
1461 |         self.inode_cache_lock = threading.Semaphore(1)
1462 | 
1463 |         self.fuse_args.mountpoint = mountpoint
1464 | 	self.fuse_args.setmod('foreground')
1465 | 	self.optdict = extraOpts
1466 |         log_debug("Mountpoint: %s" % mountpoint)
1467 | 	# obfuscate sensitive fields before logging
1468 | 	#loggableOptdict = self.optdict.copy()
1469 | 	#loggableOptdict['password'] = '*' * 8
1470 | 	#log_info("Named mount options: %s" % (loggableOptdict,))
1471 | 
1472 |         # do stuff to set up your filesystem here, if you want
1473 | 
1474 |         self.openfiles = {}
1475 |         self.flush_dirent_cache()
1476 | 
1477 |         global DefaultBlockSize
1478 | 
1479 | 
1480 | #	options_required = 1
1481 | #	if self.optdict.has_key("reference"):
1482 | #	    try:
1483 | #		reference = References[self.optdict['reference']]
1484 | #		username = reference.username
1485 | #		password = reference.password
1486 | #		fsNameVar = reference.fsname
1487 | #	    except:
1488 | #		log.error("Invalid reference supplied. Using defaults.")
1489 | #	    else:
1490 | #		options_required = 0
1491 | #
1492 | #        if not self.optdict.has_key("username"):
1493 | #	    if options_required:
1494 | #	        log_warning('mount: warning, should mount with username=gmailuser option, using default')
1495 | #        else:
1496 | #            username = self.optdict['username']
1497 | #
1498 | #        if not self.optdict.has_key("password"):
1499 | #	    if options_required:
1500 | #        	log_warning('mount: warning, should mount with password=gmailpass option, using default')
1501 | #        else:
1502 | #            password = self.optdict['password']
1503 | #
1504 | #        if not self.optdict.has_key("fsname"):
1505 | #	    if options_required:
1506 | #        	log_warning('mount: warning, should mount with fsname=name option, using default')
1507 | #        else:
1508 | #            fsNameVar = self.optdict['fsname']
1509 | #
1510 | #        if self.optdict.has_key("blocksize"):
1511 | #            DefaultBlockSize = int(self.optdict['blocksize'])
1512 | 
1513 | #04:52.69 CAPABILITIES: ('IMAP4REV1', 'UNSELECT', 'IDLE', 'NAMESPACE', 'QUOTA', 'XLIST', 'CHILDREN', 'XYZZY')
1514 | #04:52.97 < * CAPABILITY IMAP4rev1 UNSELECT LITERAL+ IDLE NAMESPACE QUOTA ID XLIST CHILDREN X-GM-EXT-1 UIDPLUS COMPRESS=DEFLATE
1515 | 
1516 | 	self.imap = self.connect_to_server()
1517 | 	# This select() can be done read-only
1518 | 	# might be useful for implementing "mount -o ro"
1519 |         log_info("Connected to gmail")
1520 | 	#resp, data = self.imap.list()
1521 | 	#log_info("list resp: " + resp)
1522 | 	#for mbox in data:
1523 | 	#	log_info("mbox: " + mbox)
1524 | 	#log_info("done listing mboxes")
1525 | 
1526 | 	#FIXME
1527 | 	# we should probably make a mkfs command to
1528 | 	# make the root inode.  We should probably
1529 | 	# also make it search out and clear all
1530 | 	# messages with the given label
1531 | 	#self.imap.debug = 4
1532 | 	trash_all = 0
1533 | 	if "IMAPFS_TRASH_ALL" in os.environ:
1534 | 		trash_all = 1
1535 | 	if trash_all:
1536 | 		print("deleting existing messages...")
1537 | 		semget(self.imap.lock)
1538 | 		resp, msgids = imap_uid(self.imap, "SEARCH", 'ALL')
1539 | 		self.imap.lock.release()
1540 | 		uids = msgids[0].split()
1541 | 		print ("%d found..." % (len(uids)))
1542 | 		joined_uids = string.join(msgids[0].split(), ",")
1543 | 		log_debug2("about to delete msgids: ->%s<-" % (joined_uids))
1544 | 		if (len(uids)):
1545 | 			imap_trash_uids(self.imap, uids)
1546 | 		print("done deleting %d existing messages" % (len(msgids[0].split())))
1547 | 		semget(self.imap.lock)
1548 | 		resp, msgids = imap_uid(self.imap, "SEARCH", 'ALL')
1549 | 		self.imap.lock.release()
1550 | 		print("mailbox now has %d messages" % (len(msgids[0].split())))
1551 | 		self.imap.expunge()
1552 | 
1553 | 	semget(self.imap.lock)
1554 | 	resp, msgids = imap_uid(self.imap, "SEARCH", 'ALL')
1555 | 	self.imap.lock.release()
1556 | 	print("mailbox now has %d messages" % (len(msgids[0].split())))
1557 | 	#exit(0)
1558 | 	#elf.mythread()
1559 | 
1560 |         pass
1561 |     #@-node:__init__
1562 | 
1563 |     #@+node:attribs
1564 |     flags = 1
1565 | 
1566 |     #@-node:attribs
1567 | 
1568 |     class GmailStat(fuse.Stat):
1569 |         def __init__(self):
1570 |             self.st_mode = 0
1571 |             self.st_ino = 0
1572 |             self.st_dev = 0
1573 |             self.st_nlink = 0
1574 |             self.st_uid = 0
1575 |             self.st_gid = 0
1576 |             self.st_size = 0
1577 |             self.st_atime = 0
1578 |             self.st_mtime = 0
1579 |             self.st_ctime = 0
1580 |             self.st_blocks = 0
1581 | 	    global IMAPBlockSize
1582 |             self.st_blksize = IMAPBlockSize
1583 |             self.st_rdev = 0
1584 | 
1585 |     #@+node:getattr
1586 |     def getattr(self, path):
1587 |         st = Gmailfs.GmailStat();
1588 |         log_debug2("getattr('%s')" % (path))
1589 |         #st_mode (protection bits)
1590 |         #st_ino (inode number)
1591 |         #st_dev (device)
1592 |         #st_nlink (number of hard links)
1593 |         #st_uid (user ID of owner)
1594 |         #st_gid (group ID of owner)
1595 |         #st_size (size of file, in bytes)
1596 |         #st_atime (time of most recent access)
1597 |         #st_mtime (time of most recent content modification)
1598 |         #st_ctime (time of most recent content modification or metadata change).
1599 | 
1600 | 	log_debug3("getattr() -1")
1601 |         inode = self.lookup_inode(path)
1602 | 	log_debug3("getattr() 0")
1603 |         if (inode == None) and (path == '/'):
1604 | 		log_info("creating root inode")
1605 | 		mode = S_IFDIR|S_IRUSR|S_IXUSR|S_IWUSR|S_IRGRP|S_IXGRP|S_IXOTH|S_IROTH
1606 |         	inode = self.mk_inode(mode, 1, 2)
1607 | 		dirent = self.link_inode(path, inode)
1608 | 		write_out(inode, "new root inode")
1609 | 		write_out(dirent, "new root dirent")
1610 | 		log_info("root inode uids: %s %s" % (dirent.dirent_msg.uid, inode.inode_msg.uid))
1611 |         	inode = self.lookup_inode(path)
1612 | 		if inode == None:
1613 | 			log_info("uh oh, can't find root inode")
1614 | 			exit(-1)
1615 | 	log_debug3("getattr() 1")
1616 | 
1617 |         if inode:
1618 | 	    log_debug3("getattr() 2")
1619 | 	    log_debug3("found inode for path: '%s'" % (path))
1620 |             st.st_mode  = inode.mode
1621 |             st.st_ino   = inode.ino
1622 |             st.st_dev   = inode.dev
1623 |             st.st_nlink = inode.i_nlink
1624 |             st.st_uid   = inode.uid
1625 |             st.st_gid   = inode.gid
1626 |             st.st_size  = inode.size
1627 |             st.st_atime = inode.atime
1628 |             st.st_mtime = inode.mtime
1629 |             st.st_ctme  = inode.ctime
1630 |             log_debug3("st.st_mode   = %d" % ( inode.mode))
1631 |             log_debug3("st.st_ino    = %d" % ( inode.ino))
1632 |             log_debug3("st.st_dev    = %d" % ( inode.dev))
1633 |             log_debug3("st.st_nlink  = %d" % ( inode.i_nlink))
1634 |             log_debug3("st.st_uid    = %d" % ( inode.uid))
1635 |             log_debug3("st.st_gid    = %d" % ( inode.gid))
1636 |             log_debug3("st.st_size   = %d" % ( inode.size))
1637 |             log_debug3("st.st_atime  = %d" % ( inode.atime))
1638 |             log_debug3("st.st_mtime  = %d" % ( inode.mtime))
1639 |             log_debug3("st.st_ctme   = %d" % ( inode.ctime))
1640 | 	    log_debug3("getattr() 3: ->%s<-" % (str(st)))
1641 |             return st
1642 |     #else:
1643 |  	#log_info("getattr ENOENT: '%s'" % (path))
1644 | 	    #e = OSError("No such file"+path)
1645 | 	    #e.errno = ENOENT
1646 | 	    #raise e
1647 |         log_debug3("getattr('%s') done" % (path))
1648 | 	return -ENOENT
1649 |     #@-node:getattr
1650 | 
1651 |     #@+node:readlink
1652 |     def readlink(self, path):
1653 |         log_entry("readlink: path='%s'" % path)
1654 |         dirent = self.lookup_dirent(path)
1655 | 	inode = dirent.inode
1656 |         if not (inode.mode & S_IFLNK):
1657 |             e = OSError("Not a link"+path)
1658 |             e.errno = EINVAL
1659 |             raise e
1660 |         log_debug("about to follow link in body:"+inode.msg.as_string())
1661 | 	body = fixQuotedPrintable(inode.msg.as_string())
1662 |         m = re.search(SymlinkTag+'='+LinkStartDelim+'(.*)'+
1663 |                       LinkEndDelim,body)
1664 |         return m.group(1)
1665 |     #@-node:readlink
1666 | 
1667 |    #@+node:readdir
1668 |     def readdir(self, path, offset):
1669 |         log_entry("[%d] readdir('%s', %d)" % (thread.get_ident(), path, offset))
1670 |         log_debug3("at top of readdir");
1671 |         log_debug3("getting dir "+path)
1672 |         fspath = _pathSeparatorEncode(path)
1673 |         log_debug3("querying for:"+''+PathNameTag+'='+PathStartDelim+
1674 |                   fspath+PathEndDelim)
1675 |         # FIX need to check if directory exists and return error if it doesnt, actually
1676 |         # this may be done for us by fuse
1677 | 	q = ''+PathNameTag+'='+PathStartDelim+fspath+PathEndDelim
1678 |         msgids = _getMsguidsByQuery("readdir", self.imap, [q])
1679 |         log_debug3("got readdir msg list")
1680 |         lst = []
1681 |         for dirlink in ".", "..":
1682 |             lst.append(dirlink)
1683 | 
1684 | 	for c_path, inode in self.dirent_cache.items():
1685 | 		c_dir, c_file = parse_path(c_path)
1686 | 		if path != c_dir:
1687 | 			continue
1688 | 		# Found "." which we already have
1689 | 		if len(c_file) == 0:
1690 | 			continue
1691 | 		log_debug2("found dir: '%s' file: '%s' for readdir('%s') in inode cache[%s]" % (c_dir, c_file, path, c_path))
1692 | 		lst.append(c_file)
1693 | 	for msgid, msg in fetch_full_messages(self.imap, msgids).items():
1694 | 		subject = msg['Subject']
1695 | 		#log_debug("thread.summary is " + thread.snippet)
1696 | 		m = re.search(FileNameTag+'='+FileStartDelim+'(.*)'+
1697 |                            FileEndDelim, subject)
1698 | 		if (m):
1699 | 			# Match succeeded, we got the whole filename.
1700 | 			log_debug("Used summary for filename")
1701 | 			filename = m.group(1)
1702 | 
1703 | 		log_debug("readdir('%s') found file: '%s'" % (path, filename))
1704 | 		# this test for length is a special case hack for the root directory to prevent ls /gmail_root
1705 | 		# returning "". This is hack is requried due to adding modifiable root directory as an afterthought, rather
1706 | 		# than designed in at the start.
1707 | 		if len(filename) <=0:
1708 | 			continue
1709 | 		filename = _pathSeparatorDecode(filename)
1710 | 		if lst.count(filename) == 0:
1711 | 			lst.append(filename)
1712 | 	log_debug2("readdir('%s') got %d entries" % (path, len(lst)))
1713 |         for r in lst:
1714 | 	    log_debug3("readdir('%s') entry: '%s'" % (path, r))
1715 | 	    yield fuse.Direntry(r)
1716 |     #@-node:getdir
1717 | 
1718 |     dirent_cache = {}
1719 |     def flush_dirent_cache(self):
1720 | 	    log_info("flush_dirent_cache()")
1721 | 	    remove_keys = []
1722 | 	    for path, dirent in self.dirent_cache.items():
1723 | 		    log_debug3("dirent_cache[%s]: '%s'" % (path, str(dirent)))
1724 | 		    if dirent.inode.dirty() or dirent.dirty():
1725 | 			    continue
1726 | 		    remove_keys.append(path)
1727 | 	    for key in remove_keys:
1728 | 		    dirent = self.dirent_cache[key]
1729 | 		    del self.dirent_cache[key]
1730 | 		    self.put_inode(dirent.inode)
1731 | 
1732 | 	    while 1:
1733 | 		try:
1734 | 			# no args means do not block, and trow
1735 | 			# exception immediately if empty
1736 | 			object = self.fs.dirty_objects.get()
1737 | 			write_out(object, "flush_dirent_cache()")
1738 | 			log_info("flush_dirent_cache() wrote out %s" % (object.to_str()))
1739 | 		except:
1740 | 			log_info("no more object to flush")
1741 | 			break
1742 | 		size = self.fs.dirty_objects.qsize()
1743 | 	    log_info("explicit flush done")
1744 | 
1745 |     #@+node:unlink
1746 |     def unlink(self, path):
1747 |         log_entry("unlink called on:"+path)
1748 |         try:
1749 |             dirent = self.lookup_dirent(path)
1750 |             dirent.unlink()
1751 |             return 0
1752 |         except:
1753 |             _logException("Error unlinking file"+path)
1754 |             e = OSError("Error unlinking file"+path)
1755 |             e.errno = EINVAL
1756 |             raise e
1757 |     #@-node:unlink
1758 | 
1759 |     #@+node:rmdir
1760 |     def rmdir(self, path):
1761 |         log_debug1("rmdir called on:"+path)
1762 |         #this is already checked before rmdir is even called
1763 |         #dirlist = self.getdir(path)
1764 |         #if len(dirlist)>0:
1765 |         #    e = OSError("directory not empty"+path)
1766 |         #    e.errno = ENOTEMPTY
1767 |         #    raise e
1768 |         dirent = self.lookup_dirent(path)
1769 |         dirent.unlink()
1770 | 
1771 |         # update number of links in parent directory
1772 | 	parentdir, filename = parse_path(path)
1773 |         log_debug("about to rmdir with parentdir:"+parentdir)
1774 | 
1775 |         parentdirinode = self.lookup_inode(parentdir)
1776 |         parentdirinode.dec_nlink()
1777 |         return 0
1778 | 
1779 |     #@-node:rmdir
1780 | 
1781 |     #@+node:symlink
1782 |     def symlink(self, oldpath, newpath):
1783 |         log_debug1("symlink: oldpath='%s', newpath='%s'" % (oldpath, newpath))
1784 | 	mode = S_IFLNK|S_IRWXU|S_IRWXG|S_IRWXO
1785 | 	inode = self.mk_inode(mode, 0, 1)
1786 | 	inode.symlink_tgt = newpath
1787 | 	self.link_inode(oldpath, inode)
1788 | 
1789 |     #@-node:symlink
1790 | 
1791 |     # This provides a single, central place to define the format
1792 |     # of the message subjects.  'str' can be something like "%s"
1793 |     # to create a printf-style format string for output.  Or, it
1794 |     # can be a regex to help with input.
1795 |     def format_dirent_subj(self, str):
1796 |         # any change in here must be reflected in the two
1797 | 	# functions below
1798 |         subject =(DirentSubjectPrefix+ " " +
1799 | 		  PathNameTag + "=" + PathStartDelim  + str + PathEndDelim + " " +
1800 | 		  FileNameTag + "=" + FileStartDelim  + str + FileEndDelim + " " +
1801 |                   RefInodeTag + "=" + str      	      + " " +
1802 | 	          FsNameTag   + "=" + MagicStartDelim + str + MagicEndDelim+ " " +
1803 | 		  VersionTag  + "=" + str)
1804 | 	return subject
1805 | 
1806 |     def parse_dirent_msg(self, msg):
1807 |         subject_re = self.format_dirent_subj('(.*)')
1808 | 	subject = msg['Subject'].replace("\r\n\t", " ")
1809 |         m = re.match(subject_re, subject)
1810 | 	log_debug3("looking for regex: '%s'" % (subject_re))
1811 | 	log_debug3("subject: '%s'" % (subject))
1812 | 	log_debug3("match: '%s'" % (str(m)))
1813 | 	ret = {}
1814 | 	# Make sure these match the order of the strings in
1815 | 	# format_dirent_subj()
1816 | 	try:
1817 | 		ret[PathNameTag] =     m.group(1)
1818 |  		ret[FileNameTag] =     m.group(2)
1819 | 		ret[RefInodeTag] = int(m.group(3))
1820 | 		ret[FsNameTag]   =     m.group(4)
1821 | 		ret[VersionTag]  = int(m.group(5))
1822 | 	except:
1823 | 		log_error("unable to match regex\n\n\n\n")
1824 | 		ret = None
1825 | 	if ret[FsNameTag] != fsNameVar:
1826 | 		log_error("msgid[%s] wrong filesystem: '%s'" % (msg.uid, ret[FsNameTag]))
1827 | 		return None
1828 | 	if ret[VersionTag] != int(GMAILFS_VERSION):
1829 | 		log_error("msgid[%s] wrong version: '%s', expected '%d'" % (msg.uid, ret[VersionTag], int(GMAILFS_VERSION)))
1830 | 		return None
1831 | 	for key, val in ret.items():
1832 | 		log_debug3("parse_dirent_msg() ret[%s]: '%s'" % (key, val))
1833 | 	return ret;
1834 | 
1835 |     def mk_dirent_msg(self, path, inode_nr_ref):
1836 | 	log_debug1("mk_dirent_msg('%s', 'ino=%s')" % (path, str(inode_nr_ref)))
1837 | 	body = ""
1838 | 	path, filename = parse_path(path)
1839 | 
1840 | 	path = _pathSeparatorEncode(path)
1841 | 	filename = _pathSeparatorEncode(filename)
1842 | 	# again, make sure these are all in the correct order
1843 | 	subject = self.format_dirent_subj("%s") % (
1844 | 			path,
1845 | 			filename,
1846 | 			inode_nr_ref,
1847 | 			fsNameVar,
1848 | 			GMAILFS_VERSION)
1849 | 	return mkmsg(subject, body)
1850 | 
1851 |     def parse_inode_msg_subj(self, inode_msg):
1852 |             subject = inode_msg['Subject'].replace('\u003d','=')
1853 |             log_debug3("parsing inode from subject:"+subject)
1854 |     	    ret = {}
1855 |             m = re.match((InodeSubjectPrefix+' '+
1856 | 			  VersionTag+'=(.*) '+
1857 | 			  InodeTag+'=(.*) '+
1858 | 			  DevTag+'=(.*) '+
1859 | 			  NumberLinksTag+'=(.*) '+
1860 | 			  FsNameTag+'='+MagicStartDelim+'(.*)'+MagicEndDelim),
1861 | 			 subject)
1862 | 	    if m == None:
1863 | 	    	return None
1864 |             ret[VersionTag]     = int(m.group(1))
1865 | 	    ret[InodeTag]       = int(m.group(2))
1866 | 	    ret[DevTag]         = int(m.group(3))
1867 | 	    ret[NumberLinksTag] = int(m.group(4))
1868 | 	    return ret
1869 | 
1870 | 
1871 |     #@+node:rename
1872 |     def rename(self, path_src, path_dst):
1873 |         log_debug1("rename from: '%s' -> '%s'" % (path_src, path_dst))
1874 |         src_dirent = self.lookup_dirent(path_src)
1875 | 	if src_dirent == None:
1876 | 		return -ENOENT
1877 | 
1878 | 	dst_dirent = self.lookup_dirent(path_dst)
1879 | 	if not dst_dirent == None:
1880 | 		dst_dirent.unlink()
1881 | 	# ensure the inode does not go away between
1882 | 	# when we unlink and relink it
1883 | 	inode = self.get_inode(src_dirent.inode.ino)
1884 | 	# do the unlink first, because otherwise we
1885 | 	# will get two dirents at the same path
1886 | 	src_dirent.unlink()
1887 | 	self.link_inode(path_dst, inode)
1888 | 	self.put_inode(inode)
1889 | 
1890 |     #@-node:rename
1891 | 
1892 |     #@+node:link
1893 |     def link(self, old_path, new_path):
1894 |         log_entry("hard link: old_path='%s', new_path='%s'" % (old_path, new_path))
1895 |         inode = self.lookup_inode(old_path)
1896 |         if not (inode.mode & S_IFREG):
1897 |             e = OSError("hard links only supported for regular files not directories:"+oldpath)
1898 |             e.errno = EPERM
1899 |             raise e
1900 |         inode.mark_dirty("link")
1901 | 	link_to(new_path, inode)
1902 |         return 0
1903 |     #@-node:link
1904 | 
1905 |     #@+node:chmod
1906 |     def chmod(self, path, mode):
1907 |         log_entry("chmod('%s', %o)" % (path, mode))
1908 |         inode = self.lookup_inode(path)
1909 |         inode.mode = (inode.mode & ~(S_ISUID|S_ISGID|S_ISVTX|S_IRWXU|S_IRWXG|S_IRWXO)) | mode
1910 |         inode.mark_dirty("chmod")
1911 |         return 0
1912 |     #@-node:chmod
1913 | 
1914 |     #@+node:chown
1915 |     def chown(self, path, user, group):
1916 |         log_entry("chown called with user:"+str(user)+" and group:"+str(group))
1917 |         inode = self.lookup_inode(path)
1918 |         inode.uid = user
1919 |         inode.gid = group
1920 |         inode.mark_dirty("chown")
1921 |         return 0
1922 |     #@-node:chown
1923 | 
1924 |     #@+node:truncate
1925 |     def truncate(self, path, size):
1926 |         inode = self.lookup_inode(path)
1927 |         log_entry("truncate '%s' to size: '%d' from '%d'" % (path, size, inode.size))
1928 |         # this is VERY lazy, we leave the truncated data around
1929 |         # it WILL be harvested when we grow the file again or
1930 |         # when we delete the file but should probably FIX
1931 | 	if inode.size != size:
1932 | 	        inode.size = size;
1933 |         	inode.mark_dirty("truncate")
1934 |         return 0
1935 |     #@-node:truncate
1936 | 
1937 |     #@+node:getxattr
1938 |     def getxattr(self, path, attr, size):
1939 |         log_entry("getxattr('%s', '%s', '%s')" % (path, attr, size))
1940 |         inode = self.lookup_inode(path)
1941 | 	# TODO check to make sure we don't overflow size
1942 | 	if attr not in inode.xattr:
1943 | 		return -ENODATA
1944 | 	ret = inode.xattr[attr]
1945 | 	if size == 0:
1946 | 		return len(ret)
1947 |         return ret
1948 |     #@-node:getxattr
1949 | 
1950 |     #@+node:setxattr
1951 |     def setxattr(self, path, attr, value, dunno):
1952 |         log_entry("setxattr('%s', '%s', '%s', '%s')" % (path, attr, value, dunno))
1953 |         inode = self.lookup_inode(path)
1954 | 	inode.xattr[attr] = value
1955 | 	inode.mark_dirty("setxattr")
1956 |     	return 0
1957 |     #@-node:setxattr
1958 | 
1959 |     #@+node:removexattr
1960 |     def removexattr(self, path, attr, value, dunno):
1961 |         log_entry("removexattr('%s', '%s')" % (path, attr))
1962 |         inode = self.lookup_dirent(path)/inode
1963 | 	try:
1964 | 		del inode.xattr[attr]
1965 | 	except:
1966 | 		return -ENOATTR
1967 | 	inode.mark_dirty("removexattr")
1968 |     	return 0
1969 |     #@-node:removexattr
1970 | 
1971 |     #@+node:listxattr
1972 |     def listxattr(self, path, size):
1973 |         log_entry("listxattr('%s', '%s')" % (path, size))
1974 |         inode = self.lookup_inode(path)
1975 | 	# We use the "user" namespace to please XFS utils
1976 | 	attrs = []
1977 | 	for attr in inode.xattr:
1978 | 	        log_debug1("listxattr() attr: '%s'" % (attr))
1979 | 		attrs.append(attr)
1980 | 	if size == 0:
1981 | 		# We are asked for size of the attr list, ie. joint size of attrs
1982 | 		# plus null separators.
1983 | 		return len("".join(attrs)) + len(attrs)
1984 |     	log_debug1("all attrs: (%s)" % (string.join(attrs, ", ")))
1985 | 	return attrs
1986 |     #@-node:listxattr
1987 | 
1988 |     #@+node:mknod
1989 |     def mknod(self, path, mode, dev):
1990 |     	""" Python has no os.mknod, so we can only do some things """
1991 |         log_entry("mknod('%s')" % (path))
1992 | 	if S_ISREG(mode) | S_ISFIFO(mode) | S_ISSOCK(mode):
1993 | 	    inode = self.mk_inode(mode, 0, 1)
1994 | 	    self.link_inode(path, inode)
1995 | 	    # update parent dir??
1996 |     	    #open(path, "w")
1997 |     	else:
1998 |     		return -EINVAL
1999 |     #@-node:mknod
2000 | 
2001 |     def mk_dirent(self, inode, path):
2002 | 	if self.dirent_cache.has_key(path):
2003 | 		log_debug("dirent cache hit on path: '%s'" % (path))
2004 | 		return self.dirent_cache[path]
2005 | 	# this should keep us from racing with lookup_dirent()
2006 | 	semget(self.lookup_lock)
2007 | 	filename, dir = parse_path(path)
2008 | 	msg = self.mk_dirent_msg(path, inode.ino)
2009 | 	dirent = GmailDirent(msg, inode, self)
2010 | 	dirent.mark_dirty("mk_dirent")
2011 | 	if len(self.dirent_cache) > 1000:
2012 |                 self.flush_dirent_cache()
2013 | 	log_debug1("added dirent to cache for path: '%s'" % (dirent.path()))
2014 |         self.dirent_cache[dirent.path()] = dirent
2015 | 	self.lookup_lock.release()
2016 | 	return dirent
2017 | 
2018 |     def mk_inode(self, mode, size, nlink=1):
2019 | 	inode = GmailInode(None, self)
2020 | 	inode.mode = int(mode)
2021 | 	inode.size = int(size)
2022 | 	inode.i_nlink = int(nlink)
2023 | 	inode.mark_dirty("new inode")
2024 | 	self.inode_cache[inode.ino] = inode
2025 | 	return inode
2026 | 
2027 |     def link_inode(self, path, inode):
2028 | 	dirent = self.mk_dirent(inode, path)
2029 | 	return dirent
2030 | 
2031 |     def lookup_inode(self, path):
2032 |     	dirent = self.lookup_dirent(path)
2033 | 	if dirent == None:
2034 | 		log_debug2("no dirent for path: '%s'" % (path))
2035 | 		return None
2036 | 	return dirent.inode
2037 | 
2038 |     #@+node:mkdir
2039 |     def mkdir(self, path, mode):
2040 |         log_entry("mkdir('%s', %o)" % (path, mode))
2041 | 	if (self.lookup_dirent(path) != None):
2042 | 		return -EEXIST
2043 |         inode = self.mk_inode(mode|S_IFDIR, 1, 2)
2044 | 	self.link_inode(path, inode)
2045 | 	parentdir, name = parse_path(path)
2046 |         parentdirinode = self.lookup_inode(parentdir)
2047 |         parentdirinode.i_nlink += 1
2048 |         parentdirinode.mark_dirty("mkdir")
2049 |     #@-node:mkdir
2050 | 
2051 |     #@+node:utime
2052 |     def utime(self, path, times):
2053 |         log_entry("utime for path:"+path+" times:"+str(times))
2054 |         inode = self.lookup_inode(path)
2055 |         inode.atime = times[0]
2056 |         inode.mtime = times[1]
2057 |         return 0
2058 |     #@-node:utime
2059 | 
2060 |     #@+node:open
2061 |     def open(self, path, flags):
2062 |         log_entry("gmailfs.py:Gmailfs:open: %s" % path)
2063 |         try:
2064 |             inode = self.lookup_inode(path)
2065 | 	    # If the same file is opened twice, use the
2066 | 	    # existing entry.  I'm not sure how
2067 | 	    # semantically correct this is.  Seems like
2068 | 	    # it could cause problems down the road.
2069 | 	    # Who knows...
2070 | 	    if self.openfiles.has_key(path):
2071 | 	    	self.openfiles[path].users += 1
2072 | 	    else:
2073 | 		f = OpenGmailFile(inode)
2074 | 		self.openfiles[path] = f
2075 |             return 0
2076 |         except:
2077 | 	    _logException("Error opening file: "+path)
2078 | 	    e = OSError("Error opening file: "+path)
2079 |             e.errno = EINVAL
2080 |             raise e
2081 |     #@-node:open
2082 | 
2083 |     #@+node:read
2084 |     def read(self, path, readlen, offset):
2085 | 	log_entry("read")
2086 |         try:
2087 |  	    log_debug1("gmailfs.py:Gmailfs:read(len=%d, offset=%d, path='%s')"
2088 | 			    % (readlen, offset, path))
2089 |             f = self.openfiles[path]
2090 |             buf = f.read(readlen,offset)
2091 |             arr = array.array('c')
2092 |             arr.fromlist(buf)
2093 |             rets = arr.tostring()
2094 | 
2095 |             return rets
2096 |         except:
2097 |             _logException("Error reading file"+path)
2098 |             e = OSError("Error reading file"+path)
2099 |             e.errno = EINVAL
2100 |             raise e
2101 |     #@-node:read
2102 | 
2103 |     #@+node:write
2104 |     def write(self, path, buf, off):
2105 | 	log_entry("write('%s', len:%d, off:%d)" % (path, len(buf), off))
2106 |         try:
2107 |             if log.isEnabledFor(logging.DEBUG):
2108 |                 log_debug3("writing file contents: ->"+str(buf)+"<-")
2109 |             f = self.openfiles[path]
2110 |             written = f.write(buf,off)
2111 | 	    log_debug2("wrote %d bytes to file: '%s'" % (written, f))
2112 |     	    return written
2113 |         except:
2114 |             _logException("Error opening file"+path)
2115 |             e = OSError("Error opening file"+path)
2116 |             e.errno = EINVAL
2117 |             raise e
2118 |     #@-node:write
2119 | 
2120 |     #@+node:release
2121 |     def release(self, path, flags):
2122 |         log_entry("gmailfs.py:Gmailfs:release: %s %x" % (path, int(flags)))
2123 | 	# I saw a KeyError get thrown out of this once.  Looking back in
2124 | 	# the logs, I saw two consecutive release:
2125 | 	# 01/20/10 17:47:47 INFO       gmailfs.py:Gmailfs:release: /linux-2.6.git/.Makefile.swp 32768
2126 | 	# 01/20/10 17:47:49 INFO       gmailfs.py:Gmailfs:release: /linux-2.6.git/.Makefile.swp 32769
2127 | 	#
2128 |         f = self.openfiles[path]
2129 | 	if f.close() == 0:
2130 | 		#write_out(f, "release")
2131 | 		# This write_out() is really slowing things down.
2132 | 		#
2133 | 		# Without it, there is a race:
2134 | 		# 1. write() and queue file in dirty writeout queue for block write
2135 | 		# 2. close(), and get in here
2136 | 		# 3. remove file from openfiles[]
2137 | 		# 4. new open(), and make a new OpenGmailFile created since
2138 | 		#    openfiles[] no longer has an entry
2139 | 		# 5. Write the same data block that is pending for write above...
2140 | 		#    we won't find the first one
2141 | 		#
2142 | 		# Do we need to make a link from inode->data blocks waiting for
2143 | 		# writeout?
2144 | 		del self.openfiles[path]
2145 |         return 0
2146 |     #@-node:release
2147 | 
2148 |     def get_quota_info(self):
2149 | 	# not really interesting because we don't care how much
2150 | 	# is in the entire account, just our particular folder
2151 | 	#resp, data = self.imap.getquota("")
2152 | 
2153 | 	# response looks like:
2154 | 	# [['"linux_fs_3" ""'], ['"" (STORAGE 368 217307895)']]
2155 | 	# numbers are in 1k-sized blocks
2156 | 	imap = self.get_imap()
2157 | 	resp, data = imap_getquotaroot(imap, fsNameVar)
2158 | 	self.put_imap(imap)
2159 | 	storage = data[1][0]
2160 | 	m = re.match('"" \(STORAGE (.*) (.*)\)', storage)
2161 | 	used_blocks = int(m.group(1))
2162 | 	allowed_blocks = int(m.group(2))
2163 | 	log_imap("quota resp: '%s'/'%s'" % (resp, data))
2164 | 	return [used_blocks * 1024, allowed_blocks * 1024]
2165 | 
2166 | 
2167 |     #@+node:statfs
2168 |     def statfs(self):
2169 | 	log_entry("statfs()")
2170 |         """
2171 |         Should return a tuple with the following 6 elements:
2172 |             - blocksize - size of file blocks, in bytes
2173 |             - totalblocks - total number of blocks in the filesystem
2174 |             - freeblocks - number of free blocks
2175 |             - availblocks - number of blocks available to non-superuser
2176 |             - totalfiles - total number of file inodes
2177 |             - freefiles - nunber of free file inodes
2178 | 
2179 |         Feel free to set any of the above values to 0, which tells
2180 |         the kernel that the info is not available.
2181 |         """
2182 |         st = fuse.StatVfs()
2183 |         block_size = 1024
2184 |         quotaBytesUsed, quotaBytesTotal = self.get_quota_info()
2185 |         blocks = quotaBytesTotal / block_size
2186 |         quotaPercent = 100.0 * quotaBytesUsed / quotaBytesTotal
2187 |         blocks_free = (quotaBytesTotal - quotaBytesUsed) / block_size
2188 |         blocks_avail = blocks_free # I guess...
2189 |         log_debug("%s of %s used. (%s)\n" % (quotaBytesUsed, quotaBytesTotal, quotaPercent))
2190 |         log_debug("Blocks: %s free, %s total\n" % (blocks_free, blocks))
2191 |         files = 0
2192 |         files_free = 0
2193 |         namelen = 80
2194 |         st.f_bsize = block_size
2195 | 	st.f_frsize = block_size
2196 | 	st.f_blocks = blocks
2197 | 	st.f_bfree = blocks_free
2198 | 	st.f_bavail = blocks_avail
2199 | 	st.f_files = files
2200 | 	st.f_ffree = files_free
2201 | 	return st
2202 |     #@-node:statfs
2203 | 
2204 |     #@+node:fsync
2205 |     def fsync(self, path, isfsyncfile):
2206 |         log_entry("gmailfs.py:Gmailfs:fsync: path=%s, isfsyncfile=%s" % (path, isfsyncfile))
2207 |         log_info("gmailfs.py:Gmailfs:fsync: path=%s, isfsyncfile=%s" % (path, isfsyncfile))
2208 |         inode = self.lookup_inode(path)
2209 |         f = self.openfiles[path]
2210 | 	write_out(inode, "fsync_inode")
2211 | 	#for block in inode._blocks:
2212 | 	#        write_out(block, "fsync_blocks")
2213 |         return 0
2214 |     #@-node:fsync
2215 | 
2216 |     #@+node:fsync
2217 |     def fsyncdir(self, path, isfsyncfile):
2218 |         log_entry("gmailfs.py:Gmailfs:fsyncdir: path=%s, isfsyncfile=%s" % (path, isfsyncfile))
2219 |         log_info("gmailfs.py:Gmailfs:fsyncdir: path=%s, isfsyncfile=%s" % (path, isfsyncfile))
2220 |         return -ENOSYS
2221 | 
2222 |     #@-node:fsync
2223 | 
2224 |     #@+node:fsync
2225 |     def flush(self, path):
2226 |         log_entry("gmailfs.py:Gmailfs:flush: path=%s" % (path))
2227 |         dirent = self.lookup_dirent(path)
2228 | 	#write_out(dirent, "flush")
2229 | 	#write_out(dirent.inode, "flush")
2230 |         return 0
2231 |     #@-node:fsync
2232 | 
2233 |     def fetch_dirent_msgs_for_path(self, dir_path):
2234 | 	log_debug2("fetch_dirent_msgs_for_path('%s')" % (dir_path))
2235 | 	encoded_path = _pathSeparatorEncode(dir_path)
2236 | 	q = "" + PathNameTag + '=' + PathStartDelim + encoded_path + PathEndDelim
2237 | 	about = ("dirent lookup('%s')" % (dir_path))
2238 |         dirent_msgids = _getMsguidsByQuery(about, self.imap, [q])
2239 | 	log_debug2("q: '%s'" % (q))
2240 | 	if len(dirent_msgids) == 0:
2241 | 		log_debug2("could not find messages for path: '%s'" % (dir_path))
2242 | 		return None
2243 | 	log_debug2("fetch_dirent_msgs_for_path('%s') got back '%d' responses" % (dir_path, len(dirent_msgids)))
2244 | 	return dirent_msgids
2245 | 
2246 |     def fetch_dirent_msg_for_path(self, path):
2247 | 	if self.dirent_cache.has_key(path):
2248 | 		return self.dirent_cache[path].dirent_msg
2249 | 	else:
2250 | 		log_debug2("fetch_dirent_msg_for_path('%s') missed the inode cache()" % (path))
2251 | 		for path, inode in self.dirent_cache.items():
2252 | 			log_debug3("in cache: '%s'" % (path))
2253 | 	dirent_msgids = fetch_dirent_msg_for_path(dirpath)
2254 | 	return dirent_msgids[0]
2255 | 
2256 |     inode_cache = {}
2257 |     inode_cache_lock = None
2258 |     def find_or_mk_inode(self, ino, msg):
2259 | 	ino = int(ino)
2260 | 	semget(self.inode_cache_lock)
2261 | 	if len(inode_cache) > 1000:
2262 | 		log_info("flushing inode cache")
2263 | 		new_inode_cache = {}
2264 | 		for ino, inode in self.inode_cache:
2265 | 			if inode.pinned < 1:
2266 | 				continue
2267 | 			new_inode_cache[ino] = inode
2268 | 		self.inode_cache = new_inode_cache
2269 | 	if self.inode_cache.has_key(ino):
2270 | 		inode = self.inode_cache[ino]
2271 | 	else:
2272 | 	        inode = GmailInode(msg, self)
2273 | 		self.inode_cache[ino] = inode
2274 | 	self.inode_cache_lock.release()
2275 | 	return inode
2276 | 
2277 |     def dirent_msg_iref(self, dirent_msg):
2278 | 	dirent_msg_hash = self.parse_dirent_msg(dirent_msg)
2279 | 	if dirent_msg_hash == None:
2280 | 	    	log_debug1("lookup_dirent() failed to parse dirent_msg for uid '%s'" % (dirent_msg.uid))
2281 | 		return None
2282 |        	return str(dirent_msg_hash[RefInodeTag])
2283 | 
2284 |     def get_inode(self, ino):
2285 | 	ino = int(ino)
2286 | 	semget(self.inode_cache_lock)
2287 | 	if not self.inode_cache.has_key(ino):
2288 | 		self.inode_cache_lock.release()
2289 | 		return None
2290 | 	inode = self.inode_cache[ino]
2291 | 	inode.pinned += 1
2292 | 	self.inode_cache_lock.release()
2293 | 	return inode
2294 | 
2295 |     def put_inode(self, inode):
2296 | 	semget(self.inode_cache_lock)
2297 | 	inode.pinned -= 1
2298 | 	self.inode_cache_lock.release()
2299 | 
2300 |     def mk_pinned_inode(self, msg):
2301 | 	subj_hash = self.parse_inode_msg_subj(msg)
2302 | 	ino = int(subj_hash[InodeTag])
2303 |         ret = None
2304 | 	semget(self.inode_cache_lock)
2305 | 	if self.inode_cache.has_key(ino):
2306 | 		ret = self.inode_cache[ino]
2307 | 		log_debug2("pinned new inode nr: '%s'" % (str(ret.ino)))
2308 | 	else:
2309 | 		ret = GmailInode(msg, self)
2310 | 		self.inode_cache[ret.ino] = ret
2311 | 		log_debug2("pinned new inode nr: '%s'" % (str(ret.ino)))
2312 | 	ret.pinned += 1
2313 | 	self.inode_cache_lock.release()
2314 | 	return ret
2315 | 
2316 |     def mk_pinned_inodes(self, msgs):
2317 | 	inodes = []
2318 | 	for uid, msg in msgs.items():
2319 | 		inode = self.mk_pinned_inode(msg)
2320 | 		inodes.append(inode)
2321 | 	return inodes
2322 | 
2323 |     def mk_iref_query(self, dirent_msgs):
2324 |         query = []
2325 |         inodes = []
2326 | 	dirent_msgs_by_iref = {}
2327 | 	for uid, dirent_msg in dirent_msgs.items():
2328 | 		iref = self.dirent_msg_iref(dirent_msg)
2329 | 		dirent_msgs_by_iref[iref] = dirent_msg
2330 | 		inode = self.get_inode(iref)
2331 | 		if not inode == None:
2332 | 			inodes.append(inode)
2333 | 			continue
2334 |     		query.append(InodeTag+'='+iref)
2335 | 	return dirent_msgs_by_iref, query, inodes
2336 | 
2337 |     def prefetch_dirent_msgs(self, dir):
2338 | 	log_debug3("prefetch_dirent_msgs() 0")
2339 |     	dirent_msgids = self.fetch_dirent_msgs_for_path(dir)
2340 |         if dirent_msgids == None:
2341 |             return None
2342 | 
2343 | 	log_debug2("prefetch_dirent_msgs('%s') going to fetch '%d' msgs" % (dir, len(dirent_msgids)))
2344 |     	dirent_msgs = fetch_full_messages(self.imap, dirent_msgids)
2345 | 	log_debug1("prefetch_dirent_msgs('%s') got back '%d' msgs" % (dir, len(dirent_msgs)))
2346 | 
2347 | 	dirent_msgs_by_iref, query, inodes = self.mk_iref_query(dirent_msgs)
2348 | 
2349 | 	if len(query):
2350 | 		inode_msguids = _getMsguidsByQuery("batch inode lookup", self.imap, query, 1)
2351 | 		i_msgs = fetch_full_messages(self.imap, inode_msguids)
2352 | 		inodes.extend(self.mk_pinned_inodes(i_msgs))
2353 | 
2354 | 	log_debug3("prefetch_dirent_msgs() end")
2355 | 	return dirent_msgs_by_iref
2356 | 
2357 |     def lookup_dirent(self, path):
2358 | 	dir, filename = parse_path(path)
2359 | 	# This cache checking is required at this point.  There
2360 | 	# are inodes in the cache that have not been written to
2361 | 	# storage, and will not show up when we do
2362 | 	# self.fetch_dirent_msgs_for_path(), we must get them
2363 | 	# from here.
2364 |         if self.dirent_cache.has_key(path):
2365 |             return self.dirent_cache[path]
2366 | 
2367 | 	# We don't want to be simultaneously prefetching the same
2368 | 	# messages in two different threads.  So, serialize the
2369 | 	# lookups for now.
2370 | 	semget(self.lookup_lock)
2371 | 	dirent_msgs_by_iref = self.prefetch_dirent_msgs(dir)
2372 | 	if dirent_msgs_by_iref == None:
2373 | 		self.lookup_lock.release()
2374 | 		return None
2375 | 
2376 |         ret_dirent = None
2377 | 	for iref, dirent_msg in dirent_msgs_by_iref.items():
2378 | 		iref = int(iref)
2379 | 		# no locking needed since we've already
2380 | 		# pinned it
2381 | 		if self.inode_cache.has_key(iref):
2382 | 			inode = self.inode_cache[iref]
2383 | 		else:
2384 | 			log_error("dirent_msg (%s) refers to ino=%d which was not fetched" % (dirent_msg.uid, iref))
2385 | 			log_error("dirent_msg subject: ->%s<-" % (dirent_msg['Subject']))
2386 | 			continue
2387 | 		new_dirent = GmailDirent(dirent_msg, inode, self)
2388 | 		log_debug2("cached dirent: '%s'" % (new_dirent.path()))
2389 | 	       	self.dirent_cache[new_dirent.path()] = new_dirent
2390 | 		if new_dirent.path() == path:
2391 | 			log_debug2("lookup_dirent() dirent: '%s'" % (new_dirent.path()))
2392 | 			ret_dirent = new_dirent
2393 | 	self.lookup_lock.release()
2394 |         return ret_dirent
2395 | 
2396 |     #@-others
2397 | 
2398 | #@-node:class Gmailfs
2399 | #@+node:mainline
2400 | 
2401 | # Setup logging
2402 | log = logging.getLogger('gmailfs')
2403 | #defaultLogLevel = logging.WARNING
2404 | defaultLogLevel = logging.DEBUG
2405 | log.setLevel(defaultLogLevel)
2406 | defaultLogFormatter = logging.Formatter("%(asctime)s %(levelname)-10s %(message)s", "%x %X")
2407 | 
2408 | # log to stdout while parsing the config while
2409 | defaultLoggingHandler = logging.StreamHandler(sys.stdout)
2410 | _addLoggingHandlerHelper(defaultLoggingHandler)
2411 | 
2412 | GmailConfig([SystemConfigFile,UserConfigFile])
2413 | try:
2414 |     libgmail.ConfigLogs(log)
2415 | except:
2416 |     pass
2417 | 
2418 | def main(mountpoint, namedOptions):
2419 |     log_debug1("Gmailfs: starting up, pid: %d" % (os.getpid()))
2420 |     global lead_thread
2421 |     lead_thread = thread.get_ident()
2422 |     if am_lead_thread():
2423 | 	    print "am lead thread"
2424 |     else:
2425 | 	    print "am NOT lead thread"
2426 |     server = Gmailfs(namedOptions,mountpoint,version="gmailfs 0.8.0",usage='',dash_s_do='setsingle')
2427 |     server.parser.mountpoint = mountpoint
2428 |     server.parse(errex=1)
2429 |     server.flags = 0
2430 |     #server.multithreaded = False;
2431 |     server.multithreaded = True;
2432 |     writeout_threads = []
2433 |     for i in range(server.nr_imap_threads):
2434 | 	    t = testthread(server, i)
2435 | 	    t.start()
2436 | 	    writeout_threads.append(t)
2437 |     server.main()
2438 |     global do_writeout
2439 |     do_writeout = 0
2440 |     for t in writeout_threads:
2441 | 	    print "joining thread..."
2442 | 	    t.join()
2443 | 	    print "done joining thread"
2444 |     log_info("unmount: flushing caches")
2445 |     server.flush_dirent_cache()
2446 |     imap_times_print(1)
2447 |     log_info("done")
2448 | 
2449 | if __name__ == '__main__':
2450 |     main(1, "2")
2451 | 
2452 | #@-node:mainline
2453 | #@-others
2454 | #@-node:@file gmailfs.py
2455 | #@-leo
2456 | 


--------------------------------------------------------------------------------