├── .gitignore
├── README.rst
└── mysqlfuse.py


/.gitignore:
--------------------------------------------------------------------------------
1 | DBG
2 | README.html
3 | 


--------------------------------------------------------------------------------
/README.rst:
--------------------------------------------------------------------------------
 1 | ******************************************************
 2 | mysqlfuse - Access a MySQL database like a file system
 3 | ******************************************************
 4 | 
 5 | This is an attempt to use FUSE to provide a view of a MySQL database as a
 6 | filesystem.  See the __doc__ string in the mysqlfuse.py file for an
 7 | explanation.
 8 | 
 9 | Currently it MOSTLY works.  But not everything really works quite right.
10 | This is v.0.00001alpha, mind you.
11 | 
12 | 
13 | Concept
14 | =======
15 | The root directory contains subdirs named for each table in the db.
16 | 
17 | Within each table, we look up the PRIMARY indexes (those have to be unique,
18 | right?).  Ummm... for the first cut, we'll take them in order given by
19 | Sequence (ideally we should be able to take them in all orders, see below).
20 | This will be better with an example::
21 | 
22 |     CREATE TABLE `testtab` (
23 |         `key1` varchar(10) default NULL,
24 |         `key2` varchar(7) default NULL,
25 |         `data1` text default NULL,
26 |         `data2` varchar(18) default NULL,
27 |         PRIMARY KEY (`key1`, `key2`)
28 |     )
29 | 
30 | This branch of mysqlfuse is going to have the directories alternating
31 | between keyname and keyvalue.  This should allow browsing by any ordering
32 | of keys.
33 | 
34 | (N.B. Handling NULL keys and different types of keys (int, varchar) is
35 | going to get tricky--actually it hasn't.) The above should lead to this::
36 | 
37 | 
38 |     .                      testtab
39 |                           /       \
40 |                          /         \
41 |                      key1           key2     (all the key fields)
42 |                _____/ |  \         / |  \
43 |               /       |   \       /  |   \
44 |              foo     ...  bar   baz ...  quux  (all the values for each field)
45 |               |            |     |         |
46 |               |            |     |         |
47 |              key2         key2  key1      key1  (the key fields not above each)
48 |            /  |  \       / | \ / |  \    / |  \
49 |           /   |   \     |  | | | |   |  |  |   \
50 |         baz  ...  quux baz ..  foo..bar foo...  bar  (values for those)
51 |        /   \
52 |      data1  data2  ...
53 | 
54 | 
55 | At the bottom level, the filenames are ``data1`` and ``data2``, i.e. all the
56 | *names* of the non-key fields, and each such file contains the *value* for
57 | that field in that row (note that each directory just above the bottom
58 | level contains a single row, assuming the keys have to be unique,
59 | i.e. primary).  Higher-level entries are all directories, not files.  They
60 | alternate between name-of-a-key-field and values-of-a-key-field.  For
61 | example, ``/books/book/My_Friend_Flicka/page/71/pagecontents``.
62 | 
63 | This way, the data can be accessed by any possible permutation of keys,
64 | since both ``/books/book/My_Friend_Flicka/page/71/`` and also
65 | ``/books/page/71/book/My_Friend_Flicka/`` are there.  This gets more fun when
66 | there are more elements in the key.
67 | 
68 | 
69 | 
70 | Usage
71 | =====
72 | ::
73 | 
74 |   $ mysqlfuse.py -o host=localhost,user=sqluser,passwd=password,db=databasename <mountpoint>
75 | 
76 | Debugging information can be found in the ``DBG`` file.
77 | 
78 | 
79 | Dependencies
80 | ============
81 | * fuse-python__
82 | * MySQL-Python__
83 | 
84 | __ https://pypi.python.org/pypi/fuse-python
85 | __ http://mysql-python.sourceforge.net/
86 | 
87 | On Debian, simply install ``python-fuse`` and ``python-mysql``::
88 | 
89 |   $ sudo apt-get install python-fuse python-mysql
90 | 


--------------------------------------------------------------------------------
/mysqlfuse.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # coding=utf-8
  3 | 
  4 | from fuse import Fuse, Stat
  5 | import fuse
  6 | import stat
  7 | from time import time
  8 | from subprocess import *
  9 | import os
 10 | import errno
 11 | import MySQLdb
 12 | from copy import copy
 13 | 
 14 | import sys
 15 | import pdb
 16 | 
 17 | 
 18 | """
 19 | So here's the general concept:
 20 | 
 21 | The root directory contains subdirs named for each table in the db.
 22 | 
 23 | Within each table, we look up the PRIMARY indexes (those have to be unique,
 24 | right?).  Ummm... for the first cut, we'll take them in order given by
 25 | Sequence (ideally we should be able to take them in all orders, see below).
 26 | This will be better with an example.
 27 | 
 28 | CREATE TABLE `testtab` (
 29 | 	`key1` varchar(10) default NULL,
 30 | 	`key2` varchar(7) default NULL,
 31 |         `data1` text default NULL,
 32 |         `data2` varchar(18) default NULL,
 33 |         PRIMARY KEY (`key1`, `key2`)
 34 | )
 35 | 
 36 | This branch of mysqlfuse is going to have the directories alternating
 37 | between keyname and keyvalue.  This should allow browsing by any ordering
 38 | of keys.
 39 | 
 40 | (N.B. Handling NULL keys and different types of keys (int, varchar) is
 41 | going to get tricky--actually it hasn't.) The above should lead to this:
 42 | 
 43 | 
 44 |                            testtab
 45 |                           /       \
 46 |                          /         \
 47 |                      key1           key2     (all the key fields)
 48 |                _____/ |  \         / |  \
 49 |               /       |   \       /  |   \
 50 |              foo     ...  bar   baz ...  quux  (all the values for each field)
 51 |               |            |     |         |
 52 |               |            |     |         |
 53 |              key2         key2  key1      key1  (the key fields not above each)
 54 |            /  |  \       / | \ / |  \    / |  \
 55 |           /   |   \     |  | | | |   |  |  |   \
 56 |         baz  ...  quux baz ..  foo..bar foo...  bar  (values for those)
 57 |        /   \
 58 |      data1  data2  ...
 59 | 
 60 | 
 61 | At the bottom level, the filenames are "data1" and "data2", i.e. all the
 62 | *names* of the non-key fields, and each such file contains the *value* for
 63 | that field in that row (note that each directory just above the bottom
 64 | level contains a single row, assuming the keys have to be unique,
 65 | i.e. primary).  Higher-level entries are all directories, not files.  They
 66 | alternate between name-of-a-key-field and values-of-a-key-field.  For
 67 | example, /books/book/My_Friend_Flicka/page/71/pagecontents.
 68 | 
 69 | This way, the data can be accessed by any possible permutation of keys,
 70 | since both /books/book/My_Friend_Flicka/page/71/ and also
 71 | /books/page/71/book/My_Friend_Flicka/ are there.  This gets more fun when
 72 | there are more elements in the key.
 73 | 
 74 | """
 75 | 
 76 | 
 77 | def getDepth(path):
 78 |     """
 79 |     Return the depth of a given path, zero-based from root ('/')
 80 |     """
 81 |     if path == '/':
 82 |         return 0
 83 |     else:
 84 |         return path.count('/')
 85 | 
 86 | def getParts(path):
 87 |     """
 88 |     Return the slash-separated parts of a given path as a list
 89 |     """
 90 |     if path == '/':
 91 |         return [['/']]
 92 |     else:
 93 |         return path.split('/')
 94 | 
 95 | def escape_for_fs(string):
 96 |     x=string
 97 |     x=x.replace("%","%%")
 98 |     x=x.replace("/","%/")
 99 |     return x
100 | 
101 | def unescape_from_fs(string):
102 |     x=string
103 |     x=x.replace("%/","/")
104 |     x=x.replace("%%","%")
105 |     return x
106 | 
107 | def escape_for_sql(string):
108 |     x=string
109 |     x=x.replace("'","''")
110 |     return x
111 | 
112 | def unescape_from_sql(string):
113 |     return string.replace("''","'")
114 | 
115 | 
116 | def make_criteria(elts):
117 |     # Elements are going to be alternating key/value pairs.
118 |     criteria=""
119 |     for i in range(0,len(elts),2): # Count by twos!
120 |         (key, name)=elts[i:i+2]
121 |         key=escape_for_sql(unescape_from_fs(key))
122 |         name=escape_for_sql(unescape_from_fs(name))
123 |         criteria+="`%s`='%s' AND "%(key,name)
124 |     if len(criteria)>5:
125 |         criteria=criteria[:-5]  # Chop off the final AND.
126 |     return criteria
127 | 
128 | class MyStat(Stat):
129 |     def __init__(self):
130 |         self.st_mode = stat.S_IFDIR | 0755
131 |         self.st_ino = 0
132 |         self.st_dev = 0
133 |         self.st_nlink = 2
134 |         self.st_uid = 0
135 |         self.st_gid = 0
136 |         self.st_size = 4096
137 |         self.st_atime = 0
138 |         self.st_mtime = 0
139 |         self.st_ctime = 0
140 | 
141 | 
142 | # decorator for debugging!
143 | def debugfunc(f):
144 |     def newf(*args, **kwargs):
145 |         MySQLFUSE.dbg.write(">>entering function %s\n"%f.__name__)
146 |         MySQLFUSE.dbg.flush()
147 |         x=f(*args, **kwargs)
148 |         MySQLFUSE.dbg.write("<<leaving function %s, returning %s\n"%
149 |                             (f.__name__, str(x)))
150 |         MySQLFUSE.dbg.flush()
151 |         return x
152 |     return newf
153 | 
154 | class MySQLFUSE(Fuse):
155 | 
156 |     dbg=open("DBG","w")
157 | 
158 |     def DBG(self,s):
159 |         MySQLFUSE.dbg.write(s+"\n")
160 |         MySQLFUSE.dbg.flush()
161 | 
162 |     @debugfunc
163 |     def __init__(self, *args, **kw):
164 |         Fuse.__init__(self, *args, **kw)
165 | 
166 |     @debugfunc
167 |     def fsinit(self):
168 |         # I assume that here I need to log into the db.
169 |         # Declare host/user/passwd/db in the -o options on the cmd line.
170 |         self.connection=MySQLdb.Connection(host=self.host,
171 |                                            user=self.user,
172 |                                            db=self.db,
173 |                                            passwd=self.passwd)
174 |         self.connection.autocommit(True);
175 |         self.cursor=self.connection.cursor()
176 |         self.dcursor=self.connection.cursor(MySQLdb.cursors.DictCursor)
177 |         # Build a data structure encoding things.  The indexes aren't
178 |         # allowed to change during a session, only their contents.
179 |         self.cursor.execute("SHOW TABLES")
180 |         self.tables=map((lambda x:x[0]), self.cursor.fetchall())
181 |         self.keys={}
182 |         self.fields={}
183 |         for table in self.tables:
184 |             self.cursor.execute("SHOW INDEXES FROM `%s` WHERE Key_name='PRIMARY'"%table)
185 |             ix=self.cursor.fetchall()
186 |             self.keys[table]=map((lambda x:x[4]), ix)
187 |             self.cursor.execute("SHOW COLUMNS FROM `%s`"%table)
188 |             fl=self.cursor.fetchall()
189 |             fl=map((lambda x:x[0]), fl)
190 |             # remove the key fields?
191 |             for k in self.keys[table]:
192 |                 fl.remove(k)
193 |             self.fields[table]=fl
194 |         self.DBG(str(self.keys))
195 | 
196 |     def is_root(self, path=None, pathelts=None):
197 |         if pathelts is None:
198 |             pathelts=getParts(path)[1:]
199 |         if path=='/' or len(pathelts)==0:
200 |             return True
201 |         else:
202 |             return False
203 | 
204 |     def is_directory(self, path=None, pathelts=None):
205 |         # Something is a directory iff it isn't deep enough to be a file.
206 |         # In order to be a file, the depth of the path must equal 1(for the
207 |         # table) plus the number of keys for that table *times two* (key
208 |         # and value), plus 1 more to get down to the file level.
209 |         if not pathelts:
210 |             pathelts=getParts(path)[1:]
211 |         if self.is_root(pathelts=pathelts):
212 |             # Root dir.  Special case.
213 |             return True
214 |         table=pathelts[0]
215 |         try:
216 |             keys=self.keys[table]
217 |         except KeyError:
218 |             # We don't have that table!  Fake it.
219 |             self.DBG("Trying to look up keys for non-existent table %s!"%table)
220 |             return False
221 |         if len(pathelts) < 2*len(keys)+2:
222 |             return True
223 |         else:
224 |             return False
225 | 
226 |     @debugfunc
227 |     def getattr(self, path):
228 |         st = MyStat()
229 |         pe = getParts(path)[1:]
230 | 
231 |         if self.is_root(pathelts=pe):
232 |             return st
233 |         table=escape_for_sql(unescape_from_fs(pe[0]))
234 |         if not table in self.tables:
235 |             return -fuse.ENOENT
236 |         if len(pe)==1:      # It's a table
237 |             return st
238 |         query="SELECT COUNT(*) from `%s` "%table
239 |         if self.is_directory(path):
240 |             # Which KIND of directory is it??
241 |             d=getDepth(path)
242 |             if d%2==1:
243 |                 # This is a keyvalue dir.
244 |                 criteria=make_criteria(pe[1:])
245 |             else:
246 |                 criteria=make_criteria(pe[1:-1])
247 |             if criteria:
248 |                 query+="WHERE %s"%criteria
249 |             self.DBG(query)
250 |             self.cursor.execute(query)
251 |             if self.cursor.fetchone()[0]<1:
252 |                 return -fuse.ENOENT
253 |             return st
254 |         else:
255 |             # Otherwise, it's a "file", i.e. an actual field.
256 |             st.st_mode = stat.S_IFREG | 0666
257 |             st.st_nlink = 1
258 |             self.DBG("getattr for a file: %s"%path)
259 |             # Oh what the hell.  Yes, we query EACH TIME.
260 |             self.DBG("about to make criteria: %s"%str(pe[1:-1]))
261 |             criteria=make_criteria(pe[1:-1])
262 |             query="SELECT length(`%s`) FROM `%s` WHERE "\
263 |                 "%s"%(escape_for_sql(unescape_from_fs(pe[-1])),
264 |                       table, criteria)
265 |             self.DBG(query)
266 |             n=self.cursor.execute(query)
267 |             if n<=0:
268 |                 return -fuse.ENOENT
269 |             sz=self.cursor.fetchone()[0]
270 |             if sz is None:
271 |                 sz=0
272 |             else:
273 |                 sz=int(sz)
274 |             st.st_size = sz
275 |         return st
276 | 
277 | 
278 |     @debugfunc
279 |     def readdir(self, path, offset):
280 |         dirents=['.', '..']
281 |         pe=path.split('/')[1:]
282 |         if path=='/':
283 |             # Populate root dir
284 |             dirents.extend(self.tables)
285 |         else:
286 |             # OK.  Apart from root (above) we are either at a "keyname"
287 |             # directory or a "keyvalue" directory.  That is, this dir is
288 |             # either named "book" or "My Friend Flicka", etc.  I also know
289 |             # all the keys (if any) specified above this level.  If I am
290 |             # reading a directory named with a keyname, its contents will
291 |             # be dirs whose names are values that are available for this
292 |             # field, searching on whatever is specified between here and
293 |             # root (really tabledir).  If this directory is named with a
294 |             # keyvalue, then the contents of this dir will be all the
295 |             # keynames not already listed on the path from tabledir to
296 |             # here.
297 |             #
298 |             # Unless, of course, we are at a bottom-level directory, in
299 |             # which case the contents are files named after the non-key
300 |             # fields in the DB.
301 |             table=escape_for_sql(unescape_from_fs(pe[0]))
302 |             d=getDepth(path)
303 |             # If depth==0, we're at root, we already did that.  If
304 |             # depth==1, we're at tabledir, reading keynames.  tabledir
305 |             # counts as a keyvalue dir.  If depth==3, we are at a keyvalue
306 |             # dir after the first key, and so forth.
307 |             self.DBG("Depth: %d"%d)
308 |             if d%2==1:
309 |                 # Odd.  Read keynames.
310 |                 if not self.is_directory(path+"/foo"):
311 |                     # One level deeper is files!  This is a bottom-level dir.
312 |                     # This has to be an odd-numbered one, a keyvalued dir.
313 |                     self.DBG("files now.")
314 |                     dirents.extend(self.fields[table])
315 |                 else:
316 |                     all_keys=copy(self.keys[table])
317 |                     self.DBG("all keynames: %s"%str(all_keys))
318 |                     for i in range(1,d,2): # Every other dir: the keynames.
319 |                         self.DBG("Removing %s"%pe[i])
320 |                         all_keys.remove(pe[i])
321 |                     # Whatever is left is what goes here.
322 |                     dirents.extend(all_keys)
323 |             else:
324 |                 # Even.  Read in keyvalues.  I need to use criteria here.
325 |                 self.DBG("Keyvalues.  pe = %s"%str(pe))
326 |                 self.DBG("About to make criteria from %s"%str(pe[1:-1]))
327 |                 criteria=make_criteria(pe[1:-1])
328 |                 self.DBG("criteria: %s"%criteria)
329 |                 query="SELECT DISTINCT `%s` FROM `%s` "%(pe[-1],table)
330 |                 if criteria:
331 |                     query+="WHERE "+criteria
332 |                 self.DBG(query)
333 |                 self.cursor.execute(query)
334 |                 l=self.cursor.fetchall()
335 |                 self.DBG("returned: %s"%str(l))
336 |                 # Mustn't forget to convert to string
337 |                 l=map((lambda x:str(x[0])), l) 
338 |                 dirents.extend(l)
339 |         for r in dirents:
340 |             yield fuse.Direntry(r)
341 | 
342 |     @debugfunc
343 |     def mknod(self, path, mode, dev):
344 |         # I don't know that we need this.
345 |         return 0
346 | 
347 |     @debugfunc
348 |     def unlink(self,path):
349 |         # You can only remove a "directory"
350 |         # So this doesn't really do anything.
351 |         return 0
352 | 
353 |     @debugfunc
354 |     def write(self, path, buf, offset):
355 |         pe=getParts(path)[1:]
356 |         # We ought to be at the bottom, at the "file" level.
357 |         if self.is_directory(path):
358 |             # We shouldn't be writing here.  Just nod and smile.
359 |             return len(buf)
360 |         table=pe[0]
361 |         criteria=make_criteria(pe[1:-1])
362 |         field=escape_for_sql(unescape_from_fs(pe[-1]))
363 |         data=escape_for_sql(buf)
364 |         # Ignoring offset crap for now.
365 |         query="UPDATE `%s` SET `%s`='%s' "%(table, field, data)
366 |         if criteria:
367 |             query+="WHERE %s"%criteria
368 |         self.DBG(query)
369 |         self.cursor.execute(query)
370 |         return len(buf)
371 | 
372 |     @debugfunc
373 |     def read(self, path, size, offset):
374 |         pe=getParts(path)[1:]
375 |         if self.is_directory(path):
376 |             # We shouldn't be here.
377 |             return ""
378 |         table=pe[0]
379 |         criteria=make_criteria(pe[1:-1])
380 |         field=pe[-1]
381 |         query="SELECT `%s` FROM `%s` "%(field,table)
382 |         if criteria:
383 |             query +="WHERE %s"%criteria
384 |         self.DBG(query)
385 |         self.cursor.execute(query)
386 |         x=self.cursor.fetchone()
387 |         self.DBG(str(x))
388 |         data=str(x[0])
389 |         # doesn't matter if we're on the wrong level of the tree.
390 |         return data[offset:offset+size]
391 | 
392 | 
393 |     @debugfunc
394 |     def mkdir(self, path, mode):
395 |         pe=getParts(path)[1:]
396 |         table=pe[0]
397 |         keys=''
398 |         seen=[]
399 |         values=''
400 |         # We can't make a keyname directory; those are fixed by the DB
401 |         # structure.  If someone asks, fail.
402 |         d=getDepth(path)
403 |         if d==1:
404 |             # Trying to create a table.  n00b.
405 |             return -fuse.ENOTDIR
406 |         if d%2==0:
407 |             # Trying to create a keyname dir.
408 |             return -fuse.ENOTDIR
409 |         for i in range(1,d,2):
410 |             (key, value)=pe[i:i+2]
411 |             value=escape_for_sql(unescape_from_fs(value))
412 |             seen.append(key)
413 |             keys+="`"+key+"`,"
414 |             values+="'%s',"%value
415 |         # Make sure we have all the required keys.
416 |         for key in self.keys[table]:
417 |             if key in seen:
418 |                 continue
419 |             keys+="`"+key+"`,"
420 |             values+="'',"
421 |         # Chop off the comma
422 |         keys=keys[:-1]
423 |         values=values[:-1]
424 |         query="REPLACE INTO `%s` (%s) VALUES (%s)"%(table, keys, values)
425 |         self.DBG(query)
426 |         self.cursor.execute(query)
427 |         return 0
428 | 
429 |     @debugfunc
430 |     def release(self, path, flags):
431 |         # nothing to do
432 |         return 0
433 | 
434 |     @debugfunc
435 |     def open(self, path, flags):
436 |         return 0
437 | 
438 |     @debugfunc
439 |     def truncate(self, path, size):
440 |         return 0
441 | 
442 |     @debugfunc
443 |     def utime(self, path, times):
444 |         return 0
445 | 
446 |     @debugfunc
447 |     def rmdir(self, path):
448 |         # You can drop a directory, if you really want to I guess.
449 |         if self.is_root(path):
450 |             return 0            # Shyeah right.  Not gonna DROP DATABASE.
451 |         pe=getParts(path)[1:]
452 |         table=pe[0]
453 |         criteria=make_criteria(pe[1:])
454 |         # I think trying to delete a table will only delete its contents,
455 |         # not the actual table.
456 |         # That violates Least Astonishment; may have to change that.
457 |         query="DELETE FROM `%s` "%table
458 |         if criteria:
459 |             query+="WHERE %s"%criteria
460 |         self.DBG(query)
461 |         self.cursor.execute(query)
462 |         return 0
463 | 
464 |     @debugfunc
465 |     def rename(self, pathfrom, pathto):
466 |         return 0
467 | 
468 |     @debugfunc
469 |     def fsync(self, path, isfsyncfile):
470 |         return 0
471 | 
472 | 
473 | fuse.fuse_python_api=(0,2)
474 | 
475 | def main():
476 |     usage="""
477 |           surveyfs:
478 |       """ + fuse.Fuse.fusage
479 | 
480 |     server = MySQLFUSE(version="%prog " + fuse.__version__,
481 |                        usage=usage, dash_s_do='setsingle')
482 |     server.parser.add_option(mountopt="host", default="localhost")
483 |     server.parser.add_option(mountopt="user", default=os.environ['USER'])
484 |     server.parser.add_option(mountopt="passwd", default=None)
485 |     server.parser.add_option(mountopt="db", default="test")
486 | 
487 |     server.parse(errex=1, values=server)
488 |     # Parse, *then* initialize.
489 |     server.fsinit()
490 |     server.main()
491 | 
492 | if __name__ == '__main__':
493 |     main()
494 | 


--------------------------------------------------------------------------------