├── .gitignore ├── fusetree ├── __init__.py ├── util.py ├── types_conv.py ├── types.py ├── nodetypes.py ├── core.py └── fusetree.py ├── setup.py ├── LICENSE ├── README.md └── examples └── testfusetree.py /.gitignore: -------------------------------------------------------------------------------- 1 | /dist 2 | /build 3 | *.egg-info 4 | .mypy_cache 5 | __pycache__ 6 | -------------------------------------------------------------------------------- /fusetree/__init__.py: -------------------------------------------------------------------------------- 1 | from fusetree.types import Stat, StatVFS, Path 2 | from fusetree.types import Bytes_Like, Stat_Like, StatVFS_Like, Node_Like, FileHandle_Like, DirEntry, DirHandle_Like 3 | from fusetree.core import Node, DirHandle, FileHandle 4 | from fusetree.fusetree import FuseTree 5 | 6 | from fusetree.nodetypes import BaseSymlink, BaseFile, BaseDir, Symlink, BlobFile, GeneratorFile, generatorfile, HttpFile, DictDir 7 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name = 'fusetree', 5 | packages = ['fusetree'], 6 | version = '0.1.0', 7 | description = 'High-level API for fusepy', 8 | author = 'Paulo Costa', 9 | author_email = 'me@paulo.costa.nom.br', 10 | url = 'https://github.com/paulo-raca/fusetree', 11 | download_url = 'https://github.com/paulo-raca/fusetree', 12 | keywords = ['fuse'], 13 | install_requires = [ 14 | "fusepy", 15 | "aiohttp" 16 | ] 17 | ) 18 | -------------------------------------------------------------------------------- /fusetree/util.py: -------------------------------------------------------------------------------- 1 | import fuse 2 | import logging 3 | import traceback 4 | 5 | class LoggingFuseOperations(fuse.Operations): 6 | log = logging.getLogger('fusetree') 7 | 8 | def __init__(self, operations): 9 | self.operations = operations 10 | 11 | def __call__(self, op, path, *args): 12 | self.log.debug('-> %s %s %s', op, path, repr(args)) 13 | ret = '[Unhandled Exception]' 14 | try: 15 | ret = self.operations.__call__(op, path, *args) 16 | return ret 17 | except OSError as e: 18 | #traceback.print_exc() 19 | ret = str(e) 20 | raise 21 | finally: 22 | self.log.debug('<- %s %s', op, '%d bytes' % len(ret) if isinstance(ret, bytes) else repr(ret)) 23 | 24 | def is_iterable(x): 25 | try: 26 | iter(x) 27 | return True 28 | except: 29 | return False 30 | 31 | 32 | def is_async_iterable(x): 33 | return hasattr(x, '__anext__') or hasattr(x, '__aiter__') 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fusetree 2 | A better Fuse API for Python 3 3 | 4 | ## Motivation: 5 | I don't like Fuse's "high-level" API: 6 | 1. Having to parse a file's path at every call doesn't make sense except in the context of a "Hello World". 7 | 2. Related to above, faving to dispatch calls to different files from the same callbacks requires custom "routing" logic, which is meh. 8 | 3. Threads are cool, but AsyncIO is would be way better ;) 9 | 10 | Fuse's The "low-level" API is much more fun, but: 11 | - The callbacks-based API is nice, but require extra work on non-trivial (slow) calls. 12 | - It also requires custom "routing" logic to make the same callbacks reach the correct file handlers. 13 | - You have to manually keep track of inodes and etc 14 | - It is, after all, too low-level for pratical purposes. 15 | 16 | ## Fusetree 17 | 18 | Fusetree tries to solve these by providing an object-oriented implementation based on asyncio. 19 | 20 | - Each inode is an instance of `Node`. Inode numbers are tracked automatically and don't require any attention from the programmer. 21 | - There is basically 1:1 mapping from fuse methods to `Node` methods, you can override whatever makes sense. 22 | - There are reasonable base classes for files, directories and symlinks. 23 | - There are various implementations for several common node types: Constant blob, dict-backed directory, files that perform an HTTP download, etc. 24 | 25 | - Automatic convertions that do the right thing: 26 | - `str`/`bytes` automatically becomes a read-only file 27 | - `dict` become a folder 28 | - `iterable` becomes a non-seekable read-only file 29 | 30 | # Examples 31 | 32 | Please take a look at [examples](examples) 33 | 34 | -------------------------------------------------------------------------------- /fusetree/types_conv.py: -------------------------------------------------------------------------------- 1 | from .types import * 2 | from . import core 3 | from . import nodetypes 4 | from . import util 5 | 6 | import fuse 7 | import errno 8 | 9 | def as_bytes(data: Bytes_Like) -> bytes: 10 | if data is None: 11 | return b'' 12 | elif isinstance(data, bytes): 13 | return data 14 | elif isinstance(data, str): 15 | return data.encode('utf-8') 16 | else: 17 | return str(data).encode('utf-8') 18 | 19 | def as_node(node: Node_Like) -> 'core.Node': 20 | if node is None: 21 | raise fuse.FuseOSError(errno.ENOENT) 22 | elif isinstance(node, core.Node): 23 | return node 24 | elif isinstance(node, bytes) or isinstance(node, str): 25 | return nodetypes.BlobFile(as_bytes(node)) 26 | elif isinstance(node, dict): 27 | return nodetypes.DictDir(node) 28 | elif util.is_iterable(node) or util.is_async_iterable(node): 29 | return nodetypes.GeneratorFile(node) 30 | 31 | raise fuse.FuseOSError(errno.EIO) 32 | 33 | def as_filehandle(node: 'core.Node', filehandle: FileHandle_Like) -> 'core.FileHandle': 34 | if isinstance(filehandle, core.FileHandle): 35 | return filehandle 36 | elif isinstance(filehandle, bytes) or isinstance(filehandle, str): 37 | return nodetypes.BlobFile.Handle(node, as_bytes(filehandle)) 38 | elif util.is_iterable(filehandle) or util.is_async_iterable(filehandle): 39 | return nodetypes.GeneratorFile.Handle(node, filehandle) 40 | 41 | raise fuse.FuseOSError(errno.EIO) 42 | 43 | def as_dirhandle(node: 'core.Node', dirhandle: DirHandle_Like) -> 'core.DirHandle': 44 | if isinstance(dirhandle, core.DirHandle): 45 | return dirhandle 46 | elif util.is_iterable(dirhandle): 47 | return nodetypes.DictDir.Handle(node, dirhandle) 48 | raise fuse.FuseOSError(errno.EIO) 49 | 50 | def as_stat(stat: Stat_Like) -> Stat: 51 | if isinstance(stat, int): 52 | return Stat(st_mode=stat) 53 | if isinstance(stat, Stat): 54 | return stat 55 | elif isinstance(stat, dict): 56 | return Stat(**stat) 57 | raise fuse.FuseOSError(errno.EIO) 58 | 59 | def as_statvfs(statvfs: StatVFS_Like) -> StatVFS: 60 | if isinstance(statvfs, StatVFS): 61 | return statvfs 62 | elif isinstance(statvfs, dict): 63 | return StatVFS(**statvfs) 64 | raise fuse.FuseOSError(errno.EIO) 65 | -------------------------------------------------------------------------------- /fusetree/types.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Iterable, Sequence, Tuple, NamedTuple, Union, Any 2 | 3 | class Stat(NamedTuple): 4 | st_dev: int = None 5 | st_ino: int = None 6 | st_nlink: int = None 7 | st_mode: int = None 8 | st_uid: int = None 9 | st_gid: int = None 10 | st_rdev: int = None 11 | st_size: int = None 12 | st_blksize: int = None 13 | st_blocks: int = None 14 | st_flags: int = None 15 | st_gen: int = None 16 | st_lspare: int = None 17 | st_qspare: int = None 18 | st_atime: float = None 19 | st_mtime: float = None 20 | st_ctime: float = None 21 | st_birthtime: float = None 22 | 23 | def with_values(self, **kwargs): 24 | values = self.as_dict() 25 | values.update(kwargs) 26 | return Stat(**values) 27 | 28 | def as_dict(self) -> dict: 29 | return { 30 | k: v 31 | for (k, v) in self._asdict().items() 32 | if v is not None 33 | } 34 | 35 | class StatVFS(NamedTuple): 36 | f_bsize: int = None 37 | f_frsize: int = None 38 | f_blocks: int = None 39 | f_bfree: int = None 40 | f_bavail: int = None 41 | f_files: int = None 42 | f_ffree: int = None 43 | f_favail: int = None 44 | f_fsid: int = None 45 | f_flag: int = None 46 | f_namemax: int = None 47 | 48 | def as_dict(self) -> dict: 49 | return { 50 | k: v 51 | for (k, v) in self._asdict().items() 52 | if v is not None 53 | } 54 | 55 | class Path: 56 | def __init__(self, elements: Sequence[Tuple[str, 'core.Node']]) -> None: 57 | self.elements = elements 58 | 59 | @property 60 | def target_node(self): 61 | return self.elements[-1][1] 62 | 63 | @property 64 | def parent_path(self): 65 | return Path(self.elements[:-1]) 66 | 67 | @property 68 | def parent_node(self): 69 | return self.elements[:-2][1] 70 | 71 | def __str__(self): 72 | return '/'.join([name for name, node in self.elements]) 73 | 74 | def __repr__(self): 75 | return 'Path([' + repr 76 | 77 | 78 | Bytes_Like = Union[bytes, str] 79 | Stat_Like = Union[Stat, dict, int] 80 | StatVFS_Like = Union[StatVFS, dict] 81 | Node_Like = Union['core.Node', Bytes_Like, Dict[str, Any]] 82 | FileHandle_Like = Union['core.FileHandle', Bytes_Like, Iterable[Union[str, bytes]]] 83 | DirEntry = Union[str, Tuple[str, Node_Like]] 84 | DirHandle_Like = Union['core.DirHandle', Iterable[str], Iterable[Tuple[str, int]]] 85 | 86 | from . import core 87 | -------------------------------------------------------------------------------- /examples/testfusetree.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append("/media/Paulo/Workspace/spotify/fusetree") 3 | 4 | 5 | import fusetree 6 | import logging 7 | from stat import S_IFDIR, S_IFLNK, S_IFREG 8 | import time 9 | import fuse 10 | 11 | @fusetree.generatorfile 12 | def count(n=None): 13 | i = 1 14 | while n is None or i <= n: 15 | yield str(i) + '\n' 16 | i += 1 17 | 18 | @fusetree.generatorfile 19 | async def bottles(n): 20 | for i in range(n, 0, -1): 21 | yield \ 22 | f"""\ 23 | {i} {"bottle" if i == 1 else "bottles"} of beer on the wall 24 | {i} {"bottle" if i == 1 else "bottles"} of beer 25 | Take one down, pass it around 26 | {i-1} {"bottle" if i - 1== 1 else "bottles"} of beer on the wall 27 | 28 | """ 29 | 30 | rootNode = { 31 | 'file': 'The content of this file is hardcoded and cannot be edited ¯\_(ツ)_/¯\n', 32 | 'dir':{'README': 'The content of this directory is hardcoded and cannot be modified'}, 33 | 'editable_file': fusetree.BlobFile(b'You can edit this file!\n', rw=True), 34 | 'editable_dir': fusetree.DictDir({'README': fusetree.BlobFile(b'You can edit the contents of this directory.\n\nTry to create files, folders, symlinks, hardlinks, rename and delete them.\n', rw=True)}, rw=True), 35 | 36 | # https://commons.wikimedia.org/wiki/File:JPEG_example_JPG_RIP_001.jpg 37 | 'blob.jpg': b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01\x00H\x00H\x00\x00\xff\xdb\x00C' 38 | + b'\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' 39 | + b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' 40 | + b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' 41 | + b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xdb\x00C\x01\xff\xff\xff\xff\xff\xff\xff' 42 | + b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' 43 | + b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' 44 | + b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' 45 | + b'\xff\xc0\x00\x11\x08\x00\xea\x019\x03\x01"\x00\x02\x11\x01\x03\x11\x01\xff' 46 | + b'\xc4\x00\x17\x00\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' 47 | + b'\x00\x00\x01\x02\x03\xff\xc4\x00$\x10\x01\x01\x01\x00\x02\x01\x04\x03\x01\x01' 48 | + b'\x01\x01\x00\x00\x00\x00\x00\x01\x11!1A\x02\x12Qqa\x81\x91\xb1\xa1\xc1\xf0' 49 | + b'\xff\xc4\x00\x15\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' 50 | + b'\x00\x00\x00\x01\xff\xc4\x00\x16\x11\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00' 51 | + b'\x00\x00\x00\x00\x00\x00\x00\x11\x01\xff\xda\x00\x0c\x03\x01\x00\x02\x11\x03' 52 | + b'\x11\x00?\x00\xca\xe2/ \x8a\x82\r`C\x00\xa0\xb3\x81S0\xe7n\xb5\xeacA\xa9>\x173' 53 | + b'\x12V\xa81e\xd4]<\xf2\x08\xbc~V\xcf1\x9d\x06\xb8\xa9\x86\xd4\xde\x01x6f2\x80' 54 | + b'\xd6\x9a\x8a\rA\x99q\xad\xdf\xa0\x17\xa4\xf2\x01o\xf0\x87\xb7|\xa7@\xd6\xa6' 55 | + b'\xfc\xb3\xb8h/\xa7\x8bV\xdeu\x16\xc0\'\xaa]\xd4\xb7S;\x01f\xdf\x94\xb3\x03' 56 | + b'\x90C\x17\xfc^\xbfb2\xbf\xa4\x00\xc5@\x05\r\x03\xa6\xa5a\xa8\n\x93\xbd\xa6\xfe' 57 | + b'?i\xe0\x1a\xb6|0\xd2`\xa2\xcd\\\xf3K\xa0q?5>\x91`-\xf5p\x9cY\xdf%\x953\xe6\x81' 58 | + b'b/\xd2\xd9\x98"\x02\xc9\xd8\'\x1eC\xdb[\xcf\x9a\xaa\xe6\xdc\xf4\xdf\xa5\x040' 59 | + b'\xc8\x00\xbc%\x9b\xd2(9\xe2\xe3Vk2\xe5\x15V_\x94\xcf\xdc=\xbf\xfdPZ\xb6lg\x99' 60 | + b'\xc5\xabgK\xaby\x07=\xbf+}_' 67 | + b'\x10\xc5\xd9\x00\x17\xb4\x00\\@\x0b4\x01\x9c\xb0\x92\xf9n%\xbc\xa2\xa6\'\xda' 68 | + b'\xb3\x9d\x83s,c\x95\x9c\x1e\x00\xed1\xae/\xe1s\xec\x198B\x88\xbb\xa9\xd2\xc9' 69 | + b'\xe4\xe8U\x9c\x96_\x0b\xc2\x83\x9d\xe0[\xcdPC\xb6\x934\t\xf9tfI>\xcdT]DP\x01A' 70 | + b'\x0c\tt\x0bdOw=\x16y\x84\xf4\xe04\xca\xdb#2\x82\x9b\x9d\x8bg\x00\xac\xfbc<' 71 | + b'\xc3h7\xb23\xeed\x05\xdb\xe0\x8b\x0c\x01@\x02\xcdU\x063\x8f\xca\xfagz\xbcE' 72 | + b'\xd4S#9<4t#\x16\x1e\xea\xb6\xa6\x8a\x1d\x154\x1a\xd2\xe3\x1a*7\xc45\x04T$\xdf*' 73 | + b'\xb2\x02\xc8\xa8\x8a\x80\x00*\x00\xd2$\xbe?\x8d\x02\'\xe6(\x0b\xac\xfa\xad:' 74 | + b'\xe7\xfa\xa0\xe6\xb1l\xca\x00\x96\xd5\x00\x86\x12(&(\xa0\x82\xb3\xee\xf8\x06' 75 | + b'\x8e#\x1b@_r[h\x02\x1a/\x08\xb8\xb7\xd5\xf8Ow\xe0\xd8\x01m\xa9\xca\xd4Q\xaa' 76 | + b'\xcf\xdbL\xd4\x0c\t\xaa\xa0M\x16D\x16O5U\x15\x10\x00\x00\x01P\x02\xf2K\xe2' 77 | + b'\x87`\xd2+\x13\x9d\x06\xab;gIt\x80\x8d&(\x00\xb8\x02\xe1\xc4\xed\x9bA\xae#>\xef' 78 | + b'\x8e\x10\x00\x00\x00\x00\x10\x05\xb2\xf7\x88\xb6\xd0H\x0b&\xa2\x92\xfc\xb5\xfa' 79 | + b'\xbf\xc2z~kY\x01\xc8\x05\x06\xb8\xac\x80\xd1\xa9)y\xa85\xaa\xce*\xa0\x00\x00' 80 | + b'\x00L\x0e\xc1\xa4gj]\xbd\x82\xdb\xe2\x7fR\x00-E\x00RB\xd9\x01Y\xbe\xaf\x84\xb6' 81 | + b'\xd0\x00\x00\x00\x00@Q\x00\x00\x00\x01F\xe6\xf8fMt\xfc@?\xea\xa5\xd98\xf0\xc7' 82 | + b'\xbf\xf0"\x06|\x88\xab\xc1\x93\xe0\x89\xa0%\xe2\x8b\xd8\x1a\xa9\x9f\xea\xaa(@' 83 | + b'\x00\x01\x15\x00\x11\xa4\x04U0\x05\xfbM\x91\x9d\xd0[~\x10\x00\x00\x01\x14\x04' 84 | + b'\x00T\x00\x00\x00\x00\x11VI|\x82\xcf\xf8\xde\xc95\x8c\xbe9.\xf1\xc0\xad\xead' 85 | + b'\xf8OWLm\xf9\xa0\xba\xac\x98\x82\xa6,\x012\x80\xa84\xcbP\x05@\x15\x17\xc2\x00' 86 | + b'\x00\x02\x80B\xdf\x84\xb7P\x00\x00\x04\x05@\x00\x00\x00\x00\x10\x14@\x05@\x05' 87 | + b'\xd4PkS\xdcc(\xad\xdb>|3\xc0\xb8\xa2\xe365zN\x10g\x96\xb4\xd6Tj\xd8\x80"\x80' 88 | + b'\n"\x80\xa8\x02\x81\xb8\x0b\xd3\x16\xe9\xd8\x08\xa0\x02\x00\x00\x00\x00\x00' 89 | + b'\x00\x8aX\x08\x0b\xd0 \x00\x00\x02\xe2\x00\xd0\x8a\x8a\x83K\x80\xcdB\xac\xb8' 90 | + b'\xa1\x996\x9f\x92\xddD\x17t\x99\xe5\x95T[\x89\xa7 \xab\xc0\x8b\x97\xe1\x11DU' 91 | + b'\reP\x05\x10\x00\x00\x00\x00\x00\x04\x05E\\\x80\xca\xe9`\x02\xa0\x8a\x94_\x08' 92 | + b'\xa8\xbb\x04\x80/f\x1a\x80\xbe"\xeb*\n\xbb\xf9dE\x15:MQ\xb8\x97\xf0H\xb3\x8e' 93 | + b'\x90ab\xe1s\xc2\xa1\x86+6"\x9a\xba\x8b\x01\x1a/H\n\xcbL\xd5@\x00\x00\x00\x00' 94 | + b'\x15\r\x15q0@]A\xa0"\xa1\xa8*\x12\x80\xcd\x14TA@0\xc6\x84\xab\x19\x1ad@\x0f' 95 | + b'\xea\x8a\x8b\x0e/\xda\t\xa6\xd3\x11EVZ\xd1U*J \xa1\x13\xa0j2\x00\xb0\xa8\xaa' 96 | + b'\x88\xa2\x00\x08\x02\xa2\x82*\x02\xa8\xa2\x084\x94\x11n \xa8\x00\x00\x02\x8dFW' 97 | + b'\xf6\x82\xab\x1f\xb6\xc1-ej\x00\x02\xa0\x00\x08\xaa\x0c\xa8\x00\x1eJ\x8a\xa9' 98 | + b'\xdf\xd9\x0b\xd8\n\x8b:Q<\xac@E@\x04X\x00\xb8b\x82\xb3B\xf6\x01\xa6\xa2\x82' 99 | + b'\xe9\xe1<\x1e\x11\x01E\x04P\x04\n\x02(\x02\xed"|\x80*\x00\x00?\xff\xd9', 100 | 'emoji': '😘☺😗😄😝👻🙈🙉🙊👶👦👧🙆💁👰👼👯🛀🏄🚴💪💏👭✌👍👊🙏👅👄💋💞💥💦💨👙👑💍💎🐼🐰🐘🐬🐟🌻🍟🍕🎂🍰🍫🍬🍭\n', 101 | '☺': fusetree.Symlink('emoji'), 102 | 'array': [ 103 | 'Mary had a little lamb,\n', 104 | 'Its fleece was white as snow;\n', 105 | 'And everywhere that Mary went,\n', 106 | 'The lamb was sure to go.\n'], 107 | 'count-100': count(100), 108 | 'count-forever': count(), 109 | '99-bottles': bottles(99), 110 | 'example.html': fusetree.HttpFile('https://example.com/'), 111 | 'xkcd.png': fusetree.HttpFile('https://xkcd.com/s/0b7742.png'), 112 | 'linux.tar.xz': fusetree.HttpFile('https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.15.4.tar.xz'), 113 | 'link-to-99-bottles': fusetree.Symlink('99-bottles'), 114 | } 115 | 116 | if __name__ == '__main__': 117 | logging.basicConfig(level=logging.DEBUG) 118 | fusetree.FuseTree(rootNode, '/tmp/fusetree', foreground=True) 119 | -------------------------------------------------------------------------------- /fusetree/nodetypes.py: -------------------------------------------------------------------------------- 1 | from stat import S_IFDIR, S_IFLNK, S_IFREG 2 | from typing import Dict, Iterable 3 | import aiohttp 4 | from io import BytesIO 5 | 6 | import os 7 | 8 | from .types import * 9 | from .core import * 10 | from . import types_conv 11 | 12 | class BaseFile(Node): 13 | def __init__(self, mode: int = 0o444) -> None: 14 | self.mode = mode & 0o777 15 | 16 | async def getattr(self) -> Stat: 17 | return Stat( 18 | st_mode=S_IFREG | self.mode 19 | ) 20 | 21 | 22 | class BaseDir(Node): 23 | def __init__(self, mode: int = 0o555) -> None: 24 | self.mode = mode & 0o777 25 | 26 | async def getattr(self) -> Stat: 27 | return Stat( 28 | st_mode=S_IFDIR | self.mode 29 | ) 30 | 31 | 32 | class BaseSymlink(Node): 33 | def __init__(self, mode: int = 0o444) -> None: 34 | self.mode = mode & 0o777 35 | 36 | async def getattr(self) -> Stat: 37 | return Stat( 38 | st_mode=S_IFLNK | self.mode 39 | ) 40 | 41 | 42 | class Symlink(BaseSymlink): 43 | def __init__(self, link: str, mode: int = 0o444) -> None: 44 | super().__init__(mode) 45 | self.link = link 46 | 47 | async def readlink(self) -> str: 48 | return self.link 49 | 50 | 51 | 52 | class BlobFile(BaseFile): 53 | def __init__(self, data: bytes = b'', mode: int = None, rw: bool = False) -> None: 54 | super().__init__(mode if mode is not None else 0o666 if rw else 0o444) 55 | self.rw = rw 56 | self.data = data 57 | self.shared_handle = None 58 | 59 | async def load(self) -> bytes: 60 | return self.data 61 | 62 | async def save(self, data: bytes) -> None: 63 | self.data = data 64 | 65 | async def open(self, mode: int) -> FileHandle: 66 | if self.shared_handle is None: 67 | self.shared_handle = BlobFile.Handle(self, await self.load()) 68 | self.shared_handle.refs += 1 69 | return self.shared_handle 70 | 71 | async def getattr(self) -> Stat: 72 | if self.shared_handle is not None: 73 | size = len(self.shared_handle.buffer.getvalue()) 74 | else: 75 | size=len(self.data) 76 | 77 | return Stat( 78 | st_mode=S_IFREG | self.mode, 79 | st_size=size 80 | ) 81 | 82 | async def truncate(self, size: int) -> None: 83 | handle = await self.open(os.O_RDWR) 84 | try: 85 | await handle.truncate(size) 86 | finally: 87 | await handle.release() 88 | 89 | class Handle(FileHandle): 90 | def __init__(self, node: Node, data: bytes) -> None: 91 | super().__init__(node) 92 | self.buffer = BytesIO(data) 93 | self.dirty = False 94 | self.refs = 0 95 | 96 | async def read(self, size: int, offset: int) -> bytes: 97 | self.buffer.seek(offset) 98 | return self.buffer.read(size) 99 | 100 | async def write(self, buffer, offset): 101 | if not self.node.rw: 102 | raise fuse.FuseOSError(errno.EPERM) 103 | 104 | self.dirty = True 105 | self.buffer.seek(offset) 106 | self.buffer.write(buffer) 107 | return len(buffer) 108 | 109 | async def truncate(self, size: int) -> None: 110 | if not self.node.rw: 111 | raise fuse.FuseOSError(errno.EPERM) 112 | 113 | self.dirty = True 114 | self.buffer.truncate(size) 115 | 116 | async def flush(self) -> None: 117 | if self.dirty: 118 | await self.node.save(self.buffer.getvalue()) 119 | self.dirty = None 120 | 121 | async def release(self) -> None: 122 | self.refs -= 1 123 | if self.refs == 0: 124 | await self.flush() 125 | self.node.shared_handle = None 126 | 127 | 128 | 129 | 130 | 131 | class GeneratorFile(BaseFile): 132 | def __init__(self, generator: Iterable[Bytes_Like], mode: int = 0o444, min_read_len: int = -1) -> None: 133 | super().__init__(mode) 134 | self.generator = generator 135 | self.min_read_len = min_read_len 136 | 137 | async def open(self, mode: int) -> FileHandle: 138 | return GeneratorFile.Handle(self, self.generator, self.min_read_len) 139 | 140 | class Handle(FileHandle): 141 | def __init__(self, node: Node, generator: Iterable[Bytes_Like], min_read_len: int = -1) -> None: 142 | super().__init__(node, direct_io=True, nonseekable=True) 143 | 144 | self.generator = self.as_generator(generator) 145 | self.current_blob = b'' 146 | self.current_blob_position = 0 147 | self.min_read_len = min_read_len 148 | 149 | async def read(self, size: int, offset: int) -> bytes: 150 | ret = b'' 151 | while size > len(ret) and self.current_blob is not None: 152 | n = min(size - len(ret), len(self.current_blob) - self.current_blob_position) 153 | 154 | if n > 0: 155 | ret += self.current_blob[self.current_blob_position : self.current_blob_position + n] 156 | self.current_blob_position += n 157 | else: 158 | try: 159 | self.current_blob = types_conv.as_bytes(await self.generator.__anext__()) 160 | except StopAsyncIteration: 161 | self.current_blob = None 162 | self.current_blob_position = 0 163 | 164 | if self.min_read_len > 0 and len(ret) >= self.min_read_len: 165 | break 166 | return ret 167 | 168 | def as_generator(self, generator): 169 | async def as_async_gen(data): 170 | for x in data: 171 | yield x 172 | 173 | if hasattr(generator, '__anext__'): 174 | return generator 175 | elif hasattr(generator, '__aiter__'): 176 | return generator.__aiter__() 177 | elif hasattr(generator, '__next__'): 178 | return as_async_gen(generator) 179 | elif hasattr(generator, '__iter__'): 180 | return as_async_gen(iter(generator)) 181 | elif callable(generator): 182 | return self.as_generator(generator()) 183 | 184 | raise TypeError('Expected iterator, iterable, async iterator, async iterable or callable') 185 | 186 | def generatorfile(func): 187 | def tmp(*args, **kwargs): 188 | return GeneratorFile(lambda: func(*args, **kwargs)) 189 | return tmp 190 | 191 | 192 | class HttpFile(BaseFile): 193 | def __init__(self, url: str, mode: int = 0o444) -> None: 194 | super().__init__(mode) 195 | self.url = url 196 | 197 | async def open(self, mode: int) -> FileHandle: 198 | session = await aiohttp.ClientSession().__aenter__() 199 | response = await (await session.get(self.url)).__aenter__() 200 | return HttpFile.Handle(self, session, response) 201 | 202 | class Handle(FileHandle): 203 | def __init__(self, node: Node, session, response) -> None: 204 | super().__init__(node, direct_io=True, nonseekable=True) 205 | self.session = session 206 | self.response = response 207 | 208 | async def read(self, size: int, offset: int) -> bytes: 209 | return await self.response.content.read(size) 210 | 211 | async def release(self) -> None: 212 | await self.response.__aexit__(None, None, None) 213 | await self.session.__aexit__(None, None, None) 214 | 215 | 216 | class DictDir(BaseDir): 217 | def __init__(self, contents: Dict[str, Node_Like], mode: int = None, rw: bool = False) -> None: 218 | super().__init__(mode if mode is not None else 0o777 if rw else 0o555) 219 | self.rw = rw 220 | self.contents = contents 221 | 222 | # ====== RO operations ====== 223 | 224 | async def lookup(self, name: str) -> Node_Like: 225 | return self.contents.get(name, None) 226 | 227 | async def opendir(self) -> DirHandle_Like: 228 | return DictDir.Handle(self, self.contents.keys()) 229 | 230 | class Handle(DirHandle): 231 | def __init__(self, node: Node, items: Iterable[DirEntry]) -> None: 232 | super().__init__(node) 233 | self.items = items 234 | 235 | async def readdir(self) -> Iterable[DirEntry]: 236 | for item in self.items: 237 | yield item 238 | 239 | # ====== RW operations ====== 240 | 241 | async def mknod(self, name: str, mode: int, dev: int) -> Node_Like: 242 | if not self.rw: 243 | raise fuse.FuseOSError(errno.EPERM) 244 | 245 | if dev != 0: 246 | raise fuse.FuseOSError(errno.ENOSYS) 247 | 248 | new_file = BlobFile(b'', mode, rw=True) 249 | self.contents[name] = new_file 250 | return new_file 251 | 252 | async def mkdir(self, name: str, mode: int) -> Node_Like: 253 | if not self.rw: 254 | raise fuse.FuseOSError(errno.EPERM) 255 | 256 | new_dir = DictDir({}, mode, rw=True) 257 | self.contents[name] = new_dir 258 | return new_dir 259 | 260 | async def unlink(self, name: str) -> None: 261 | if not self.rw: 262 | raise fuse.FuseOSError(errno.EPERM) 263 | 264 | del self.contents[name] 265 | 266 | async def rmdir(self, name: str) -> None: 267 | if not self.rw: 268 | raise fuse.FuseOSError(errno.EPERM) 269 | 270 | del self.contents[name] 271 | 272 | async def symlink(self, name: str, target: str) -> Node_Like: 273 | if not self.rw: 274 | raise fuse.FuseOSError(errno.EPERM) 275 | 276 | new_link = Symlink(target) 277 | self.contents[name] = new_link 278 | 279 | async def rename(self, old_name: str, new_parent: Node, new_name: str) -> None: 280 | if not isinstance(new_parent, DictDir): 281 | raise fuse.FuseOSError(errno.ENOSYS) 282 | 283 | if not self.rw or not new_parent.rw: 284 | raise fuse.FuseOSError(errno.EPERM) 285 | 286 | node = self.contents[name] 287 | del self.contents[name] 288 | new_parent.contents[name] = node 289 | 290 | async def link(self, name: str, node: Node) -> Node_Like: 291 | self.contents[name] = node 292 | -------------------------------------------------------------------------------- /fusetree/core.py: -------------------------------------------------------------------------------- 1 | import fuse 2 | from fuse import fuse_file_info 3 | from typing import Dict, Iterator, Iterable, Sequence, Tuple, Optional, Any, NamedTuple, Union, List 4 | 5 | import logging 6 | import errno 7 | import time 8 | import threading 9 | import traceback 10 | 11 | from . import util 12 | from .types import * 13 | 14 | class Node: 15 | """ 16 | A node is the superclass of every entry in your filesystem. 17 | 18 | When working with the FUSE api, you have only one callback for each function, 19 | and must decide what to do based on the path. 20 | 21 | In constrast, FuseTree will find the node represented by the path and 22 | call the matching function on that object. 23 | 24 | e.g., `getattr('/foo/bar')` becomes `root['foo']['bar'].getattr()` 25 | 26 | Since node-creating operations, like `create`, `mkdir`, etc, receive paths that don't yet exist, 27 | they are instead called on their parent directories instead. 28 | 29 | e.g., `mkdir('/bar/foo/newdir')` becomes `root['bar']['foo'].mkdir('newdir')` 30 | 31 | Common implementations to several common node types are provided on `nodetypes`. 32 | """ 33 | 34 | @property 35 | def attr_timeout(self): 36 | return 1 37 | 38 | @property 39 | def entry_timeout(self): 40 | return 1 41 | 42 | async def remember(self) -> None: 43 | """ 44 | Hint that this node has been added to the kernel cache. 45 | On the root node it will be called once, when the FS is mounted -- Therefore it acts as fuse_init() 46 | """ 47 | pass 48 | 49 | async def forget(self) -> None: 50 | """ 51 | Hint that this node has been removed from the kernel cache 52 | 53 | On the root node it will be called once, when the FS is unmounted -- Therefore it acts as fuse_destroy() 54 | """ 55 | pass 56 | 57 | async def lookup(self, name: str) -> Node_Like: 58 | """ 59 | get one of this directory's child nodes by name 60 | (Or None if it doesn't exist) 61 | """ 62 | return None 63 | 64 | async def getattr(self) -> Stat_Like: 65 | """ 66 | Get file attributes. 67 | 68 | Similar to stat(). The 'st_dev' and 'st_blksize' fields are 69 | ignored. The 'st_ino' field is ignored except if the 'use_ino' 70 | mount option is given. In that case it is passed to userspace, 71 | but libfuse and the kernel will still assign a different 72 | inode for internal use (called the "nodeid"). 73 | """ 74 | raise fuse.FuseOSError(errno.ENOSYS) 75 | 76 | async def setattr(self, new_attr: Stat, to_set: List[str]) -> Stat_Like: 77 | cur_attr = await self.getattr() 78 | 79 | if 'st_mode' in to_set: 80 | await self.chmod(new_attr.st_mode) 81 | if 'st_uid' in to_set or 'st_gid' in to_set: 82 | await self.chown( 83 | new_attr.st_uid if 'st_uid' in to_set else cur_attr.st_uid, 84 | new_attr.st_gid if 'st_gid' in to_set else cur_attr.st_gid) 85 | if 'st_size' in to_set: 86 | await self.truncate(new_attr.st_size) 87 | if 'st_atime' in to_set or 'st_mtime' in to_set or 'st_ctime' in to_set: 88 | await self.utimens( 89 | new_attr.st_atime if 'st_atime' in to_set else cur_attr.st_atime, 90 | new_attr.st_mtime if 'st_mtime' in to_set else cur_attr.st_mtime) 91 | 92 | return await self.getattr() 93 | 94 | async def chmod(self, amode: int) -> None: 95 | """ 96 | Change the permission bits of a file 97 | """ 98 | raise fuse.FuseOSError(errno.ENOSYS) 99 | 100 | async def chown(self, uid: int, gid: int) -> None: 101 | """ 102 | Change the owner and group of a file. 103 | """ 104 | raise fuse.FuseOSError(errno.ENOSYS) 105 | 106 | async def truncate(self, length: int) -> None: 107 | """ 108 | Change the size of a file 109 | """ 110 | raise fuse.FuseOSError(errno.ENOSYS) 111 | 112 | async def utimens(self, atime: float, mtime: float) -> None: 113 | """ 114 | Change the access and modification times of a file with 115 | nanosecond resolution 116 | """ 117 | raise fuse.FuseOSError(errno.ENOSYS) 118 | 119 | async def readlink(self) -> str: 120 | """ 121 | Read the target of a symbolic link 122 | """ 123 | raise fuse.FuseOSError(errno.ENOSYS) 124 | 125 | async def mknod(self, name: str, mode: int, dev: int) -> Node_Like: 126 | """ 127 | Create a file node 128 | 129 | This is called for creation of all non-directory, non-symlink 130 | nodes. If the filesystem defines a create() method, then for 131 | regular files that will be called instead. 132 | """ 133 | raise fuse.FuseOSError(errno.ENOSYS) 134 | 135 | async def mkdir(self, name: str, mode: int) -> Node_Like: 136 | """ 137 | Create a directory 138 | 139 | Note that the mode argument may not have the type specification 140 | bits set, i.e. S_ISDIR(mode) can be false. To obtain the 141 | correct directory type bits use mode|S_IFDIR 142 | """ 143 | raise fuse.FuseOSError(errno.ENOSYS) 144 | 145 | async def unlink(self, name: str) -> None: 146 | """ 147 | Remove a file 148 | """ 149 | raise fuse.FuseOSError(errno.ENOSYS) 150 | 151 | async def rmdir(self, name: str) -> None: 152 | """ 153 | Remove a directory 154 | """ 155 | raise fuse.FuseOSError(errno.ENOSYS) 156 | 157 | async def symlink(self, name: str, target: str) -> Node_Like: 158 | """ 159 | Create a symbolic link 160 | """ 161 | raise fuse.FuseOSError(errno.ENOSYS) 162 | 163 | async def rename(self, old_name: str, new_parent: 'Node', new_name: str) -> None: 164 | """ 165 | Rename a file 166 | """ 167 | raise fuse.FuseOSError(errno.ENOSYS) 168 | 169 | async def link(self, name: str, node: 'Node') -> Node_Like: 170 | """ 171 | Create a hard link to a file 172 | """ 173 | raise fuse.FuseOSError(errno.ENOSYS) 174 | 175 | async def open(self, mode: int) -> 'FileHandle': 176 | """ 177 | File open operation 178 | 179 | No creation (O_CREAT, O_EXCL) and by default also no 180 | truncation (O_TRUNC) flags will be passed to open(). If an 181 | application specifies O_TRUNC, fuse first calls truncate() 182 | and then open(). Only if 'atomic_o_trunc' has been 183 | specified and kernel version is 2.6.24 or later, O_TRUNC is 184 | passed on to open. 185 | 186 | Unless the 'default_permissions' mount option is given, 187 | open should check if the operation is permitted for the 188 | given flags. Optionally open may also return an arbitrary 189 | filehandle in the fuse_file_info structure, which will be 190 | passed to all file operations. 191 | """ 192 | raise fuse.FuseOSError(errno.ENOSYS) 193 | 194 | async def setxattr(self, name: str, value: bytes, flags: int) -> None: 195 | """ 196 | Set extended attributes 197 | """ 198 | raise fuse.FuseOSError(errno.ENOSYS) 199 | 200 | async def getxattr(self, name: str) -> bytes: 201 | """ 202 | Get extended attributes 203 | """ 204 | raise fuse.FuseOSError(errno.ENOSYS) 205 | 206 | async def listxattr(self) -> Iterable[str]: 207 | """ 208 | List extended attributes 209 | """ 210 | raise fuse.FuseOSError(errno.ENOSYS) 211 | 212 | async def removexattr(self, name: str) -> None: 213 | """ 214 | Remove extended attributes 215 | """ 216 | raise fuse.FuseOSError(errno.ENOSYS) 217 | 218 | async def opendir(self) -> DirHandle_Like: 219 | """ 220 | Open directory 221 | 222 | Unless the 'default_permissions' mount option is given, 223 | this method should check if opendir is permitted for this 224 | directory. Optionally opendir may also return an arbitrary 225 | filehandle in the fuse_file_info structure, which will be 226 | passed to readdir, closedir and fsyncdir. 227 | """ 228 | raise fuse.FuseOSError(errno.ENOSYS) 229 | 230 | async def statfs(self) -> StatVFS: 231 | """ 232 | Get file system statistics 233 | 234 | The 'f_favail', 'f_fsid' and 'f_flag' fields are ignored 235 | """ 236 | raise fuse.FuseOSError(errno.ENOSYS) 237 | 238 | 239 | async def access(self, amode: int) -> None: 240 | """ 241 | Check file access permissions 242 | 243 | This will be called for the access() system call. If the 244 | 'default_permissions' mount option is given, this method is not 245 | called. 246 | """ 247 | pass 248 | 249 | async def create(self, name: str, mode: int) -> 'FileHandle': 250 | """ 251 | Check file access permissions 252 | 253 | This will be called for the access() system call. If the 254 | 'default_permissions' mount option is given, this method is not 255 | called. 256 | """ 257 | raise fuse.FuseOSError(errno.ENOSYS) 258 | 259 | 260 | 261 | class DirHandle: 262 | """ 263 | A DirHandle is what you get with a call to `opendir()`. 264 | 265 | It can be used to list the directory contents (very important) and to fsync (optional) 266 | 267 | You probably don't need to deal with this class directly: Just return a collection 268 | of the file names on `opendir` and a new DirHandle will be created automatically 269 | """ 270 | 271 | def __init__(self, node: Node = None) -> None: 272 | self.node = node 273 | 274 | async def readdir(self) -> Iterable[DirEntry]: 275 | """ 276 | Read directory 277 | """ 278 | raise fuse.FuseOSError(errno.ENOSYS) 279 | 280 | async def fsyncdir(self, datasync: int) -> None: 281 | """ 282 | Synchronize directory contents 283 | 284 | If the datasync parameter is non-zero, then only the user data 285 | should be flushed, not the meta data 286 | """ 287 | raise fuse.FuseOSError(errno.ENOSYS) 288 | 289 | async def releasedir(self) -> None: 290 | """ 291 | Release directory 292 | """ 293 | pass 294 | 295 | 296 | 297 | class FileHandle: 298 | """ 299 | A FileHandle is what you get with a call to `open()` or `create()`. 300 | 301 | Most importantly, you should implement `read` and/or `write`. 302 | 303 | You probably don't need to deal with this class directly: Just return a blob from `read()` 304 | and a FileHandle will be created automatically -- Otherwise, check the many common implementation in `nodetypes` 305 | """ 306 | 307 | def __init__(self, node: Node = None, direct_io: bool = False, nonseekable: bool = False) -> None: 308 | self.node = node 309 | self.direct_io = direct_io 310 | self.nonseekable = nonseekable 311 | 312 | async def getattr(self) -> Stat_Like: 313 | """ 314 | Get file attributes of an open file. 315 | 316 | Similar to stat(). The 'st_dev' and 'st_blksize' fields are 317 | ignored. The 'st_ino' field is ignored except if the 'use_ino' 318 | mount option is given. In that case it is passed to userspace, 319 | but libfuse and the kernel will still assign a different 320 | inode for internal use (called the "nodeid"). 321 | """ 322 | raise fuse.FuseOSError(errno.ENOSYS) 323 | 324 | async def setattr(self, new_attr: Stat, to_set: List[str]) -> Stat_Like: 325 | cur_attr = await self.getattr() 326 | 327 | if 'st_mode' in to_set: 328 | await self.chmod(new_attr.st_mode) 329 | if 'st_uid' in to_set or 'st_gid' in to_set: 330 | await self.chown( 331 | new_attr.st_uid if 'st_uid' in to_set else cur_attr.st_uid, 332 | new_attr.st_gid if 'st_gid' in to_set else cur_attr.st_gid) 333 | if 'st_size' in to_set: 334 | await self.truncate(new_attr.st_size) 335 | if 'st_atime' in to_set or 'st_mtime' in to_set or 'st_ctime' in to_set: 336 | await self.utimens( 337 | new_attr.st_atime if 'st_atime' in to_set else cur_attr.st_atime, 338 | new_attr.st_mtime if 'st_mtime' in to_set else cur_attr.st_mtime) 339 | 340 | return await self.getattr() 341 | 342 | async def chmod(self, amode: int) -> None: 343 | """ 344 | Change the permission bits of an open file. 345 | """ 346 | raise fuse.FuseOSError(errno.ENOSYS) 347 | 348 | async def chown(self, uid: int, gid: int) -> None: 349 | """ 350 | Change the owner and group of an open file. 351 | """ 352 | raise fuse.FuseOSError(errno.ENOSYS) 353 | 354 | async def truncate(self, length: int) -> None: 355 | """ 356 | Change the size of an open file. 357 | """ 358 | raise fuse.FuseOSError(errno.ENOSYS) 359 | 360 | async def utimens(self, atime: float, mtime: float) -> None: 361 | """ 362 | Change the access and modification times of a file with 363 | nanosecond resolution 364 | """ 365 | raise fuse.FuseOSError(errno.ENOSYS) 366 | 367 | async def read(self, size: int, offset: int) -> bytes: 368 | """ 369 | Read data from an open file 370 | 371 | Read should return exactly the number of bytes requested except 372 | on EOF or error, otherwise the rest of the data will be 373 | substituted with zeroes. An exception to this is when the 374 | 'direct_io' mount option is specified, in which case the return 375 | value of the read system call will reflect the return value of 376 | this operation. 377 | """ 378 | raise fuse.FuseOSError(errno.ENOSYS) 379 | 380 | async def write(self, data: bytes, offset: int) -> int: 381 | """ 382 | Write data to an open file 383 | 384 | Write should return exactly the number of bytes requested 385 | except on error. An exception to this is when the 'direct_io' 386 | mount option is specified (see read operation). 387 | """ 388 | raise fuse.FuseOSError(errno.ENOSYS) 389 | 390 | async def flush(self) -> None: 391 | """ 392 | Possibly flush cached data 393 | 394 | BIG NOTE: This is not equivalent to fsync(). It's not a 395 | request to sync dirty data. 396 | 397 | Flush is called on each close() of a file descriptor. So if a 398 | filesystem wants to return write errors in close() and the file 399 | has cached dirty data, this is a good place to write back data 400 | and return any errors. Since many applications ignore close() 401 | errors this is not always useful. 402 | 403 | NOTE: The flush() method may be called more than once for each 404 | open(). This happens if more than one file descriptor refers 405 | to an opened file due to dup(), dup2() or fork() calls. It is 406 | not possible to determine if a flush is final, so each flush 407 | should be treated equally. Multiple write-flush sequences are 408 | relatively rare, so this shouldn't be a problem. 409 | 410 | Filesystems shouldn't assume that flush will always be called 411 | after some writes, or that if will be called at all. 412 | """ 413 | pass 414 | 415 | async def release(self) -> None: 416 | """ 417 | Release an open file 418 | 419 | Release is called when there are no more references to an open 420 | file: all file descriptors are closed and all memory mappings 421 | are unmapped. 422 | 423 | For every open() call there will be exactly one release() call 424 | with the same flags and file descriptor. It is possible to 425 | have a file opened more than once, in which case only the last 426 | release will mean, that no more reads/writes will happen on the 427 | file. The return value of release is ignored. 428 | """ 429 | pass 430 | 431 | async def fsync(self, datasync: int) -> None: 432 | """ 433 | Release an open file 434 | 435 | Release is called when there are no more references to an open 436 | file: all file descriptors are closed and all memory mappings 437 | are unmapped. 438 | 439 | For every open() call there will be exactly one release() call 440 | with the same flags and file descriptor. It is possible to 441 | have a file opened more than once, in which case only the last 442 | release will mean, that no more reads/writes will happen on the 443 | file. The return value of release is ignored. 444 | """ 445 | pass 446 | 447 | async def lock(self, cmd: int, lock: Any) -> None: 448 | """ 449 | Perform POSIX file locking operation 450 | 451 | The cmd argument will be either F_GETLK, F_SETLK or F_SETLKW. 452 | 453 | For the meaning of fields in 'struct flock' see the man page 454 | for fcntl(2). The l_whence field will always be set to 455 | SEEK_SET. 456 | 457 | For checking lock ownership, the 'fuse_file_info->owner' 458 | argument must be used. 459 | 460 | For F_GETLK operation, the library will first check currently 461 | held locks, and if a conflicting lock is found it will return 462 | information without calling this method. This ensures, that 463 | for local locks the l_pid field is correctly filled in. The 464 | results may not be accurate in case of race conditions and in 465 | the presence of hard links, but it's unlikely that an 466 | application would rely on accurate GETLK results in these 467 | cases. If a conflicting lock is not found, this method will be 468 | called, and the filesystem may fill out l_pid by a meaningful 469 | value, or it may leave this field zero. 470 | 471 | For F_SETLK and F_SETLKW the l_pid field will be set to the pid 472 | of the process performing the locking operation. 473 | 474 | Note: if this method is not implemented, the kernel will still 475 | allow file locking to work locally. Hence it is only 476 | interesting for network filesystems and similar. 477 | 478 | FIXME: fusepy doesn't seem to support it properly 479 | """ 480 | raise fuse.FuseOSError(errno.ENOSYS) 481 | 482 | -------------------------------------------------------------------------------- /fusetree/fusetree.py: -------------------------------------------------------------------------------- 1 | import fusell 2 | from fusell import fuse_file_info 3 | from fuse import FuseOSError 4 | from typing import Dict, Iterable, Tuple, Optional, Any 5 | 6 | import logging 7 | import errno 8 | import time 9 | import asyncio 10 | import os 11 | import ctypes 12 | import threading 13 | import time 14 | 15 | from .util import LoggingFuseOperations 16 | from .types import * 17 | from .core import * 18 | from .types_conv import * 19 | 20 | 21 | def pretty(x): 22 | if isinstance(x, ctypes.Structure): 23 | return f'{x.__class__.__name__}(%s)' % (', '.join([ 24 | f'{entry[0]}={pretty(getattr(x, entry[0]))}' 25 | for entry in x._fields_ 26 | if getattr(x, entry[0]) 27 | ])) 28 | elif isinstance(x, list): 29 | return '[' + ', '.join([pretty(item) for item in x]) + ']' 30 | else: 31 | return str(x) 32 | 33 | def copy_struct(struct): 34 | copy = type(struct)() 35 | ctypes.pointer(copy)[0] = struct 36 | return copy 37 | 38 | 39 | class FuseTree(fusell.FUSELL): 40 | def __init__(self, rootNode: Node_Like, mountpoint, log=True, encoding='utf-8', loop=None, **kwargs) -> None: 41 | self.rootNode = as_node(rootNode) 42 | 43 | self._req_seq_lock = asyncio.Lock() 44 | self._next_req_seq = 1 45 | 46 | self._loop = loop 47 | self._loop_thread = None 48 | self._handle_lock = asyncio.Lock() 49 | self._inodes: Dict[int, Node] = {} 50 | self._inodes_refs: Dict[int, int] = {} 51 | self._next_fd = 1 52 | self._file_fds: Dict[int, FileHandle] = {} 53 | self._dir_fds: Dict[int, DirHandle] = {} 54 | 55 | #This maps automatically mapping of "Node-like" things to nodes 56 | self._node_like_to_node: Dict[int, Any] = {} 57 | self._node_to_node_like: Dict[int, Any] = {} 58 | 59 | super().__init__(mountpoint, encoding=encoding) 60 | 61 | async def _as_node(self, node: Node_Like) -> Node: 62 | async with self._handle_lock: 63 | existing = self._node_like_to_node.get(id(node), None) 64 | if existing is not None: 65 | return existing 66 | 67 | actual_node = as_node(node) 68 | if actual_node is not node: 69 | async with self._handle_lock: 70 | self._node_like_to_node[id(node)] = actual_node 71 | self._node_to_node_like[id(actual_node)] = node 72 | return actual_node 73 | 74 | async def _ino_to_node(self, inode: int) -> Node: 75 | async with self._handle_lock: 76 | try: 77 | return self._inodes[inode] 78 | except KeyError: 79 | raise FuseOSError(errno.ENOENT) 80 | 81 | async def _node_to_ino(self, node: Node, remember: bool = False) -> int: 82 | return 1 if node is self.rootNode else id(node) 83 | 84 | async def _update_inodef_refs(self, node: Node, update: int) -> int: 85 | ino = await self._node_to_ino(node) 86 | 87 | async with self._handle_lock: 88 | if update > 0: 89 | if ino not in self._inodes: 90 | await node.remember() 91 | self._inodes[ino] = node 92 | self._inodes_refs[ino] = update 93 | else: 94 | self._inodes_refs[ino] += update 95 | 96 | elif update < 0: 97 | if ino not in self._inodes: 98 | raise FuseOSError(errno.ENOENT) 99 | else: 100 | self._inodes_refs[ino] += update # Remember, it's a negative number 101 | if self._inodes_refs[ino] == 0: 102 | await node.forget() 103 | del self._inodes_refs[ino] 104 | del self._inodes[ino] 105 | 106 | node_like = self._node_to_node_like[id(node)] 107 | if node_like is not None: 108 | del self._node_to_node_like[id(node)] 109 | del self._node_like_to_node[id(node_like)] 110 | 111 | else: 112 | pass # update == 0 -- Shouldn't happen, but is a no-op No-op 113 | 114 | new_refcount = self._inodes_refs.get(ino, 0) 115 | return new_refcount 116 | 117 | 118 | 119 | async def _reply_err(self, req, err: int) -> (int, str): 120 | self.reply_err(req, err) 121 | if err == 0: 122 | return 'Success' 123 | else: 124 | return 'ERR %d: %s' % (err, os.strerror(err)) 125 | 126 | async def _reply_entry(self, req, node: Node) -> (int, Node): 127 | ino = await self._node_to_ino(node) 128 | await self._update_inodef_refs(node, +1) 129 | stat = as_stat(await node.getattr()) 130 | stat = stat.with_values(st_ino=ino) 131 | 132 | entry = dict( 133 | ino=ino, 134 | attr=stat.as_dict(), 135 | attr_timeout=node.attr_timeout, 136 | entry_timeout=node.entry_timeout) 137 | 138 | self.reply_entry(req, entry) 139 | return ino, node 140 | 141 | async def _reply_attr(self, req, attr: Stat, ino: int, attr_timeout: float) -> Stat: 142 | attr = attr.with_values(st_ino=ino) 143 | 144 | self.reply_attr(req, attr.as_dict(), attr_timeout) 145 | return attr 146 | 147 | async def _reply_readlink(self, req, link: str) -> str: 148 | self.reply_readlink(req, link) 149 | return link 150 | 151 | async def _reply_none(self, req, ret=None): 152 | self.reply_none(req) 153 | return ret 154 | 155 | async def _reply_write(self, req, n: int) -> str: 156 | self.reply_write(req, n) 157 | return f'{n} bytes' 158 | 159 | def _wrapper(method): 160 | def safe_arg(arg): 161 | """ 162 | Copy arguments that are passed as pointers, as fuse_low_level destroys 163 | them as soon as the callback is returned 164 | """ 165 | if isinstance(arg, ctypes._Pointer): 166 | if not arg: 167 | return None 168 | 169 | contents = arg.contents 170 | if isinstance(contents, ctypes.Structure): 171 | return copy_struct(contents) 172 | else: 173 | raise Exception(f'Unsupported pointer type: {type(arg).__name__}') 174 | else: 175 | return arg 176 | 177 | 178 | async def wrapped(self, req, *args): 179 | async with self._req_seq_lock: 180 | req_seq = self._next_req_seq 181 | self._next_req_seq += 1 182 | 183 | print(f'{req_seq} > {method.__name__[5:]}:', ', '.join([pretty(arg) for arg in args])) 184 | 185 | try: 186 | result = await method(self, req, *args) 187 | except OSError as e: 188 | #traceback.print_exc() 189 | result = await self._reply_err(req, e.errno) 190 | except Exception as e: 191 | traceback.print_exc() 192 | await self._reply_err(req, errno.EFAULT) 193 | result = repr(e) 194 | 195 | print(f'{req_seq} < {method.__name__[5:]}: {pretty(result)}') 196 | 197 | def wrapper(self, *args): 198 | # Pointer arguments are destroyed when the callback exits, 199 | # therefore they must be copied 200 | if method.__name__ == 'fuse_write': 201 | args = list(args) 202 | args[2] = ctypes.string_at(args[2], args[3]) 203 | 204 | if method.__name__ == 'fuse_forget_multi': 205 | args = list(args) 206 | args[2] = [copy_struct(args[2][i]) for i in range(args[1])] 207 | 208 | args = map(safe_arg, args) 209 | 210 | future = wrapped(self, *args) 211 | self._loop.call_soon_threadsafe(self._loop.create_task, future) 212 | 213 | return wrapper 214 | 215 | 216 | def fuse_init(self, userdata, conn): 217 | # Start event loop 218 | if self._loop is None: 219 | self._loop = asyncio.new_event_loop() 220 | def f(loop): 221 | asyncio.set_event_loop(loop) 222 | #print("AsyncIO loop started...") 223 | loop.run_forever() 224 | #print("AsyncIO loop completed") 225 | self._loop_thread = threading.Thread(target=f, args=(self._loop,)) 226 | self._loop_thread.start() 227 | 228 | # Remember root node 229 | async def remember_root(): 230 | await self._update_inodef_refs(self.rootNode, 1) 231 | asyncio.run_coroutine_threadsafe(remember_root(), self._loop).result() 232 | 233 | 234 | def fuse_destroy(self, userdata): 235 | # Forget all nodes 236 | async def forget_all(): 237 | async with self._handle_lock: 238 | # Forgets every node except the root 239 | await asyncio.wait([ 240 | node.forget() 241 | for node in self._inodes.values() 242 | if node is not self.rootNode 243 | ]) 244 | 245 | # The root node is the last one to be forgotten, and acts as a destroy() 246 | await self.rootNode.forget() 247 | self._inodes.clear() 248 | self._inodes_refs.clear() 249 | self._node_like_to_node.clear() 250 | self._node_to_node_like.clear() 251 | 252 | asyncio.run_coroutine_threadsafe(forget_all(), self._loop).result() 253 | 254 | 255 | # Stop event loop 256 | if self._loop_thread is not None: 257 | self._loop.call_soon_threadsafe(self._loop.stop) 258 | self._loop_thread.join() 259 | 260 | self._loop = None 261 | self._loop_thread = None 262 | 263 | 264 | @_wrapper 265 | async def fuse_lookup(self, req, parent_ino, name): 266 | parent = await self._ino_to_node(parent_ino) 267 | _child = await parent.lookup(name.decode(self.encoding)) 268 | child = await self._as_node(_child) 269 | if child is not _child: 270 | print(f'Converted child {name} for {type(_child)} to {type(child)}') 271 | 272 | return await self._reply_entry(req, child) 273 | 274 | async def _forget(self, ino, nlookup): 275 | node = await self._ino_to_node(ino) 276 | remaining_refs = await self._update_inodef_refs(node, -nlookup) 277 | return f'{ino} forgotten' if remaining_refs == 0 else f'{ino} has {remaining_refs} references remaining' 278 | 279 | @_wrapper 280 | async def fuse_forget(self, req, ino, nlookup): 281 | return await self._reply_none(req, await self._forget(ino, nlookup)) 282 | 283 | @_wrapper 284 | async def fuse_forget_multi(self, req, count, forgets): 285 | #TODO: Could probably be more efficient, but good enough for now 286 | tasks = [ 287 | self._forget(forgets[i].ino, forgets[i].nlookup) 288 | for i in range(count) 289 | ] 290 | 291 | results, _ = await asyncio.wait(tasks) 292 | return await self._reply_none(req, ', '.join(result.result() for result in results)) 293 | 294 | @_wrapper 295 | async def fuse_getattr(self, req, ino, fi): 296 | node = await self._ino_to_node(ino) 297 | 298 | ## Try to send setattr to the FileHandle 299 | if fi is not None: 300 | try: 301 | async with self._handle_lock: 302 | handle = self._file_fds[fi.fh] 303 | attr = as_stat(await handle.getattr()) 304 | return await self._reply_attr(req, attr, ino, node.attr_timeout) 305 | except OSError as e: 306 | # Ignore not-implemented -- We will fallback to the same operation on the node 307 | if e.errno != errno.ENOSYS: 308 | raise 309 | 310 | attr = as_stat(await node.getattr()) 311 | return await self._reply_attr(req, attr, ino, node.attr_timeout) 312 | 313 | @_wrapper 314 | async def fuse_setattr(self, req, ino, attr, to_set, fi): 315 | node = await self._ino_to_node(ino) 316 | 317 | now = time.time() 318 | to_set = fusell.setattr_mask_to_list(to_set) 319 | attr = as_stat({ 320 | k[:-4] if k.endswith('_now') else k: now if k.endswith('_now') else v 321 | for k, v in fusell.stat_to_dict(ctypes.pointer(attr)).items() 322 | if k in to_set 323 | }) 324 | 325 | ## Try to send setattr to the FileHandle 326 | if fi is not None: 327 | try: 328 | async with self._handle_lock: 329 | handle = self._file_fds[fi.fh] 330 | new_attr = as_stat(await handle.setattr(attr, to_set)) 331 | except OSError as e: 332 | # Ignore not-implemented -- We will fallback to the same operation on the node 333 | if e.errno != errno.ENOSYS: 334 | raise 335 | 336 | # send setattr to the Node 337 | new_attr = as_stat(await node.setattr(attr, to_set)) 338 | return await self._reply_attr(req, new_attr, ino, node.attr_timeout) 339 | 340 | 341 | @_wrapper 342 | async def fuse_readlink(self, req, ino): 343 | node = await self._ino_to_node(ino) 344 | link = await node.readlink() 345 | 346 | return await self._reply_readlink(req, link) 347 | 348 | @_wrapper 349 | async def fuse_mknod(self, req, parent_ino, name, mode, rdev): 350 | parent = await self._ino_to_node(parent_ino) 351 | new_node = await self._as_node(await parent.mknod(name.decode(self.encoding), mode, rdev)) 352 | 353 | return await self._reply_entry(req, new_node) 354 | 355 | @_wrapper 356 | async def fuse_mkdir(self, req, parent_ino, name, mode): 357 | parent = await self._ino_to_node(parent_ino) 358 | new_dir = await self._as_node(await parent.mkdir(name.decode(self.encoding), mode)) 359 | 360 | return await self._reply_entry(req, new_dir) 361 | 362 | @_wrapper 363 | async def fuse_unlink(self, req, parent_ino, name): 364 | parent = await self._ino_to_node(parent_ino) 365 | await parent.unlink(name.decode(self.encoding)) 366 | 367 | return await self._reply_err(req, 0) 368 | 369 | @_wrapper 370 | async def fuse_rmdir(self, req, parent_ino, name): 371 | parent = await self._ino_to_node(parent_ino) 372 | await parent.rmdir(name.decode(self.encoding)) 373 | 374 | return await self._reply_err(req, 0) 375 | 376 | @_wrapper 377 | async def fuse_symlink(self, req, link, parent_ino, name): 378 | parent = await self._ino_to_node(parent_ino) 379 | new_symlink = await parent.symlink(name.decode(self.encoding), link.decode(self.encoding)) 380 | 381 | return await self._reply_entry(req, new_symlink) 382 | 383 | @_wrapper 384 | async def fuse_rename(self, req, old_parent_ino, name, new_parent_ino, new_name): 385 | old_parent = await self._ino_to_node(old_parent_ino) 386 | new_parent = await self._ino_to_node(new_parent_ino) 387 | await old_parent.rename(name, new_parent, new_name) 388 | 389 | return await self._reply_err(req, 0) 390 | 391 | @_wrapper 392 | async def fuse_link(self, req, ino, new_parent_ino, new_name): 393 | node = await self._ino_to_node(ino) 394 | new_parent = await self._ino_to_node(new_parent_ino) 395 | new_link = await self._as_node(await new_parent.link(ino, new_name)) 396 | 397 | return await self._reply_entry(req, new_link) 398 | 399 | 400 | 401 | # FileHandle 402 | 403 | @_wrapper 404 | async def fuse_open(self, req, ino, fi): 405 | node = await self._ino_to_node(ino) 406 | handle = as_filehandle(node, await node.open(fi.flags)) 407 | fi.direct_io = handle.direct_io 408 | fi.nonseekable = handle.nonseekable 409 | 410 | async with self._handle_lock: 411 | fd = self._next_fd 412 | fi.fh = fd 413 | self._file_fds[fd] = handle 414 | self._next_fd += 1 415 | 416 | self.libfuse.fuse_reply_open(req, ctypes.byref(fi)) 417 | return fd, handle 418 | 419 | @_wrapper 420 | async def fuse_read(self, req, ino, size, off, fi): 421 | async with self._handle_lock: 422 | handle = self._file_fds[fi.fh] 423 | 424 | buf = await handle.read(size, off) 425 | 426 | self.libfuse.fuse_reply_buf(req, buf, len(buf)) 427 | return f'{len(buf)} bytes' 428 | 429 | @_wrapper 430 | async def fuse_write(self, req, ino, buf, size, off, fi): 431 | async with self._handle_lock: 432 | handle = self._file_fds[fi.fh] 433 | 434 | n = await handle.write(buf, off) 435 | 436 | return await self._reply_write(req, n) 437 | 438 | @_wrapper 439 | async def fuse_flush(self, req, ino, fi): 440 | async with self._handle_lock: 441 | handle = self._file_fds[fi.fh] 442 | 443 | await handle.flush() 444 | 445 | return await self._reply_err(req, 0) 446 | 447 | @_wrapper 448 | async def fuse_release(self, req, ino, fi): 449 | async with self._handle_lock: 450 | handle = self._file_fds[fi.fh] 451 | del self._file_fds[fi.fh] 452 | 453 | await handle.release() 454 | 455 | return await self._reply_err(req, 0) 456 | 457 | 458 | @_wrapper 459 | async def fuse_fsync(self, req, ino, datasync, fi): 460 | async with self._handle_lock: 461 | handle = self._file_fds[fi.fh] 462 | 463 | await handle.fsync(datasync) 464 | 465 | return await self._reply_err(req, 0) 466 | 467 | 468 | 469 | # DirHandle 470 | 471 | @_wrapper 472 | async def fuse_opendir(self, req, ino, fi): 473 | node = await self._ino_to_node(ino) 474 | dirhandle = as_dirhandle(node, await node.opendir()) 475 | 476 | async with self._handle_lock: 477 | dirfd = self._next_fd 478 | fi.fh = dirfd 479 | self._dir_fds[dirfd] = dirhandle 480 | self._next_fd += 1 481 | 482 | self.libfuse.fuse_reply_open(req, ctypes.byref(fi)) 483 | return dirfd, dirhandle 484 | 485 | @_wrapper 486 | async def fuse_readdir(self, req, ino, size, off, fi): 487 | async with self._handle_lock: 488 | dirhandle = self._dir_fds[fi.fh] 489 | 490 | node = await self._ino_to_node(ino) 491 | node_stat = as_stat(await node.getattr()) 492 | node_stat = node_stat.with_values(st_ino=ino) 493 | entries = [('.', node_stat.as_dict()), ('..', node_stat.as_dict())] 494 | entry_names = [] 495 | 496 | #TODO: Increase concurrency 497 | async for entry in dirhandle.readdir(): 498 | if isinstance(entry, str): 499 | child_name = entry 500 | child = await node.lookup(child_name) 501 | else: 502 | child_name, child = entry 503 | 504 | child = await self._as_node(child) 505 | 506 | child_ino = await self._node_to_ino(child) 507 | child_stat = as_stat(await child.getattr()) 508 | child_stat = child_stat.with_values(st_ino=child_ino) 509 | 510 | entries.append((child_name, child_stat.as_dict())) 511 | entry_names.append(child_name) 512 | 513 | self.reply_readdir(req, size, off, entries) 514 | return entry_names 515 | 516 | @_wrapper 517 | async def fuse_releasedir(self, req, ino, fi): 518 | async with self._handle_lock: 519 | handle = self._dir_fds[fi.fh] 520 | del self._dir_fds[fi.fh] 521 | 522 | await handle.releasedir() 523 | 524 | return await self._reply_err(req, 0) 525 | 526 | @_wrapper 527 | async def fuse_fsyncdir(self, req, ino, datasync, fi): 528 | async with self._handle_lock: 529 | dirhandle = self._dir_fds[fi.fh] 530 | 531 | await dirhandle.fsyncdir(datasync) 532 | 533 | return await self._reply_err(req, 0) 534 | --------------------------------------------------------------------------------