├── fsspec_disk ├── __init__.py ├── demo_s3.py └── utils.py ├── img ├── 1.png └── 2.png ├── readme.md ├── setup.py └── test.py /fsspec_disk/__init__.py: -------------------------------------------------------------------------------- 1 | import io 2 | import logging 3 | import datetime 4 | from threading import Lock 5 | from types import FunctionType 6 | from typing import Tuple, List 7 | from pathlib import Path 8 | 9 | import fsspec 10 | 11 | from winfspy import ( 12 | FileSystem, 13 | BaseFileSystemOperations, 14 | FILE_ATTRIBUTE, 15 | CREATE_FILE_CREATE_OPTIONS, 16 | NTStatusObjectNameNotFound, 17 | NTStatusObjectNameCollision, 18 | ) 19 | from winfspy.plumbing.win32_filetime import filetime_now 20 | from winfspy.plumbing.security_descriptor import SecurityDescriptor 21 | 22 | 23 | SD = SecurityDescriptor.from_string("O:BAG:BAD:P(A;;FA;;;SY)(A;;FA;;;BA)(A;;FA;;;WD)") 24 | 25 | 26 | class FalseOpen: 27 | def __init__(self, file_name, create_options, granted_access): 28 | self.file_name = file_name 29 | self.create_options = create_options 30 | self.granted_access = granted_access 31 | 32 | def close(self): 33 | None 34 | 35 | def __repr__(self): 36 | return f'' 37 | 38 | 39 | class Barbarossa(BaseFileSystemOperations): 40 | def __init__(self, fsspec_system: fsspec.AbstractFileSystem, use_elf_mkdir=False, log=True, volume_label='fsspec_disk'): 41 | super().__init__() 42 | self.fsspec_system = fsspec_system 43 | self.log = log 44 | self.use_elf_mkdir = use_elf_mkdir 45 | self._opened_files = [] 46 | self._locks = {} 47 | self._lock_lock = Lock() 48 | self._volume_label = volume_label 49 | 50 | def _get_lock(self, file_name): 51 | with self._lock_lock: 52 | if file_name not in self._locks: 53 | self._locks[file_name] = Lock() 54 | return self._locks[file_name] 55 | 56 | def list_opened_files(self): 57 | self._opened_files = [i for i in self._opened_files if i.closed] 58 | return self._opened_files 59 | 60 | def _replace_name(self, file_name): 61 | file_name = file_name.replace('\\', '/') 62 | if file_name != '/': 63 | file_name = file_name.removeprefix('/') 64 | return file_name 65 | 66 | def _get_info(self, file_name): 67 | file_name = self._replace_name(file_name) 68 | try: 69 | return self.fsspec_system.info(file_name) 70 | except FileNotFoundError: 71 | if file_name == '/': 72 | return { 73 | 'name': '/', 74 | 'type': 'directory', 75 | } 76 | raise NTStatusObjectNameNotFound() 77 | 78 | def _fs_info_to_file_info(self, info: dict): 79 | def _get_float(key: str): 80 | value = info.get('created', 0) 81 | if isinstance(value, datetime.datetime): 82 | value = value.timestamp() 83 | return value 84 | return { 85 | "file_name": info['name'], 86 | "file_attributes": FILE_ATTRIBUTE.FILE_ATTRIBUTE_DIRECTORY if info['type'] == 'directory' else FILE_ATTRIBUTE.FILE_ATTRIBUTE_NORMAL, 87 | "allocation_size": info.get('size') or 0, 88 | "file_size": info.get('size') or 0, 89 | "creation_time": int(_get_float('created')*10000000)+116444736000000000, 90 | "last_access_time": int(_get_float('mtime')*10000000)+116444736000000000, 91 | "last_write_time": int(_get_float('mtime')*10000000)+116444736000000000, 92 | "change_time": int(_get_float('mtime')*10000000)+116444736000000000, 93 | "index_number": 0, 94 | } 95 | 96 | def _open(self, file_name, *args): 97 | file_name = self._replace_name(file_name) 98 | o = self.fsspec_system.open(file_name, *args) 99 | o.file_name = file_name 100 | return o 101 | 102 | def get_security(self, file_context): 103 | return SD 104 | 105 | def get_security_by_name(self, file_name) -> Tuple: 106 | return ( 107 | self._fs_info_to_file_info(self._get_info(file_name))['file_attributes'], 108 | SD.handle, 109 | SD.size, 110 | ) 111 | 112 | def read_directory(self, file_context, marker) -> List: 113 | entries = [self._fs_info_to_file_info(info) for info in self.fsspec_system.ls(file_context.file_name, detail=True)] 114 | for entry in [*entries]: 115 | entry["file_name"] = entry["file_name"].replace('\\', '/').split('/')[-1] 116 | if entry["file_name"] == 'elf': 117 | entries.remove(entry) 118 | if file_context.file_name != '/': 119 | entries.extend([{"file_name": "."}, {"file_name": ".."}]) 120 | entries = sorted(entries, key=lambda x: x["file_name"]) 121 | if marker is None: 122 | return entries 123 | for i, entry in enumerate(entries): 124 | if entry["file_name"] == marker: 125 | return entries[i + 1 :] 126 | logging.error(f'找不到marker={marker}，entries={[e["file_name"] for e in entries]}') 127 | return entries 128 | 129 | def can_delete(self, file_context, file_name: str) -> None: 130 | None 131 | 132 | def set_security(self, file_context, security_information, modification_descriptor): 133 | None 134 | 135 | def create( 136 | self, 137 | file_name, 138 | create_options, 139 | granted_access, 140 | file_attributes, 141 | security_descriptor, 142 | allocation_size, 143 | ): 144 | file_name = self._replace_name(file_name) 145 | if self.fsspec_system.exists(file_name): 146 | raise NTStatusObjectNameCollision() 147 | if create_options & CREATE_FILE_CREATE_OPTIONS.FILE_DIRECTORY_FILE: 148 | self.fsspec_system.mkdir(file_name) 149 | if self.use_elf_mkdir: 150 | self.fsspec_system.touch(f'{file_name}/elf') 151 | return FalseOpen(file_name, create_options, granted_access) 152 | else: 153 | self.fsspec_system.touch(file_name) 154 | return self._open(file_name, 'wb') 155 | 156 | def rename(self, file_context, file_name, new_file_name, replace_if_exists): 157 | file_name = self._replace_name(file_name) 158 | new_file_name = self._replace_name(new_file_name) 159 | file_context.close() 160 | try: 161 | self.fsspec_system.mv_file(file_name, new_file_name, recursive=True) 162 | except Exception as e: 163 | self.fsspec_system.mv(file_name, new_file_name, recursive=True) 164 | 165 | def set_file_size(self, file_context, new_size, set_allocation_size): 166 | None 167 | 168 | def write(self, file_context, buffer, offset, write_to_end_of_file, constrained_io): 169 | assert not write_to_end_of_file 170 | assert not constrained_io 171 | if getattr(file_context, 'loc', None) != offset: 172 | file_context.seek(offset) 173 | file_context.write(bytes(buffer)) 174 | return len(buffer) 175 | 176 | def overwrite(self, file_context, file_attributes, replace_file_attributes: bool, allocation_size: int) -> None: 177 | try: 178 | file_context.truncate(0) 179 | file_context.seek(0) 180 | except io.UnsupportedOperation: 181 | self.fsspec_system.touch(file_context.file_name) 182 | 183 | # https://learn.microsoft.com/en-us/windows/win32/wmisdk/file-and-directory-access-rights-constants 184 | def open(self, file_name, create_options, granted_access): 185 | file_name = self._replace_name(file_name) 186 | 是文件夹 = file_name == '\\' or create_options & CREATE_FILE_CREATE_OPTIONS.FILE_DIRECTORY_FILE 187 | 是文件 = create_options & CREATE_FILE_CREATE_OPTIONS.FILE_NON_DIRECTORY_FILE 188 | if not 是文件夹 and not 是文件: 189 | if self.fsspec_system.isfile(file_name): 190 | 是文件 = True 191 | else: 192 | 是文件夹 = True 193 | if 是文件夹: 194 | return FalseOpen(file_name, create_options, granted_access) 195 | elif 是文件: 196 | if granted_access & 1 and granted_access & 2: 197 | mode = 'r+b' 198 | elif granted_access & 1: 199 | mode = 'rb' 200 | elif granted_access & 2: 201 | mode = 'ab' 202 | else: 203 | return FalseOpen(file_name, create_options, granted_access) 204 | return self._open(file_name, mode) 205 | 206 | def cleanup(self, file_context, file_name, flags) -> None: 207 | file_context.close() 208 | FspCleanupDelete = 0x01 209 | if flags & FspCleanupDelete: 210 | self.fsspec_system.rm(file_context.file_name, recursive=True) 211 | 212 | def get_volume_info(self): 213 | return { 214 | "total_size": 2 ** 30, 215 | "free_size": 2 ** 30, 216 | "volume_label": self._volume_label, 217 | } 218 | 219 | def get_file_info(self, file_context) -> dict: 220 | import time; time.sleep(0.01) 221 | return self._fs_info_to_file_info(self._get_info(file_context.file_name)) 222 | 223 | def set_basic_info(self, file_context, file_attributes, creation_time, last_access_time, last_write_time, change_time, file_info): 224 | return self._fs_info_to_file_info(self._get_info(file_context.file_name)) 225 | 226 | def read(self, file_context, offset: int, length: int) -> bytes: 227 | file_context.seek(offset, 0) 228 | return file_context.read(length) 229 | 230 | # python cleanup 231 | def close(self, file_context): 232 | None 233 | 234 | 235 | for k, v in [*Barbarossa.__dict__.items()]: 236 | if k[0] != '_' and isinstance(v, FunctionType): 237 | def 新f(self, *args, _k=k, _v=v): 238 | try: 239 | res = _v(self, *args) 240 | except Exception as e: 241 | res = e.__class__.__name__ 242 | raise 243 | finally: 244 | if self.log: 245 | if len(str(res)) < 1000: 246 | print(f'{_k}{str(args)} -> {res}') 247 | else: 248 | print(f'{_k}{str(args)}') 249 | return res 250 | setattr(Barbarossa, k, 新f) 251 | 252 | 253 | def fsspec_disk(mountpoint: str, fsspec_system: fsspec.AbstractFileSystem, **kwargs): 254 | mountpoint = Path(mountpoint) 255 | is_drive = mountpoint.parent == mountpoint 256 | reject_irp_prior_to_transact0 = not is_drive 257 | 258 | operations = Barbarossa(fsspec_system, **kwargs) 259 | fs = FileSystem( 260 | str(mountpoint), 261 | operations, 262 | sector_size=512, 263 | sectors_per_allocation_unit=1, 264 | volume_creation_time=filetime_now(), 265 | volume_serial_number=0, 266 | file_info_timeout=1000, 267 | case_sensitive_search=1, 268 | case_preserved_names=1, 269 | unicode_on_disk=1, 270 | persistent_acls=1, 271 | post_cleanup_when_modified_only=1, 272 | um_file_context_is_user_context2=1, 273 | file_system_name=str(mountpoint), 274 | reject_irp_prior_to_transact0=reject_irp_prior_to_transact0, 275 | ) 276 | return fs 277 | -------------------------------------------------------------------------------- /fsspec_disk/demo_s3.py: -------------------------------------------------------------------------------- 1 | import time 2 | import fire 3 | 4 | from fsspec.implementations.dirfs import DirFileSystem 5 | from fsspec.implementations.cached import SimpleCacheFileSystem 6 | import s3fs 7 | 8 | from fsspec_disk import fsspec_disk 9 | from fsspec_disk.utils import CacheInfoFileSystem 10 | 11 | 12 | def ember(bucket, endpoint_url, key, secret, volume_label='fsspec_disk'): 13 | s3 = CacheInfoFileSystem( 14 | SimpleCacheFileSystem(fs=DirFileSystem(path=bucket, fs=s3fs.S3FileSystem(endpoint_url=endpoint_url, key=key, secret=secret))) 15 | ) 16 | fs = fsspec_disk('u:', s3, use_elf_mkdir=True, log=False, volume_label=volume_label) 17 | try: 18 | fs.start() 19 | while True: 20 | time.sleep(1) 21 | finally: 22 | fs.stop() 23 | 24 | 25 | if __name__ == '__main__': 26 | fire.Fire(ember) 27 | -------------------------------------------------------------------------------- /fsspec_disk/utils.py: -------------------------------------------------------------------------------- 1 | from fsspec.spec import AbstractFileSystem 2 | 3 | 4 | class CacheInfoFileSystem: 5 | def __init__(self, fs: AbstractFileSystem): 6 | self._info_cache = {} 7 | self._fs = fs 8 | 9 | def info(self, path): 10 | if path not in self._info_cache: 11 | try: 12 | self._info_cache[path] = self._fs.info(path) 13 | except FileNotFoundError: 14 | self._info_cache[path] = 'not found' 15 | if self._info_cache[path] == 'not found': 16 | raise FileNotFoundError() 17 | else: 18 | return self._info_cache[path] 19 | 20 | def _magic(k): 21 | def _f(self, path, *t, **d): 22 | self._info_cache.pop(path, None) 23 | return getattr(self._fs, k)(path, *t, **d) 24 | return _f 25 | 26 | def _magic2(k): 27 | def _f(self, file_name, new_file_name, *t, **d): 28 | self._info_cache.pop(file_name, None) 29 | self._info_cache.pop(new_file_name, None) 30 | return getattr(self._fs, k)(file_name, new_file_name, *t, **d) 31 | return _f 32 | 33 | def open(self, path, mode, *t, **d): 34 | if mode in ['r', 'rb']: 35 | return self._fs.open(path, mode, *t, **d) 36 | self._info_cache.pop(path, None) 37 | o = self._fs.open(path, mode, *t, **d) 38 | _close = o.close 39 | def 假close(): 40 | r = _close() 41 | self._info_cache.pop(path, None) 42 | return r 43 | o.close = 假close 44 | return o 45 | 46 | mkdir = _magic('mkdir') 47 | rm = _magic('rm') 48 | touch = _magic('touch') 49 | mv_file = _magic2('mv_file') 50 | mv = _magic2('mv') 51 | 52 | def __getattr__(self, k): 53 | return getattr(self._fs, k) 54 | -------------------------------------------------------------------------------- /img/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RimoChan/fsspec_disk/eab6afd12241faf106a0e9a76e83bc510a0c01f9/img/1.png -------------------------------------------------------------------------------- /img/2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RimoChan/fsspec_disk/eab6afd12241faf106a0e9a76e83bc510a0c01f9/img/2.png -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # 万能硬盘！ 2 | 3 | 大家知道fsspec吗？fsspec是1个可以将各种东西映射为Python文件系统的库，比如可以open 1个云存储上的文件。 4 | 5 | 于是聪明的莉沫酱就想到，既然可以映射到Python文件系统，那是不是也可以进1步映射到windows的硬盘？ 6 | 7 | 这样1来，我们只要去弄1些免费的云存储，比如Cloudflare R2，就相当于多了1块免费的移动硬盘，真是太好啦！ 8 | 9 | 10 | ## 安装方法 11 | 12 | 1. 首先你要有1台装了Python的windows电脑。 13 | 14 | 2. 先安装1个[WinFSP](https://winfsp.dev/rel/)。 15 | 16 | 3. 然后`pip install git+https://github.com/RimoChan/fsspec_disk.git`就可以啦！ 17 | 18 | 19 | ## 样例 20 | 21 | 以cloudflare r2为例—— 22 | 23 | 首先我们去[cloudflare r2](https://www.cloudflare.com/developer-platform/products/r2/)，在仪表盘里面新建1个存储桶+1个API令牌。 24 | 25 | ![img/1.png](img/1.png) 26 | 27 | 接下来运行这个命令: 28 | 29 | ```sh 30 | python -m fsspec_disk.demo_s3 --bucket 你的存储桶的名字 --endpoint_url 你的endpoint_url --key 你的令牌的key --secret 你的令牌的secret --volume_label mydisk 31 | ``` 32 | 33 | 然后就可以在电脑里看到这个新硬盘了，这个硬盘里的东西就和你的存储桶里面是同步的。 34 | 35 | ![img/2.png](img/2.png) 36 | 37 | 你还可以把这个命令发给别人，这样就可以直接和他们共享云盘，非常方便！ 38 | 39 | 40 | ## 接口 41 | 42 | 接口只有1个，就是`fsspec_disk`。 43 | 44 | ```python 45 | def fsspec_disk(mountpoint: str, fsspec_system: fsspec.AbstractFileSystem, **kwargs) 46 | ``` 47 | 48 | - mountpoint: 要把盘挂在哪个盘符下。 49 | 50 | - fsspec_system: 1个fsspec的文件系统。 51 | 52 | - kwargs目前有3个，分别是: 53 | - log: 是否显示log 54 | - volume_label: 硬盘显示的名字 55 | - use_elf_mkdir: 是否使用假文件来创建目录，因为有些云存储不支持mkdir 56 | 57 | 58 | ## 结束 59 | 60 | 对了，这个东西还在开发中，所以应该会有bug，大家1定要把重要文件在本地做上备份，因为文件丢了我也不会给你赔钱的！ 61 | 62 | 就这样，我要去买SSD了，大家88！ 63 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | 4 | setuptools.setup( 5 | name='fsspec_disk', 6 | version='1.0.0', 7 | author='RimoChan', 8 | author_email='the@librian.net', 9 | description='fsspec_disk', 10 | long_description=open('readme.md', encoding='utf8').read(), 11 | long_description_content_type='text/markdown', 12 | url='https://github.com/RimoChan/fsspec_disk', 13 | packages=[ 14 | 'fsspec_disk', 15 | ], 16 | classifiers=[ 17 | 'Programming Language :: Python :: 3', 18 | 'Operating System :: OS Independent', 19 | ], 20 | install_requires=[ 21 | 'winfspy>=0.8.4', 22 | 's3fs>=2024.10.0', 23 | 'fsspec>=2024.10.0', 24 | ], 25 | ) 26 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import random 4 | import shutil 5 | from pathlib import Path 6 | 7 | from fsspec.implementations.local import LocalFileSystem 8 | from fsspec.implementations.dirfs import DirFileSystem 9 | from fsspec.implementations.cached import SimpleCacheFileSystem 10 | from fsspec.implementations.memory import MemoryFileSystem 11 | from fsspec.implementations.local import LocalFileOpener 12 | from fsspec.implementations.zip import ZipFileSystem 13 | 14 | from fsspec_disk import fsspec_disk 15 | from fsspec_disk.utils import CacheInfoFileSystem 16 | 17 | 18 | disk = 't' 19 | 20 | temp_dir = Path(__file__).parent.absolute() / 'temp' 21 | os.makedirs(temp_dir, exist_ok=True) 22 | 23 | 24 | def test( 25 | fsspec_system, 26 | use_elf_mkdir=False, 27 | log=True, 28 | assert_size=True, 29 | need_sleep_on_write=False, 30 | need_sleep_on_remove=False, 31 | ): 32 | def _sleep(t='write'): 33 | if t == 'write' and need_sleep_on_write: 34 | time.sleep(3) 35 | if t == 'remove' and need_sleep_on_remove: 36 | print('sleep!') 37 | time.sleep(3) 38 | fssd = fsspec_disk(mountpoint=f'{disk}:', fsspec_system=fsspec_system, use_elf_mkdir=use_elf_mkdir, log=log) 39 | fssd.start() 40 | try: 41 | x = random.randint(1000, 9999) 42 | for test_dir in [f'{disk}:/test_dir_{x}', f'{disk}:/test_dddir_{x}/test_dddir_{x}/']: 43 | os.makedirs(test_dir, exist_ok=True) 44 | 45 | assert Path(f'{test_dir}/1.txt').exists() == False 46 | assert len(os.listdir(f'{test_dir}')) == 0 47 | 48 | with open(f'{test_dir}/1.txt', 'w') as a: 49 | assert len(os.listdir(f'{test_dir}')) == 1 50 | a.write('123') 51 | _sleep() 52 | assert open(f'{test_dir}/1.txt', 'r').read() == '123' 53 | Path(f'{test_dir}/2.txt').touch() 54 | _sleep() 55 | assert len(os.listdir(f'{test_dir}')) == 2 56 | 57 | a = open(f'{test_dir}/1.txt', 'a') 58 | a.write('456') 59 | a.close() 60 | assert open(f'{test_dir}/1.txt', 'r').read() == '123456' 61 | if assert_size: 62 | assert os.stat(f'{test_dir}/1.txt').st_size == 6 63 | open(f'{test_dir}/1.txt', 'w').close() 64 | assert open(f'{test_dir}/1.txt', 'r').read() == '' 65 | if assert_size: 66 | assert os.stat(f'{test_dir}/1.txt').st_size == 0 67 | 68 | assert Path(f'{test_dir}/1.txt').exists() == True 69 | assert Path(f'{test_dir}/1.txt').is_file() == True 70 | assert Path(f'{test_dir}/1.txt').is_dir() == False 71 | os.remove(f'{test_dir}/1.txt') 72 | _sleep('remove') 73 | assert Path(f'{test_dir}/1.txt').exists() == False 74 | assert Path(f'{test_dir}/1.txt').is_file() == False 75 | assert Path(f'{test_dir}/1.txt').is_dir() == False 76 | 77 | for i in [1, 100, 10000, 1000000]: 78 | with open(f'{test_dir}/3.txt', 'wb') as a: 79 | a.write(b'123'*i) 80 | if assert_size: 81 | print('什么情况', fssd.operations.list_opened_files()) 82 | assert os.stat(f'{test_dir}/3.txt').st_size == 3 * i 83 | # exit() 84 | assert len(os.listdir(f'{test_dir}')) == 2 85 | os.rename(f'{test_dir}/3.txt', f'{test_dir}/33.txt') 86 | _sleep('remove') 87 | assert len(os.listdir(f'{test_dir}')) == 2 88 | os.remove(f'{test_dir}/33.txt') 89 | 90 | assert len(os.listdir(f'{test_dir}')) == 1 91 | 92 | shutil.rmtree(f'{test_dir}/') 93 | 94 | assert Path(f'{test_dir}').exists() == False 95 | print('好！', fsspec_system) 96 | finally: 97 | fssd.stop() 98 | 99 | 100 | if __name__ == '__main__': 101 | test(DirFileSystem(path=temp_dir, fs=LocalFileSystem())) 102 | 103 | test(SimpleCacheFileSystem(fs=DirFileSystem(path=temp_dir, fs=LocalFileSystem())), assert_size=False) 104 | 105 | test(MemoryFileSystem()) 106 | 107 | test(SimpleCacheFileSystem(fs=MemoryFileSystem())) 108 | --------------------------------------------------------------------------------