├── .gitattributes ├── .gitignore ├── README.md ├── package.json ├── src ├── anyfs │ ├── _ftp.ts │ ├── _fuse.ts │ ├── anyfs.ts │ ├── fs-chunk.ts │ ├── fs-file.ts │ ├── fs-folder.ts │ ├── fs-object.ts │ ├── ftp.ts │ ├── fuse.ts │ ├── index.ts │ ├── internal-types.ts │ ├── provider.ts │ ├── reader.ts │ └── writer.ts ├── examples │ └── local-fs.ts └── index.ts └── tsconfig.json /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !package.json 3 | !src 4 | !src/** 5 | !.gitignore 6 | !tsconfig.json 7 | !README.md 8 | !.gitattributes 9 | **/.DS_Store -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AnyFS 2 | 3 | A simple filesystem which stores metadata using JSON. Made with the goal of simplifying the creation of filesystems on places that are generally not supposed to store files. 4 | 5 | ## Usage 6 | 7 | The smallest unit of storage in AnyFS is called an object. Each object stores JSON data, and optionally a null byte followed by binary data. Object data is encrypted with AES-256. Each object has a unique ID which can either be a number or a string. 8 | 9 | Each AnyFS filesystem has a data provider. This data provider is responsible for creating, reading, updating and deleting objects, and nothing else. The only job of a data provider is storing and retrieving binary data. It does not need to (and should not try to) decrypt or parse this data. 10 | 11 | You can create your own AnyFS filesystem by implementing your own AnyFS data provider. The functions you need to implement are `createObject()`, `writeObject(objectID, data)` and `readObject(objectID)`. There is also an optional `deleteObject(objectID)` method. Filesystem functionality will not be affected if it is missing, but it is highly recommended that you implement it if possible. 12 | 13 | See [src/anyfs/provider.ts](https://github.com/pixelomer/AnyFS/blob/main/src/anyfs/provider.ts) for details on implementing a new AnyFS data provider. 14 | 15 | ### Usage Examples 16 | 17 | - See [src/examples/local-fs.ts](https://github.com/pixelomer/AnyFS/blob/main/src/examples/local-fs.ts) and [src/index.ts](https://github.com/pixelomer/AnyFS/blob/main/src/index.ts) for an example AnyFS implementation that stores objects as files. Also useful when debugging AnyFS itself. 18 | - [discord-fs](https://github.com/pixelomer/discord-fs) is an AnyFS filesystem for storing files as Discord messages. 19 | 20 | ## Functionality 21 | 22 | AnyFS provides a FUSE filesystem and an FTP server for accessing filesystems. You may also use the `AnyFS` class in your code to access files programmatically. However, in its current state, the `AnyFS` class only contains very low level functions and may be hard to use. This is planned to be fixed in future versions. 23 | 24 | The FTP implementation does not have any known issues. However, the FUSE implementation is known to corrupt data on write, so it is recommended that you only use it in read-only mode. This may be an issue with AnyFS or node-fuse-bindings but the culprit is not known at this time. 25 | 26 | ## LocalFS Usage 27 | 28 | AnyFS comes with LocalFS, an AnyFS data provider that stores AnyFS data in the local filesystem. This is useful for debugging purposes but not for much else. You can mount a LocalFS filesystem with the following command. 29 | 30 | ```bash 31 | npm start /path/to/storage /path/to/mnt 32 | ``` 33 | 34 | This will mount a read-only LocalFS filesystem at `/path/to/mnt` using the data in `/path/to/storage`, initializing a new filesystem if necessary. Additionally, an FTP server will be started on `http://127.0.0.1:2121`. -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "anyfs", 3 | "version": "1.0.0", 4 | "description": "Turn anything into a filesystem", 5 | "main": "dist/index.js", 6 | "scripts": { 7 | "start": "tsc && node .", 8 | "build": "tsc", 9 | "test": "echo \"Error: no test specified\" && exit 1" 10 | }, 11 | "keywords": [ 12 | "fs", 13 | "fuse" 14 | ], 15 | "author": "pixelomer", 16 | "license": "UNLICENSED", 17 | "dependencies": { 18 | "typescript": "^4.5.5" 19 | }, 20 | "optionalDependencies": { 21 | "ftp-srv": "^4.5.0", 22 | "fuse-bindings": "npm:node-fuse-bindings@2.12.4" 23 | }, 24 | "devDependencies": { 25 | "@types/node": "^17.0.10" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/anyfs/_ftp.ts: -------------------------------------------------------------------------------- 1 | import { AnyFSFolder } from "./fs-folder"; 2 | import { Readable, Writable } from "stream"; 3 | import { AnyFS } from "./anyfs"; 4 | import { AnyFSFile } from "./fs-file"; 5 | //@ts-ignore 6 | import { FileSystem } from "ftp-srv"; 7 | 8 | class AnyFSFTPFileSystem extends FileSystem { 9 | anyfsCwd: AnyFSFolder; 10 | 11 | async _get(cwd: AnyFSFolder, filename: string): Promise { 12 | const file = await cwd.atPath(filename); 13 | if (file == null) { 14 | throw new Error("No such file or directory."); 15 | } 16 | const basename = AnyFSFolder.basename(filename); 17 | if (file.isFolder()) { 18 | return { 19 | name: basename, 20 | isDirectory: () => true, 21 | size: 1, 22 | atime: new Date(0), 23 | mtime: new Date(0), 24 | ctime: new Date(0), 25 | uid: 0, 26 | gid: 0 27 | }; 28 | } 29 | else if (file.isFile()) { 30 | const stat = await file.stat(); 31 | return { 32 | name: basename, 33 | isDirectory: () => false, 34 | size: stat.size, 35 | atime: new Date(0), 36 | mtime: new Date(0), 37 | ctime: new Date(0), 38 | uid: 0, 39 | gid: 0 40 | }; 41 | } 42 | } 43 | 44 | get(filename: string): Promise { 45 | return this._get(this.anyfsCwd, filename); 46 | } 47 | 48 | async list(path: string = ".") { 49 | const file = await this.anyfsCwd.atPath(path); 50 | if (file == null) { 51 | throw new Error("No such file or directory."); 52 | } 53 | if (!file.isFolder()) { 54 | throw new Error("Not a directory."); 55 | } 56 | const contents = await this.anyfsCwd.listContents(); 57 | const list = []; 58 | for (const item of contents) { 59 | try { 60 | list.push(await this._get(file, item.name)); 61 | } 62 | catch {} 63 | } 64 | return list; 65 | } 66 | 67 | currentDirectory(): string { 68 | return this.anyfsCwd.getAbsolutePath(); 69 | } 70 | 71 | async chdir(path: string = "."): Promise { 72 | const newCwd = await this.anyfsCwd.atPath(path); 73 | if (newCwd == null) { 74 | throw new Error("No such file or directory."); 75 | } 76 | else if (!newCwd.isFolder()) { 77 | throw new Error("Not a directory."); 78 | } 79 | else { 80 | this.anyfsCwd = newCwd; 81 | } 82 | return this.anyfsCwd.getAbsolutePath(); 83 | } 84 | 85 | async write(filename: string, options?: { append?: boolean; start?: any; }): Promise { 86 | const parent = await this.anyfsCwd.parentForPath(filename); 87 | const basename = AnyFSFolder.basename(filename); 88 | let file: AnyFSFile; 89 | if (!(await parent.exists(basename))) { 90 | file = await parent.createFile(basename); 91 | } 92 | else { 93 | const atPath = await this.anyfsCwd.atPath(filename); 94 | if (!atPath.isFile()) { 95 | throw new Error("Is a directory."); 96 | } 97 | file = atPath; 98 | } 99 | const stat = await file.stat(); 100 | const append = (options?.append != null) ? !!options.append : false; 101 | const start = (typeof options?.start === 'number') ? options.start : 0; 102 | if (!append && (start !== stat.size)) { 103 | throw new Error(`Writing in the middle of a file is not supported. Delete and reupload instead. (filesize=${stat.size}, offset=${start})`); 104 | } 105 | const stream = new Writable({ 106 | write: async function(chunk, encoding, callback) { 107 | if (!(chunk instanceof Buffer)) { 108 | chunk = Buffer.from(chunk); 109 | } 110 | try { 111 | await file.append(chunk); 112 | callback(); 113 | } 114 | catch (err) { 115 | callback(err); 116 | } 117 | } 118 | }); 119 | return { 120 | stream: stream, 121 | clientPath: file.getAbsolutePath() 122 | } 123 | } 124 | 125 | async read(filename: string, options?: { start?: any; }): Promise { 126 | const file = await this.anyfsCwd.atPath(filename); 127 | if (file == null) { 128 | throw new Error("No such file or directory."); 129 | } 130 | else if (!file.isFile()) { 131 | throw new Error("Is a directory.") 132 | } 133 | if (options?.start == null) { 134 | options = { start: 0 }; 135 | } 136 | let seek = options.start; 137 | return { 138 | stream: new Readable({ 139 | read: async function(size) { 140 | if (seek == null) { 141 | return null; 142 | } 143 | const newSeek = seek + size; 144 | let data: Buffer; 145 | try { 146 | data = await file.read(seek, size); 147 | } 148 | catch (err) { 149 | this.destroy(err); 150 | return; 151 | } 152 | if (data.length === 0) { 153 | this.push(null); 154 | } 155 | else { 156 | this.push(data); 157 | } 158 | if (data.length !== size) { 159 | seek = null; 160 | this.push(null); 161 | } 162 | else { 163 | seek = newSeek; 164 | } 165 | } 166 | }), 167 | clientPath: file.getAbsolutePath() 168 | }; 169 | } 170 | 171 | async delete(path: string) { 172 | const parent = await this.anyfsCwd.parentForPath(path, true); 173 | await parent.deleteEntry(AnyFSFolder.basename(path)); 174 | } 175 | 176 | async mkdir(path: string) { 177 | const parent = await this.anyfsCwd.parentForPath(path, false); 178 | await parent.createFolder(AnyFSFolder.basename(path)); 179 | } 180 | 181 | async rename(from: string, to: string) { 182 | const sourceFile = await this.anyfsCwd.atPath(from); 183 | if (sourceFile == null) { 184 | throw new Error("No such file or directory."); 185 | } 186 | const sourceParent = sourceFile.parent; 187 | const targetParent = await this.anyfsCwd.parentForPath(to, false); 188 | const sourceBasename = AnyFSFolder.basename(from); 189 | const targetBasename = AnyFSFolder.basename(to); 190 | await targetParent.link(targetBasename, sourceFile.isFile() ? "file" : "folder", sourceFile.objectID, true); 191 | await sourceParent.deleteEntry(sourceBasename, true); 192 | } 193 | 194 | async chmod(path: string, mode: string) { 195 | throw new Error("Operation not supported."); 196 | } 197 | } 198 | 199 | export async function getFTP(anyfs: AnyFS) { 200 | const root = await anyfs.root(); 201 | return new (class extends AnyFSFTPFileSystem { 202 | constructor() { 203 | //@ts-ignore 204 | super(...arguments); 205 | this.anyfsCwd = root; 206 | } 207 | }); 208 | } -------------------------------------------------------------------------------- /src/anyfs/_fuse.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import path from "path"; 3 | import os from "os"; 4 | import crypto from "crypto"; 5 | import util from "util"; 6 | import { AnyFSFile } from "./fs-file"; 7 | import { AnyFS } from "./anyfs"; 8 | import { AnyFSFolder } from "./fs-folder"; 9 | import { AnyFSMountOptions } from "./fuse"; 10 | //@ts-ignore 11 | import fuse from "fuse-bindings"; 12 | 13 | class OpenFile { 14 | path: string 15 | fd: number 16 | file: AnyFSFile 17 | flags: number 18 | tmpdir: string 19 | previousWrite: { 20 | position: number, 21 | data: Buffer 22 | } 23 | 24 | private _IOLock: Promise; 25 | private _writeLock: Promise; 26 | private _localFilePath: string; 27 | private _localFileHandle: fs.promises.FileHandle; 28 | private _closed: boolean; 29 | 30 | constructor(file: AnyFSFile, path: string, flags: number, fd: number, tmpdir: string) { 31 | this.fd = fd; 32 | this.path = path; 33 | this.file = file; 34 | this.flags = flags; 35 | this.tmpdir = tmpdir; 36 | } 37 | 38 | private async _canPerformIO() { 39 | await this._IOLock; 40 | if (this._closed) { 41 | throw new Error("Attempted to perform I/O on a closed file.") 42 | } 43 | } 44 | 45 | private async _cleanup() { 46 | if (this._localFileHandle != null) { 47 | try { await this._localFileHandle.close() } 48 | catch {} 49 | this._localFileHandle = null; 50 | } 51 | if (this._localFilePath != null) { 52 | try { fs.unlinkSync(this._localFilePath); } 53 | catch {} 54 | this._localFilePath = null; 55 | } 56 | } 57 | 58 | async close() { 59 | await this._IOLock; 60 | this._closed = true; 61 | if (this._localFileHandle != null) { 62 | await this._localFileHandle.close(); 63 | const fileData = await fs.promises.readFile(this._localFilePath); 64 | await this.file.writeAll(fileData); 65 | } 66 | await this._cleanup(); 67 | } 68 | 69 | async write(data: Buffer, position: number): Promise { 70 | await this._canPerformIO(); 71 | if (this._localFileHandle == null) { 72 | // Lock I/O until the download is finished 73 | let unlockIO: () => void; 74 | this._IOLock = new Promise((resolve) => { 75 | unlockIO = resolve; 76 | }); 77 | 78 | try { 79 | // Download the whole file 80 | const objectIDHash = crypto 81 | .createHash('sha256') 82 | .update(this.file.objectID.toString()) 83 | .digest('hex') 84 | + `_${this.fd.toString()}`; 85 | this._localFilePath = path.join(this.tmpdir, objectIDHash); 86 | const writeStream = fs.createWriteStream(this._localFilePath); 87 | try { 88 | await this.file.readAll(async(chunk, index, total) => { 89 | await util.promisify(writeStream.write.bind(writeStream))(chunk); 90 | }); 91 | } 92 | finally { 93 | await util.promisify(writeStream.end.bind(writeStream))(); 94 | await util.promisify(writeStream.close.bind(writeStream))(); 95 | } 96 | this._localFileHandle = await fs.promises.open(this._localFilePath, "r+"); 97 | } 98 | catch (err) { 99 | // Something went wrong, delete the file and rethrow 100 | await this._cleanup(); 101 | throw err; 102 | } 103 | finally { 104 | // Unlock I/O 105 | unlockIO(); 106 | } 107 | } 108 | 109 | await this._writeLock; 110 | let unlockWrite: () => void; 111 | this._writeLock = new Promise((resolve) => { 112 | unlockWrite = resolve; 113 | }); 114 | let bytesWritten: number; 115 | try { 116 | /*console.log("Writing data", { 117 | data: data, 118 | plaintextData: data.toString('utf-8'), 119 | offset: 0, 120 | length: data.length, 121 | positon: position 122 | })*/ 123 | const result = await this._localFileHandle.write(data, 0, data.length, position); 124 | //await this._localFileHandle.sync(); 125 | this.previousWrite = { data, position }; 126 | bytesWritten = result.bytesWritten; 127 | } 128 | finally { 129 | unlockWrite(); 130 | } 131 | return bytesWritten; 132 | } 133 | 134 | async read(position: number, length: number): Promise { 135 | await this._canPerformIO() 136 | if (this._localFileHandle != null) { 137 | const buffer = Buffer.allocUnsafe(length); 138 | const result = await this._localFileHandle.read(buffer, 0, length, position); 139 | const finalBuffer = Buffer.allocUnsafe(result.bytesRead); 140 | result.buffer.copy(finalBuffer); 141 | return finalBuffer; 142 | } 143 | else { 144 | return await this.file.read(position, length); 145 | } 146 | } 147 | } 148 | 149 | export async function fuseMount(FS: AnyFS, mountPoint: string, options?: AnyFSMountOptions, onDestroy?: () => void) { 150 | mountPoint = fs.realpathSync(mountPoint); 151 | options = options ? { ...options } : {}; 152 | const log = (options.verbose ?? false) ? console.log.bind(console) : ()=>{}; 153 | const isReadonly = options.allowWrite ? !options.allowWrite : true; 154 | const blockSize = options.reportedBlockSize ?? 512; 155 | const blockCount = options.reportedBlocks ?? (1024 * 1024); 156 | 157 | const root = await FS.root(); 158 | 159 | const tmpdir = fs.mkdtempSync(path.join(os.tmpdir(), "anyfs")); 160 | 161 | // Used for silencing TypeScript errors related to 162 | // interface properties 163 | const nothing: any = {}; 164 | 165 | let nextFd = 0; 166 | const openFiles = new Map(); 167 | 168 | const filesystem: fuse.MountOptions = { 169 | async getattr(path, callback: (code: number, stats?: fuse.Stats) => void) { 170 | log("getattr(%s)", path); 171 | try { 172 | const file = await FS.atPath(path); 173 | if (file == null) { 174 | callback(fuse.ENOENT); 175 | return; 176 | } 177 | const stats: fuse.Stats = { 178 | ...nothing, 179 | mtime: new Date(0), 180 | atime: new Date(0), 181 | ctime: new Date(0), 182 | mode: 0o777, 183 | uid: process.getuid(), 184 | gid: process.getgid() 185 | }; 186 | if (file instanceof AnyFSFolder) { 187 | stats.mode |= fs.constants.S_IFDIR; 188 | stats.nlink = 2; 189 | stats.size = 8; 190 | } 191 | else if (file instanceof AnyFSFile) { 192 | stats.mode |= fs.constants.S_IFREG; 193 | const realStats = await file.stat(); 194 | stats.size = realStats.size; 195 | stats.nlink = 1; 196 | } 197 | else { 198 | callback(fuse.ENOENT); 199 | return; 200 | } 201 | callback(0, stats); 202 | } 203 | catch (err) { 204 | log(err); 205 | callback(fuse.EIO); 206 | } 207 | }, 208 | 209 | async readdir(path, callback: (code: number, dirs: string[]) => void) { 210 | log("readdir(%s)", path); 211 | try { 212 | const file = await FS.atPath(path); 213 | if (file == null) { 214 | callback(fuse.ENOENT, []); 215 | return; 216 | } 217 | else if (!(file instanceof AnyFSFolder)) { 218 | callback(fuse.ENOTDIR, []); 219 | return; 220 | } 221 | const contents = await file.listContents(); 222 | callback(0, contents.map((a) => a.name)); 223 | } 224 | catch (err) { 225 | log(err); 226 | callback(fuse.EIO, []); 227 | } 228 | }, 229 | 230 | async open(path, flags, callback: (code: number, fd: number) => void) { 231 | let fd = null; 232 | try { 233 | const file = await FS.atPath(path); 234 | if (file == null) { 235 | callback(fuse.ENOENT, -1); 236 | return; 237 | } 238 | else if (!(file instanceof AnyFSFile)) { 239 | callback(fuse.EISDIR, -1); 240 | return; 241 | } 242 | fd = nextFd++; 243 | const openFile = new OpenFile(file, path, flags, fd, tmpdir); 244 | openFiles.set(fd, openFile); 245 | callback(0, fd); 246 | } 247 | catch (err) { 248 | log(err); 249 | callback(fuse.EIO, -1); 250 | } 251 | finally { 252 | log("open(%s, %d)" + ((fd != null) ? ` = ${fd}` : ""), path, flags); 253 | } 254 | }, 255 | 256 | async mknod(path: string, mode: number, dev: number, callback: (code: number) => void) { 257 | if (isReadonly) { 258 | callback(fuse.EROFS); 259 | return; 260 | } 261 | log("mknod(%s, %d, %d)", path, mode, dev); 262 | try { 263 | if (!(mode & fs.constants.S_IFREG)) { 264 | callback(fuse.EPERM); 265 | return; 266 | } 267 | const parent = await root.parentForPath(path); 268 | await parent.createFile(AnyFSFolder.basename(path)); 269 | callback(0); 270 | } 271 | catch (err) { 272 | log(err); 273 | if (err?.message === 'No such file or directory.') { 274 | callback(fuse.ENOENT); 275 | } 276 | else if (err?.message === 'File exists.') { 277 | callback(fuse.EEXIST); 278 | } 279 | else { 280 | callback(fuse.EIO); 281 | } 282 | } 283 | }, 284 | 285 | async read(path, fd, buffer, length, position, callback: (bytesReadOrError: number) => void) { 286 | log("read(%s, %d, , %d, %d)", path, fd, length, position); 287 | try { 288 | if (!openFiles.has(fd)) { 289 | throw new Error("openFiles does not have fd: " + fd); 290 | } 291 | const file = openFiles.get(fd); 292 | const data = await file.read(position, length); 293 | const copied = data.copy(buffer); 294 | callback(copied); 295 | } 296 | catch (err) { 297 | log(err); 298 | callback(fuse.EIO); 299 | } 300 | }, 301 | 302 | async truncate(path: string, size: number, callback: (code: number) => void) { 303 | if (isReadonly) { 304 | callback(fuse.EROFS); 305 | return; 306 | } 307 | log("truncate(%s, %d)", path, size); 308 | try { 309 | const file = await root.atPath(path); 310 | if (file == null) { 311 | callback(fuse.ENOENT); 312 | return; 313 | } 314 | else if (!(file instanceof AnyFSFile)) { 315 | callback(fuse.EISDIR); 316 | return; 317 | } 318 | const contents = await file.readAll(); 319 | if (contents.length > size) { 320 | await file.writeAll(contents.slice(0, size)); 321 | } 322 | else if (contents.length < size) { 323 | //FIXME: This is a waste of RAM 324 | const buffer = Buffer.alloc(size - contents.length); 325 | await file.append(buffer); 326 | } 327 | callback(0); 328 | } 329 | catch (err) { 330 | log(err); 331 | callback(fuse.EIO); 332 | } 333 | }, 334 | 335 | async release(path: string, fd: number, callback: (code: number) => void) { 336 | log("release(%s, %d)", path, fd); 337 | const openFile = openFiles.get(fd); 338 | if (openFile != null) { 339 | await openFile.close(); 340 | } 341 | openFiles.delete(fd); 342 | callback(0); 343 | }, 344 | 345 | async rmdir(path: string, callback: (code: number) => void) { 346 | if (isReadonly) { 347 | callback(fuse.EROFS); 348 | return; 349 | } 350 | log("rmdir(%s)", path); 351 | try { 352 | const file = await FS.atPath(path); 353 | if (!(file instanceof AnyFSFolder)) { 354 | callback(fuse.ENOTDIR); 355 | return; 356 | } 357 | else if (file.objectID === root.objectID) { 358 | callback(fuse.EPERM); 359 | return; 360 | } 361 | const contents = await file.listContents(); 362 | if (contents.length !== 0) { 363 | callback(fuse.ENOTEMPTY); 364 | return; 365 | } 366 | //@ts-ignore 367 | const parent: AnyFSFolder = file.parent; 368 | await parent.deleteEntry(file.name); 369 | callback(0); 370 | } 371 | catch (err) { 372 | log(err); 373 | callback(fuse.EIO); 374 | } 375 | }, 376 | 377 | async access(path: string, mode: number, callback: (code: number) => void) { 378 | // access() is called a lot 379 | //log("access(%s, %d)", path, mode); 380 | try { 381 | const file = await root.atPath(path); 382 | if (file == null) { 383 | callback(fuse.ENOENT); 384 | return; 385 | } 386 | if (isReadonly && (mode & fs.constants.W_OK)) { 387 | callback(fuse.EROFS); 388 | } 389 | else { 390 | callback(0); 391 | } 392 | } 393 | catch (err) { 394 | log(err); 395 | callback(fuse.EIO); 396 | } 397 | }, 398 | 399 | async unlink(path: string, callback: (code: number) => void) { 400 | if (isReadonly) { 401 | callback(fuse.EROFS); 402 | return; 403 | } 404 | log("unlink(%s)", path); 405 | try { 406 | const file = await FS.atPath(path); 407 | if (!(file instanceof AnyFSFile)) { 408 | callback(fuse.EISDIR); 409 | return; 410 | } 411 | //@ts-ignore 412 | const parent: AnyFSFolder = file.parent; 413 | await parent.deleteEntry(file.name); 414 | callback(0); 415 | } 416 | catch (err) { 417 | log(err); 418 | callback(fuse.EIO); 419 | } 420 | }, 421 | 422 | async mkdir(path: string, mode: number, callback: (code: number) => void) { 423 | if (isReadonly) { 424 | callback(fuse.EROFS); 425 | return; 426 | } 427 | log("mkdir(%s, %d)", path, mode); 428 | try { 429 | const parent = await root.parentForPath(path); 430 | await parent.createFolder(AnyFSFolder.basename(path)); 431 | callback(0); 432 | } 433 | catch (err) { 434 | log(err); 435 | if (err?.message === 'No such file or directory.') { 436 | callback(fuse.ENOENT); 437 | } 438 | else if (err?.message === 'File exists.') { 439 | callback(fuse.EEXIST); 440 | } 441 | else { 442 | callback(fuse.EIO); 443 | } 444 | } 445 | }, 446 | 447 | async write(path: string, fd: number, data: Buffer, length: number, position: number, callback: (bytesWrittenOrError: number) => void) { 448 | if (isReadonly) { 449 | callback(fuse.EROFS); 450 | return; 451 | } 452 | log("write(%s, %d, , %d, %d)", path, fd, data.length, length, position); 453 | try { 454 | const file = openFiles.get(fd); 455 | const copiedData = Buffer.alloc(data.length); 456 | data.copy(copiedData); 457 | const writtenBytes = await file.write(copiedData, position); 458 | callback(writtenBytes); 459 | } 460 | catch (err) { 461 | log(err); 462 | callback(fuse.EIO); 463 | } 464 | }, 465 | 466 | async rename(source: string, dest: string, callback: (code: number) => void) { 467 | if (isReadonly) { 468 | callback(fuse.EROFS); 469 | return; 470 | } 471 | log("rename(%s, %s)", source, dest); 472 | try { 473 | const sourceFile = await root.atPath(source); 474 | //@ts-ignore 475 | const sourceDirectory: AnyFSFolder = sourceFile.parent; 476 | const sourceBasename = sourceFile.name; 477 | const targetDirectory = await root.parentForPath(dest); 478 | const targetBasename = AnyFSFolder.basename(dest); 479 | const type = (sourceFile instanceof AnyFSFile) ? "file" : "folder"; 480 | await targetDirectory.link(targetBasename, type, sourceFile.objectID, true); 481 | await sourceDirectory.deleteEntry(sourceBasename, true); 482 | callback(0); 483 | } 484 | catch (err) { 485 | log(err); 486 | if (err.message === 'File exists.') { 487 | callback(fuse.EEXIST); 488 | } 489 | if (err.message === 'Is a directory.') { 490 | callback(fuse.EISDIR); 491 | } 492 | else if (err.message === 'No such file or directory.') { 493 | callback(fuse.ENOENT); 494 | } 495 | else { 496 | callback(fuse.EIO); 497 | } 498 | } 499 | }, 500 | 501 | chmod(path: string, mode: number, callback: (code: number) => void) { 502 | if (isReadonly) { 503 | callback(fuse.EROFS); 504 | return; 505 | } 506 | callback(0); 507 | }, 508 | 509 | setxattr(path: string, name: string, buffer: Buffer, length: number, offset: number, flags: number, callback: (code: number) => void) { 510 | callback(0); 511 | }, 512 | 513 | listxattr(path: string, buffer: Buffer, length: number, callback: (code: number, reqBufSize: number) => void) { 514 | callback(0, 0); 515 | }, 516 | 517 | getxattr(path: string, name: string, buffer: Buffer, length: number, offset: number, callback: (code: number) => void) { 518 | // ENOATTR 519 | callback(-93); 520 | }, 521 | 522 | removexattr(path: string, name: string, callback: (code: number) => void) { 523 | callback(0); 524 | }, 525 | 526 | statfs(path: string, callback: (code: number, fsStat: fuse.FSStat) => void) { 527 | callback(0, { 528 | ...nothing, 529 | bsize: blockSize, 530 | blocks: blockCount, 531 | ffree: blockCount, 532 | bfree: blockCount, 533 | bavail: blockCount, 534 | namemax: 256 535 | }) 536 | }, 537 | 538 | destroy(callback: (code: number) => void) { 539 | callback(0); 540 | if (onDestroy != null) { 541 | onDestroy(); 542 | } 543 | } 544 | } 545 | 546 | fuse.mount(mountPoint, filesystem, (err) => { 547 | if (err == null) { 548 | // Exit cleanly on interrupt 549 | const interruptSignals: NodeJS.Signals[] = [ "SIGINT" ]; 550 | for (const signal of interruptSignals) { 551 | process.on(signal, (signal) => { 552 | log(`Received ${signal}, cleaning up...`); 553 | fuse.unmount(mountPoint, (err) => { 554 | if (err == null) { 555 | log(`Unmounted ${mountPoint}`); 556 | process.exit(0); 557 | } 558 | else { 559 | log(`Unmount failed: ${err}`); 560 | process.exit(1); 561 | } 562 | }); 563 | }); 564 | } 565 | 566 | log(`Mounted on ${mountPoint}`); 567 | } 568 | else { 569 | log(`Mount failed: ${err}`); 570 | throw err; 571 | } 572 | }); 573 | } -------------------------------------------------------------------------------- /src/anyfs/anyfs.ts: -------------------------------------------------------------------------------- 1 | import { AnyFSProvider } from "./provider"; 2 | import { AnyFSReader } from "./reader"; 3 | import { AnyFSWriter } from "./writer"; 4 | import { AnyFSFile } from "./fs-file"; 5 | import { AnyFSFolder } from "./fs-folder"; 6 | import { ObjectID, AnyFSObjectRaw, AnyFSFolderMetadata } from "./internal-types"; 7 | import * as fuse from "./fuse"; 8 | import { getFTP } from "./ftp"; 9 | 10 | export class AnyFS { 11 | _AESKey: Buffer; 12 | _cache = new Map>(); 13 | _FSProvider: AnyFSProvider; 14 | chunkSize: number; 15 | 16 | private _rootObjectID: ObjectID; 17 | private _currentReaders = new Set(); 18 | private _currentWriter: AnyFSWriter = null; 19 | private _awaitingReaders = new Set(); 20 | private _awaitingWriters = new Array(); 21 | 22 | _getRead() { 23 | return new Promise((resolve) => { 24 | if ((this._currentWriter == null) && (this._awaitingWriters.length === 0)) { 25 | const reader = new AnyFSReader(this); 26 | this._currentReaders.add(reader); 27 | resolve(reader); 28 | } 29 | else { 30 | this._awaitingReaders.add(resolve); 31 | } 32 | }); 33 | } 34 | 35 | private _resolveNextWriter() { 36 | const resolve = this._awaitingWriters.shift(); 37 | if (resolve == null) { 38 | return; 39 | } 40 | this._currentWriter = new AnyFSWriter(this); 41 | resolve(this._currentWriter); 42 | } 43 | 44 | _getWrite() { 45 | return new Promise((resolve) => { 46 | this._awaitingWriters.push(resolve); 47 | if (this._currentReaders.size === 0) { 48 | this._resolveNextWriter(); 49 | } 50 | }); 51 | } 52 | 53 | _release(readerOrWriter: AnyFSReader | AnyFSWriter) { 54 | if (this._currentWriter === readerOrWriter) { 55 | this._currentWriter = null; 56 | const willResolve = this._awaitingReaders; 57 | this._awaitingReaders = new Set(); 58 | for (const resolve of willResolve) { 59 | const reader = new AnyFSReader(this); 60 | this._currentReaders.add(reader); 61 | resolve(reader); 62 | } 63 | } 64 | else { 65 | this._currentReaders.delete(readerOrWriter); 66 | } 67 | if (this._currentReaders.size === 0) { 68 | this._resolveNextWriter(); 69 | } 70 | } 71 | 72 | async atPath(path: string): Promise { 73 | return await (await this.root()).atPath(path); 74 | } 75 | 76 | async root(): Promise { 77 | // Initialize if necessary 78 | let reader: AnyFSReader = await this._getRead(); 79 | try { 80 | await reader.readObject(this._rootObjectID); 81 | } 82 | catch { 83 | reader.release(); 84 | const writer = await this._getWrite(); 85 | reader = writer; 86 | await writer.writeObject(this._rootObjectID, { 87 | metadata: { 88 | type: "folder", 89 | entries: [] 90 | }, 91 | data: null 92 | }); 93 | } 94 | finally { 95 | reader.release(); 96 | } 97 | return new AnyFSFolder(this, null, "/", this._rootObjectID); 98 | } 99 | 100 | fuseMount(mountPoint: string, options?: fuse.AnyFSMountOptions, onDestroy?: () => void) { 101 | return fuse.fuseMount(this, mountPoint, options, onDestroy); 102 | } 103 | 104 | getFTP() { 105 | return getFTP(this); 106 | } 107 | 108 | constructor(FSProvider: AnyFSProvider, AESKey: Buffer, chunkSize: number, rootID: ObjectID) { 109 | this._FSProvider = FSProvider; 110 | this._AESKey = AESKey; 111 | this._rootObjectID = rootID; 112 | this.chunkSize = chunkSize; 113 | } 114 | } -------------------------------------------------------------------------------- /src/anyfs/fs-chunk.ts: -------------------------------------------------------------------------------- 1 | import { AnyFS } from "./anyfs"; 2 | import { AnyFSObject } from "./fs-object"; 3 | import { AnyFSDataMetadata, AnyFSFileMetadata, AnyFSFileStat, ObjectID } from "./internal-types"; 4 | import { AnyFSReader } from "./reader"; 5 | import { AnyFSWriter } from "./writer"; 6 | 7 | export class AnyFSFileChunk extends AnyFSObject { 8 | constructor(FS: AnyFS, parent: AnyFSObject, objectID: ObjectID) { 9 | super(FS, parent, null, objectID); 10 | } 11 | 12 | async updateData(newData: Buffer, writer?: AnyFSWriter) { 13 | let _writer: AnyFSWriter; 14 | if (writer == null) { 15 | _writer = await this.FS._getWrite(); 16 | writer = _writer; 17 | } 18 | try { 19 | await writer.writeObject(this.objectID, { 20 | metadata: { type: 'data' }, 21 | data: newData 22 | }); 23 | } 24 | finally { 25 | _writer?.release(); 26 | } 27 | } 28 | 29 | static async create(FS: AnyFS, parent: AnyFSObject, writer: AnyFSWriter, data?: Buffer): Promise { 30 | const objectID = await writer.createObject(); 31 | const chunk = new this(FS, parent, objectID); 32 | await chunk.updateData(data ?? Buffer.alloc(0), writer); 33 | return chunk; 34 | } 35 | } -------------------------------------------------------------------------------- /src/anyfs/fs-file.ts: -------------------------------------------------------------------------------- 1 | import { AnyFSFolder } from "."; 2 | import { AnyFS } from "./anyfs"; 3 | import { AnyFSFileChunk } from "./fs-chunk"; 4 | import { AnyFSObject } from "./fs-object"; 5 | import { AnyFSDataMetadata, AnyFSFileMetadata, AnyFSFileStat } from "./internal-types"; 6 | import { AnyFSReader } from "./reader"; 7 | import { AnyFSWriter } from "./writer"; 8 | 9 | export type AnyFSFileReadCallback = (chunk: Buffer, index: number, total: number) => unknown; 10 | 11 | export interface AnyFSFile { 12 | parent: AnyFSFolder; 13 | } 14 | 15 | export class AnyFSFile extends AnyFSObject { 16 | static async create(FS: AnyFS, parent: AnyFSObject, name: string): Promise { 17 | const writer = await FS._getWrite(); 18 | try { 19 | const objectID = await writer.createObject(); 20 | await writer.writeObject(objectID, { 21 | metadata: { 22 | type: "file", 23 | chunks: [], 24 | size: 0 25 | }, 26 | data: null 27 | }); 28 | return new this(FS, parent, name, objectID); 29 | } 30 | finally { 31 | writer.release(); 32 | } 33 | } 34 | 35 | async readAll(): Promise; 36 | async readAll(progress: AnyFSFileReadCallback): Promise; 37 | 38 | async readAll(progress?: AnyFSFileReadCallback): Promise { 39 | const reader = await this.FS._getRead(); 40 | try { 41 | const metadata = (await reader.readObject(this.objectID)).metadata; 42 | if (progress != null) { 43 | const chunkSize = this.FS.chunkSize; 44 | const count = Math.ceil(metadata.size / this.FS.chunkSize); 45 | for (let i=0; i { 61 | if ((position < 0) || (length < 0)) { 62 | throw new Error("Attempted to read with negative values."); 63 | } 64 | const metadata = (await reader.readObject(this.objectID)).metadata; 65 | if ((position + length) > metadata.size) { 66 | length = metadata.size - position; 67 | } 68 | if ((position < 0) || (length <= 0)) { 69 | return Buffer.alloc(0); 70 | } 71 | const firstChunkIndex = Math.floor(position / this.FS.chunkSize); 72 | const lastChunkIndex = Math.floor((position + length) / this.FS.chunkSize); 73 | const initialBufferSize = (lastChunkIndex - firstChunkIndex + 1) * this.FS.chunkSize; 74 | const data = Buffer.allocUnsafe(initialBufferSize); 75 | let copiedLength = 0; 76 | for (let i=firstChunkIndex; i<=lastChunkIndex; i++) { 77 | const chunkID = metadata.chunks[i]; 78 | if (chunkID == null) { 79 | break; 80 | } 81 | const chunk = await reader.readObject(chunkID); 82 | copiedLength += chunk.data.copy( 83 | data, 84 | (i - firstChunkIndex) * this.FS.chunkSize, 85 | 0, 86 | this.FS.chunkSize 87 | ); 88 | } 89 | data.fill(0, copiedLength); 90 | const requestedDataStart = position - (firstChunkIndex * this.FS.chunkSize); 91 | const requestedData = Buffer.from(data.slice(requestedDataStart, requestedDataStart + length)); 92 | return requestedData; 93 | } 94 | 95 | async read(position: number, length: number): Promise { 96 | const reader = await this.FS._getRead(); 97 | try { 98 | return await this._read(reader, position, length); 99 | } 100 | finally { 101 | reader.release(); 102 | } 103 | } 104 | 105 | async truncate(): Promise { 106 | const writer = await this.FS._getWrite(); 107 | try { 108 | const metadata = (await writer.readObject(this.objectID)).metadata; 109 | for (const chunk of metadata.chunks) { 110 | await writer.deleteObject(chunk); 111 | } 112 | await writer.writeObject(this.objectID, { 113 | metadata: { 114 | type: "file", 115 | chunks: [], 116 | size: 0 117 | }, 118 | data: null 119 | }); 120 | } 121 | finally { 122 | writer.release(); 123 | } 124 | } 125 | 126 | async append(data: Buffer): Promise { 127 | const writer = await this.FS._getWrite(); 128 | try { 129 | const metadata = (await writer.readObject(this.objectID)).metadata; 130 | let startIndex: number; 131 | let newData: Buffer; 132 | if (metadata.chunks.length > 0) { 133 | startIndex = metadata.chunks.length - 1; 134 | const lastChunk = metadata.chunks[startIndex]; 135 | const lastChunkData = (await writer.readObject(lastChunk)).data; 136 | newData = Buffer.concat([lastChunkData, data]); 137 | } 138 | else { 139 | startIndex = 0; 140 | newData = data; 141 | } 142 | await this._write(writer, startIndex, newData); 143 | } 144 | finally { 145 | writer.release(); 146 | } 147 | } 148 | 149 | getAbsolutePath() { 150 | const parentPath = this.parent.getAbsolutePath(); 151 | return `${parentPath}/${this.name}`; 152 | } 153 | 154 | async writeAll(newData: Buffer): Promise { 155 | const writer = await this.FS._getWrite(); 156 | try { 157 | await this._write(writer, 0, newData); 158 | } 159 | finally { 160 | writer.release(); 161 | } 162 | } 163 | 164 | private async _write(writer: AnyFSWriter, startIndex: number, newData: Buffer): Promise { 165 | const metadata = (await writer.readObject(this.objectID)).metadata; 166 | 167 | // Reuse existing data objects 168 | const chunks = metadata.chunks; 169 | 170 | // Create new data objects and modify existing ones 171 | let i: number, seek: number; 172 | for (i=startIndex, seek=0; seek(this.objectID, { 188 | metadata: { 189 | type: "file", 190 | chunks: chunks, 191 | size: (startIndex * this.FS.chunkSize) + newData.length 192 | }, 193 | data: null 194 | }); 195 | } 196 | 197 | async stat(): Promise { 198 | const reader = await this.FS._getRead(); 199 | try { 200 | const metadata = (await reader.readObject(this.objectID)).metadata; 201 | return { 202 | size: metadata.size 203 | }; 204 | } 205 | finally { 206 | reader.release(); 207 | } 208 | } 209 | } -------------------------------------------------------------------------------- /src/anyfs/fs-folder.ts: -------------------------------------------------------------------------------- 1 | import { AnyFS } from "./anyfs"; 2 | import { AnyFSFile } from "./fs-file"; 3 | import { AnyFSObject } from "./fs-object"; 4 | import { AnyFSFolderEntry, AnyFSFolderMetadata, ObjectID } from "./internal-types"; 5 | import { AnyFSReader } from "./reader"; 6 | import { AnyFSWriter } from "./writer"; 7 | 8 | export interface AnyFSFolder { 9 | parent: AnyFSFolder; 10 | } 11 | 12 | export class AnyFSFolder extends AnyFSObject { 13 | private async _listContents(reader: AnyFSReader): Promise { 14 | const objectData = await reader.readObject(this.objectID); 15 | return objectData.metadata.entries; 16 | } 17 | 18 | async listContents(): Promise { 19 | const reader = await this.FS._getRead(); 20 | try { 21 | return await this._listContents(reader); 22 | } 23 | finally { 24 | reader.release(); 25 | } 26 | } 27 | 28 | private static _splitComponents(path: string) { 29 | return path.split("/").filter((a) => a !== ''); 30 | } 31 | 32 | private async _atPath(components: string[]): Promise { 33 | let folder: AnyFSFolder | AnyFSFile = this; 34 | for (const component of components) { 35 | if (!(folder instanceof AnyFSFolder)) { 36 | return null; 37 | } 38 | else if (component === '.') { 39 | continue; 40 | } 41 | else if (component === '..') { 42 | folder = folder.parent; 43 | continue; 44 | } 45 | try { 46 | folder = await folder.get(component); 47 | } 48 | catch { 49 | return null; 50 | } 51 | } 52 | return folder; 53 | } 54 | 55 | static basename(path: string) { 56 | const components = this._splitComponents(path); 57 | return components[components.length-1]; 58 | } 59 | 60 | async atPath(path: string): Promise { 61 | if (path.startsWith("/") && (this.name !== "/")) { 62 | const root = await this.FS.root(); 63 | return await root.atPath(path); 64 | } 65 | const components = AnyFSFolder._splitComponents(path); 66 | return await this._atPath(components); 67 | } 68 | 69 | async parentForPath(path: string, shouldExist?: boolean): Promise { 70 | const components = AnyFSFolder._splitComponents(path); 71 | const parent = await this._atPath(components.slice(0, components.length-1)); 72 | if (!(parent instanceof AnyFSFolder)) { 73 | throw new Error("Not a directory."); 74 | } 75 | if (shouldExist != null) { 76 | const exists = await parent.exists(components[components.length-1]); 77 | if (shouldExist && !exists) { 78 | throw new Error("No such file or directory."); 79 | } 80 | else if (!shouldExist && exists) { 81 | throw new Error("File exists."); 82 | } 83 | } 84 | return parent; 85 | } 86 | 87 | static async create(FS: AnyFS, parent: AnyFSObject, name: string): Promise { 88 | const writer = await FS._getWrite(); 89 | try { 90 | const objectID = await writer.createObject(); 91 | await writer.writeObject(objectID, { 92 | metadata: { 93 | type: "folder", 94 | entries: [] 95 | }, 96 | data: null 97 | }); 98 | return new this(FS, parent, name, objectID); 99 | } 100 | finally { 101 | writer.release(); 102 | } 103 | } 104 | 105 | private async _getEntry(name: string, reader?: AnyFSReader): Promise { 106 | const contents = ( 107 | (reader == null) ? 108 | await this.listContents() : 109 | await this._listContents(reader) 110 | ); 111 | const object = contents.find((value) => value.name === name); 112 | return object; 113 | } 114 | 115 | async get(name: string, reader?: AnyFSReader): Promise { 116 | const entry = await this._getEntry(name, reader); 117 | if (entry == null) { 118 | return null; 119 | } 120 | if (entry.type === 'file') { 121 | return new AnyFSFile(this.FS, this, name, entry.objectID); 122 | } 123 | else if (entry.type === 'folder') { 124 | return new AnyFSFolder(this.FS, this, name, entry.objectID); 125 | } 126 | throw new Error("The requested file has an unknown type. It might be corrupted."); 127 | } 128 | 129 | getAbsolutePath() { 130 | const components = []; 131 | let folder: AnyFSFolder = this; 132 | while (folder.name !== "/") { 133 | components.push(folder.name); 134 | folder = folder.parent; 135 | } 136 | components.reverse(); 137 | return `/${components.join("/")}`; 138 | } 139 | 140 | async exists(name: string): Promise { 141 | return (await this._getEntry(name)) != null; 142 | } 143 | 144 | async link(name: string, type: "folder" | "file", objectID: ObjectID, force?: boolean): Promise { 145 | const writer = await this.FS._getWrite(); 146 | try { 147 | const oldItem = await this.get(name, writer); 148 | if (oldItem != null) { 149 | if ((force == null) || !force) { 150 | throw new Error("File exists."); 151 | } 152 | if (!oldItem.isFile()) { 153 | throw new Error("Is a directory."); 154 | } 155 | await this.deleteEntry(oldItem.name, writer, true); 156 | } 157 | else if (["..", "."].includes(name)) { 158 | throw new Error("Reserved name."); 159 | } 160 | else if (name.includes("/")) { 161 | throw new Error("Filenames cannot contain slashes (/)."); 162 | } 163 | const objectData = await writer.readObject(this.objectID); 164 | const entries = objectData.metadata.entries; 165 | entries.push({ type, name, objectID }); 166 | await writer.writeObject(this.objectID, objectData); 167 | } 168 | finally { 169 | writer.release(); 170 | } 171 | } 172 | 173 | async deleteEntry(name: string): Promise; 174 | async deleteEntry(name: string, force: boolean): Promise; 175 | async deleteEntry(name: string, writer: AnyFSWriter): Promise; 176 | async deleteEntry(name: string, writer: AnyFSWriter, force: boolean): Promise; 177 | 178 | async deleteEntry(name: string, arg2?: AnyFSWriter | boolean, arg3?: boolean): Promise { 179 | let writer: AnyFSWriter; 180 | let releaseWriter = true; 181 | let force: boolean = false; 182 | if (arg2 instanceof AnyFSWriter) { 183 | writer = arg2; 184 | arg2 = arg3; 185 | releaseWriter = false; 186 | } 187 | else { 188 | writer = await this.FS._getWrite(); 189 | } 190 | if (typeof arg2 === 'boolean') { 191 | force = arg2; 192 | } 193 | try { 194 | const objectData = await writer.readObject(this.objectID); 195 | const entryIndex = objectData.metadata.entries.findIndex((value) => value.name === name); 196 | if (entryIndex === -1) { 197 | throw new Error("No such file or directory."); 198 | } 199 | else if (!force && (objectData.metadata.entries[entryIndex].type === 'folder')) { 200 | //@ts-ignore 201 | const subFolder: AnyFSFolder = await this.get(name, writer); 202 | const contents = await subFolder._listContents(writer); 203 | if (contents.length !== 0) { 204 | throw new Error("Directory not empty."); 205 | } 206 | } 207 | const [removedEntry] = objectData.metadata.entries.splice(entryIndex, 1); 208 | const removedItem = await this.get(removedEntry.name, writer); 209 | await writer.writeObject(this.objectID, objectData); 210 | if (!force) { 211 | if (removedItem.isFile()) { 212 | await removedItem.truncate(); 213 | } 214 | await writer.deleteObject(removedItem.objectID); 215 | } 216 | } 217 | finally { 218 | if (releaseWriter) { 219 | writer.release(); 220 | } 221 | } 222 | } 223 | 224 | async createFile(name: string): Promise { 225 | const file = await AnyFSFile.create(this.FS, this, name); 226 | await this.link(name, "file", file.objectID); 227 | return file; 228 | } 229 | 230 | async createFolder(name: string): Promise { 231 | const folder = await AnyFSFolder.create(this.FS, this, name); 232 | await this.link(name, "folder", folder.objectID); 233 | return folder; 234 | } 235 | } -------------------------------------------------------------------------------- /src/anyfs/fs-object.ts: -------------------------------------------------------------------------------- 1 | import { AnyFS } from "./anyfs"; 2 | import { AnyFSFileChunk } from "./fs-chunk"; 3 | import { AnyFSFile } from "./fs-file"; 4 | import { AnyFSFolder } from "./fs-folder"; 5 | import { ObjectID } from "./internal-types"; 6 | 7 | export class AnyFSObject { 8 | FS: AnyFS; 9 | objectID: ObjectID; 10 | parent: AnyFSObject; 11 | name: string; 12 | 13 | constructor(FS: AnyFS, parent: AnyFSObject, name: string, objectID: ObjectID) { 14 | this.FS = FS; 15 | this.objectID = objectID; 16 | this.parent = parent ?? this; 17 | this.name = name; 18 | } 19 | 20 | isFile(): this is AnyFSFile { 21 | return this instanceof AnyFSFile; 22 | } 23 | 24 | isFolder(): this is AnyFSFolder { 25 | return this instanceof AnyFSFolder; 26 | } 27 | 28 | isFileChunk(): this is AnyFSFileChunk { 29 | return this instanceof AnyFSFileChunk; 30 | } 31 | } -------------------------------------------------------------------------------- /src/anyfs/ftp.ts: -------------------------------------------------------------------------------- 1 | import { AnyFS } from "./anyfs"; 2 | 3 | export async function getFTP(FS: AnyFS): Promise { 4 | let ftp; 5 | try { 6 | ftp = require('./_ftp'); 7 | } 8 | catch { 9 | throw new Error("The ftp-srv package is missing."); 10 | } 11 | return ftp.getFTP(FS); 12 | } -------------------------------------------------------------------------------- /src/anyfs/fuse.ts: -------------------------------------------------------------------------------- 1 | import { AnyFS } from "./anyfs"; 2 | 3 | export interface AnyFSMountOptions { 4 | verbose?: boolean, 5 | reportedBlocks?: number, 6 | reportedBlockSize?: number, 7 | allowWrite?: boolean 8 | } 9 | 10 | export async function fuseMount(FS: AnyFS, mountPoint: string, options?: AnyFSMountOptions, onDestroy?: () => void) { 11 | let fuse: any; 12 | try { 13 | fuse = require("./_fuse"); 14 | } 15 | catch { 16 | throw new Error("The fuse-bindings package is missing."); 17 | } 18 | return fuse.fuseMount(FS, mountPoint, options, onDestroy); 19 | } -------------------------------------------------------------------------------- /src/anyfs/index.ts: -------------------------------------------------------------------------------- 1 | export { AnyFS } from "./anyfs"; 2 | export { AnyFSFile } from "./fs-file"; 3 | export { AnyFSFolder } from "./fs-folder"; 4 | export { AnyFSProvider } from "./provider"; -------------------------------------------------------------------------------- /src/anyfs/internal-types.ts: -------------------------------------------------------------------------------- 1 | export type ObjectID = string | number; 2 | 3 | export interface AnyFSMetadata { 4 | type: string, 5 | xattr?: any 6 | } 7 | 8 | export interface AnyFSDataMetadata extends AnyFSMetadata { 9 | type: "data" 10 | } 11 | 12 | export interface AnyFSFileStat { 13 | size: number 14 | } 15 | 16 | export interface AnyFSFileMetadata extends AnyFSMetadata, AnyFSFileStat { 17 | type: "file", 18 | chunks: ObjectID[] 19 | } 20 | 21 | export interface AnyFSFolderEntry { 22 | name: string, 23 | type: "file" | "folder", 24 | objectID: ObjectID 25 | } 26 | 27 | export interface AnyFSFolderMetadata extends AnyFSMetadata { 28 | type: "folder", 29 | entries: AnyFSFolderEntry[] 30 | } 31 | 32 | export interface AnyFSObjectRaw { 33 | metadata: T, 34 | data: Buffer | null 35 | } -------------------------------------------------------------------------------- /src/anyfs/provider.ts: -------------------------------------------------------------------------------- 1 | import { ObjectID } from "./internal-types"; 2 | 3 | export interface AnyFSProvider { 4 | /** 5 | * Reads the entirety of the data for the given object ID 6 | * and returns it. If the object ID is invalid, it will 7 | * throw an exception. 8 | * @param objectID Object ID. 9 | * @returns Data for the object ID. 10 | */ 11 | readObject(objectID: ObjectID): Promise; 12 | 13 | /** 14 | * Replaces the current data for the given object ID with 15 | * the given data. `objectID` will always be an ID that was 16 | * previously returned from `createObject()`. May throw 17 | * an exception on failure. 18 | * @param objectID Object ID. 19 | * @param data New data. 20 | */ 21 | writeObject(objectID: ObjectID, data: Buffer): Promise; 22 | 23 | /** 24 | * Creates a new object. The data for this new object ID 25 | * may be anything. This data is irrelevant because it will 26 | * be immediately overwritten with a `writeObject()` call. 27 | * 28 | * **Warning:** When implementing this function, never return 29 | * the same object ID more than once. Doing so will cause 30 | * filesystem corruption. 31 | * @returns Object ID for the new object. 32 | */ 33 | createObject(): Promise; 34 | 35 | /** 36 | * Deletes an object if possible. This object ID will never 37 | * be used again if deletion is successful. Do not implement 38 | * this function if you aren't able to delete objects. 39 | * @returns `true` if the deletion was successful, `false` 40 | * otherwise. 41 | */ 42 | deleteObject?(objectID: ObjectID): Promise; 43 | } -------------------------------------------------------------------------------- /src/anyfs/reader.ts: -------------------------------------------------------------------------------- 1 | import { AnyFS } from "./anyfs"; 2 | import v8 from "v8"; 3 | import crypto from "crypto"; 4 | import { AnyFSObjectRaw, ObjectID } from "./internal-types"; 5 | 6 | export class AnyFSReader { 7 | FS: AnyFS 8 | 9 | get invalidated() { 10 | return this._invalidated; 11 | } 12 | 13 | private _invalidated: boolean 14 | private _ongoingReadCount = 0; 15 | 16 | constructor(FS: AnyFS) { 17 | this.FS = FS; 18 | } 19 | 20 | release() { 21 | if (this._ongoingReadCount !== 0) { 22 | throw new Error("Cannot release a reader while there are ongoing reads."); 23 | } 24 | this._invalidated = true; 25 | this.FS._release(this); 26 | } 27 | 28 | async readObject(objectID: ObjectID): Promise> { 29 | if (this.invalidated) { 30 | throw new Error("This reader was invalidated."); 31 | } 32 | if (objectID == null) { 33 | throw new Error("Object ID must not be null."); 34 | } 35 | this._ongoingReadCount++; 36 | try { 37 | let result: AnyFSObjectRaw; 38 | if (this.FS._cache.has(objectID)) { 39 | result = this.FS._cache.get(objectID); 40 | } 41 | else { 42 | const storedData = await this.FS._FSProvider.readObject(objectID); 43 | 44 | // Decrypt data (AES256) 45 | const iv = storedData.slice(0, 16); 46 | const encrypted = storedData.slice(16); 47 | const decipher = crypto.createDecipheriv('aes-256-cbc', this.FS._AESKey, iv); 48 | const incomplete = decipher.update(encrypted); 49 | const decrypted = Buffer.concat([ incomplete, decipher.final() ]); 50 | 51 | // Parse data (JSON and binary sections) 52 | const jsonEnd = decrypted.indexOf(0); 53 | const jsonData = (jsonEnd !== -1) ? decrypted.slice(0, jsonEnd) : decrypted; 54 | const metadata = JSON.parse(jsonData.toString('utf-8')); 55 | const data = (jsonEnd !== -1) ? decrypted.slice(jsonEnd + 1) : null; 56 | 57 | result = { metadata, data }; 58 | this.FS._cache.set(objectID, result); 59 | //console.log("[READ]", { objectID, ...result }); 60 | } 61 | 62 | result = v8.deserialize(v8.serialize(result)); 63 | return result; 64 | } 65 | finally { 66 | this._ongoingReadCount--; 67 | } 68 | } 69 | } -------------------------------------------------------------------------------- /src/anyfs/writer.ts: -------------------------------------------------------------------------------- 1 | import { AnyFSReader } from "./reader"; 2 | import { AnyFSObjectRaw, ObjectID } from "./internal-types"; 3 | import crypto from "crypto"; 4 | import v8 from "v8"; 5 | 6 | export class AnyFSWriter extends AnyFSReader { 7 | private _ongoingWriteCount = 0; 8 | 9 | release() { 10 | if (this._ongoingWriteCount !== 0) { 11 | throw new Error("Cannot release a writer while there are ongoing writes."); 12 | } 13 | super.release(); 14 | } 15 | 16 | private async _performWrite(run: () => Promise): Promise { 17 | if (this.invalidated) { 18 | throw new Error("This writer was invalidated."); 19 | } 20 | this._ongoingWriteCount++; 21 | try { 22 | return await run(); 23 | } 24 | finally { 25 | this._ongoingWriteCount--; 26 | } 27 | } 28 | 29 | async writeObject(objectID: ObjectID, object: AnyFSObjectRaw) { 30 | if (objectID == null) { 31 | throw new Error("Object ID must not be null."); 32 | } 33 | await this._performWrite(async() => { 34 | //console.log("[WRITE]", { objectID, ...object }); 35 | 36 | // Generate unencrypted data 37 | const jsonData = Buffer.from(JSON.stringify(object.metadata), 'utf-8'); 38 | const hasData = (object.data != null); 39 | const unencryptedSize = jsonData.length + (hasData ? (object.data.length + 1) : 0); 40 | const unencrypted = Buffer.allocUnsafe(unencryptedSize); 41 | jsonData.copy(unencrypted); 42 | if (hasData) { 43 | unencrypted[jsonData.length] = 0; 44 | object.data.copy(unencrypted, jsonData.length + 1); 45 | } 46 | 47 | // Encrypt data (AES256) 48 | const iv = crypto.randomBytes(16); 49 | const cipher = crypto.createCipheriv('aes-256-cbc', this.FS._AESKey, iv); 50 | const incomplete = cipher.update(unencrypted); 51 | const encrypted = Buffer.concat([iv, incomplete, cipher.final()]); 52 | 53 | await this.FS._FSProvider.writeObject(objectID, encrypted); 54 | 55 | this.FS._cache.set(objectID, v8.deserialize(v8.serialize(object))); 56 | }); 57 | } 58 | 59 | async createObject(): Promise { 60 | return await this._performWrite(async() => { 61 | return await this.FS._FSProvider.createObject(); 62 | }); 63 | } 64 | 65 | async deleteObject(objectID: ObjectID): Promise { 66 | if (objectID == null) { 67 | throw new Error("Object ID must not be null."); 68 | } 69 | return await this._performWrite(async() => { 70 | this.FS._cache.delete(objectID); 71 | if (this.FS._FSProvider.deleteObject != null) { 72 | return await this.FS._FSProvider.deleteObject(objectID); 73 | } 74 | else { 75 | return false; 76 | } 77 | }); 78 | } 79 | } -------------------------------------------------------------------------------- /src/examples/local-fs.ts: -------------------------------------------------------------------------------- 1 | import { AnyFS, AnyFSProvider } from "../anyfs"; 2 | import { existsSync, readFileSync, writeFileSync, unlinkSync } from "fs"; 3 | import crypto from "crypto"; 4 | import path from "path"; 5 | 6 | class LocalFSProvider implements AnyFSProvider { 7 | private storagePath: string; 8 | 9 | path(objectID: number) { 10 | return path.join(this.storagePath, objectID.toString()); 11 | } 12 | async readObject(objectID: number) { 13 | return Buffer.from(readFileSync(this.path(objectID), 'utf-8'), 'base64'); 14 | } 15 | async writeObject(objectID: number, data: Buffer) { 16 | return writeFileSync(this.path(objectID), data.toString('base64')); 17 | } 18 | async createObject() { 19 | let objectID: number; 20 | do { 21 | objectID = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER); 22 | } 23 | while (existsSync(this.path(objectID))); 24 | await this.writeObject(objectID, Buffer.alloc(0)); 25 | return objectID; 26 | } 27 | async deleteObject(objectID: any): Promise { 28 | try { 29 | const path = this.path(objectID); 30 | if (existsSync(path)) { 31 | unlinkSync(path); 32 | return true; 33 | } 34 | return false; 35 | } 36 | catch { 37 | return false; 38 | } 39 | } 40 | constructor(storagePath: string) { 41 | this.storagePath = storagePath; 42 | } 43 | } 44 | 45 | export interface LocalFSAuth { 46 | key: Buffer, 47 | root: number 48 | }; 49 | 50 | export class LocalFS extends AnyFS { 51 | static async createKey(storagePath: string) { 52 | const provider = new LocalFSProvider(storagePath); 53 | const root = await provider.createObject(); 54 | const key = crypto.randomBytes(32); 55 | return { root, key }; 56 | } 57 | 58 | static authenticate(storagePath: string, { root, key }: LocalFSAuth) { 59 | const provider = new LocalFSProvider(storagePath); 60 | return new LocalFS(provider, key, root); 61 | } 62 | 63 | private constructor(FSProvider: LocalFSProvider, AESKey: Buffer, rootID: number) { 64 | super(FSProvider, AESKey, 16 * 1024, rootID); 65 | } 66 | } -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import path from "path"; 2 | import { LocalFS, LocalFSAuth } from "./examples/local-fs"; 3 | import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; 4 | import { FtpSrv } from "ftp-srv"; 5 | 6 | export * from "./anyfs"; 7 | 8 | async function main() { 9 | if (process.argv.length < 4) { 10 | console.log("Usage:", path.basename(process.argv[1]), " "); 11 | process.exit(1); 12 | } 13 | const storagePath = process.argv[2]; 14 | const mountPath = process.argv[3]; 15 | if (!existsSync(storagePath)) { 16 | mkdirSync(storagePath); 17 | } 18 | const authDataPath = path.join(storagePath, "auth.json"); 19 | let authData: LocalFSAuth; 20 | if (existsSync(authDataPath)) { 21 | const fileData = JSON.parse(readFileSync(authDataPath, 'utf-8')); 22 | authData = { 23 | key: Buffer.from(fileData.key, 'base64'), 24 | root: fileData.root 25 | }; 26 | } 27 | else { 28 | authData = await LocalFS.createKey(storagePath); 29 | const outputData = { 30 | key: authData.key.toString('base64'), 31 | root: authData.root 32 | }; 33 | writeFileSync(authDataPath, JSON.stringify(outputData)); 34 | } 35 | const fs = LocalFS.authenticate(storagePath, authData); 36 | await fs.fuseMount(mountPath, { verbose: true }, () => { 37 | process.exit(0); 38 | }); 39 | const ftpServer = new FtpSrv({ 40 | anonymous: true, 41 | url: "http://127.0.0.1:2121", 42 | pasv_url: "http://127.0.0.1:2121" 43 | }); 44 | ftpServer.on("login", async(loginData, resolve, reject) => { 45 | try { 46 | resolve({ 47 | fs: await fs.getFTP(), 48 | cwd: "/" 49 | }); 50 | } 51 | catch (err) { 52 | reject(err); 53 | } 54 | }); 55 | ftpServer.listen(); 56 | } 57 | 58 | if (require.main === module) { 59 | main(); 60 | } -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "target": "es2015", 5 | "declaration": true, 6 | "outDir": "./dist", 7 | "sourceMap": true, 8 | "esModuleInterop": true 9 | }, 10 | "include": [ 11 | "src/**/*" 12 | ] 13 | } --------------------------------------------------------------------------------