├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── README.md ├── bf1_chimera_rdma_res.txt ├── bf1_res.txt ├── chimera_rdma_res.txt ├── chimera_res.txt ├── rdma-fuse.org ├── remote_build.sh └── src ├── file.rs ├── lib.rs ├── local.rs ├── main.rs └── remote.rs /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "aho-corasick" 7 | version = "0.7.15" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" 10 | dependencies = [ 11 | "memchr", 12 | ] 13 | 14 | [[package]] 15 | name = "ansi_term" 16 | version = "0.11.0" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" 19 | dependencies = [ 20 | "winapi", 21 | ] 22 | 23 | [[package]] 24 | name = "atty" 25 | version = "0.2.14" 26 | source = "registry+https://github.com/rust-lang/crates.io-index" 27 | checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" 28 | dependencies = [ 29 | "hermit-abi", 30 | "libc", 31 | "winapi", 32 | ] 33 | 34 | [[package]] 35 | name = "autocfg" 36 | version = "1.1.0" 37 | source = "registry+https://github.com/rust-lang/crates.io-index" 38 | checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" 39 | 40 | [[package]] 41 | name = "bincode" 42 | version = "1.3.2" 43 | source = "registry+https://github.com/rust-lang/crates.io-index" 44 | checksum = "d175dfa69e619905c4c3cdb7c3c203fa3bdd5d51184e3afdb2742c0280493772" 45 | dependencies = [ 46 | "byteorder", 47 | "serde", 48 | ] 49 | 50 | [[package]] 51 | name = "bindgen" 52 | version = "0.58.1" 53 | source = "registry+https://github.com/rust-lang/crates.io-index" 54 | checksum = "0f8523b410d7187a43085e7e064416ea32ded16bd0a4e6fc025e21616d01258f" 55 | dependencies = [ 56 | "bitflags", 57 | "cexpr", 58 | "clang-sys", 59 | "clap", 60 | "env_logger", 61 | "lazy_static", 62 | "lazycell", 63 | "log", 64 | "peeking_take_while", 65 | "proc-macro2", 66 | "quote", 67 | "regex", 68 | "rustc-hash", 69 | "shlex", 70 | "which", 71 | ] 72 | 73 | [[package]] 74 | name = "bitflags" 75 | version = "1.2.1" 76 | source = "registry+https://github.com/rust-lang/crates.io-index" 77 | checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" 78 | 79 | [[package]] 80 | name = "byteorder" 81 | version = "1.3.4" 82 | source = "registry+https://github.com/rust-lang/crates.io-index" 83 | checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" 84 | 85 | [[package]] 86 | name = "cc" 87 | version = "1.0.67" 88 | source = "registry+https://github.com/rust-lang/crates.io-index" 89 | checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" 90 | 91 | [[package]] 92 | name = "cexpr" 93 | version = "0.4.0" 94 | source = "registry+https://github.com/rust-lang/crates.io-index" 95 | checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27" 96 | dependencies = [ 97 | "nom", 98 | ] 99 | 100 | [[package]] 101 | name = "cfg-if" 102 | version = "1.0.0" 103 | source = "registry+https://github.com/rust-lang/crates.io-index" 104 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 105 | 106 | [[package]] 107 | name = "clang-sys" 108 | version = "1.2.0" 109 | source = "registry+https://github.com/rust-lang/crates.io-index" 110 | checksum = "853eda514c284c2287f4bf20ae614f8781f40a81d32ecda6e91449304dfe077c" 111 | dependencies = [ 112 | "glob", 113 | "libc", 114 | "libloading", 115 | ] 116 | 117 | [[package]] 118 | name = "clap" 119 | version = "2.33.3" 120 | source = "registry+https://github.com/rust-lang/crates.io-index" 121 | checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" 122 | dependencies = [ 123 | "ansi_term", 124 | "atty", 125 | "bitflags", 126 | "strsim", 127 | "textwrap", 128 | "unicode-width", 129 | "vec_map", 130 | ] 131 | 132 | [[package]] 133 | name = "env_logger" 134 | version = "0.8.3" 135 | source = "registry+https://github.com/rust-lang/crates.io-index" 136 | checksum = "17392a012ea30ef05a610aa97dfb49496e71c9f676b27879922ea5bdf60d9d3f" 137 | dependencies = [ 138 | "atty", 139 | "humantime", 140 | "log", 141 | "regex", 142 | "termcolor", 143 | ] 144 | 145 | [[package]] 146 | name = "fuser" 147 | version = "0.7.0" 148 | source = "git+https://github.com/iwahbe/fuser.git?branch=from-metadata-for-fileattr#643245e0234a32ae818b91f274e4bd5e47c90e93" 149 | dependencies = [ 150 | "libc", 151 | "log", 152 | "page_size", 153 | "pkg-config", 154 | "serde", 155 | "users", 156 | "zerocopy", 157 | ] 158 | 159 | [[package]] 160 | name = "glob" 161 | version = "0.3.0" 162 | source = "registry+https://github.com/rust-lang/crates.io-index" 163 | checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" 164 | 165 | [[package]] 166 | name = "hermit-abi" 167 | version = "0.1.18" 168 | source = "registry+https://github.com/rust-lang/crates.io-index" 169 | checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" 170 | dependencies = [ 171 | "libc", 172 | ] 173 | 174 | [[package]] 175 | name = "humantime" 176 | version = "2.1.0" 177 | source = "registry+https://github.com/rust-lang/crates.io-index" 178 | checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" 179 | 180 | [[package]] 181 | name = "ibverbs" 182 | version = "0.5.0" 183 | source = "registry+https://github.com/rust-lang/crates.io-index" 184 | checksum = "266ed13b230032af1ad71b1ba91e3d7a8d341c8668a85020b0dfc3809df8ae7d" 185 | dependencies = [ 186 | "bindgen", 187 | "serde", 188 | ] 189 | 190 | [[package]] 191 | name = "lazy_static" 192 | version = "1.4.0" 193 | source = "registry+https://github.com/rust-lang/crates.io-index" 194 | checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" 195 | 196 | [[package]] 197 | name = "lazycell" 198 | version = "1.3.0" 199 | source = "registry+https://github.com/rust-lang/crates.io-index" 200 | checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" 201 | 202 | [[package]] 203 | name = "libc" 204 | version = "0.2.126" 205 | source = "registry+https://github.com/rust-lang/crates.io-index" 206 | checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" 207 | 208 | [[package]] 209 | name = "libloading" 210 | version = "0.7.0" 211 | source = "registry+https://github.com/rust-lang/crates.io-index" 212 | checksum = "6f84d96438c15fcd6c3f244c8fce01d1e2b9c6b5623e9c711dc9286d8fc92d6a" 213 | dependencies = [ 214 | "cfg-if", 215 | "winapi", 216 | ] 217 | 218 | [[package]] 219 | name = "log" 220 | version = "0.4.14" 221 | source = "registry+https://github.com/rust-lang/crates.io-index" 222 | checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" 223 | dependencies = [ 224 | "cfg-if", 225 | ] 226 | 227 | [[package]] 228 | name = "memchr" 229 | version = "2.3.4" 230 | source = "registry+https://github.com/rust-lang/crates.io-index" 231 | checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" 232 | 233 | [[package]] 234 | name = "memmap2" 235 | version = "0.2.1" 236 | source = "registry+https://github.com/rust-lang/crates.io-index" 237 | checksum = "04e3e85b970d650e2ae6d70592474087051c11c54da7f7b4949725c5735fbcc6" 238 | dependencies = [ 239 | "libc", 240 | ] 241 | 242 | [[package]] 243 | name = "memoffset" 244 | version = "0.6.5" 245 | source = "registry+https://github.com/rust-lang/crates.io-index" 246 | checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" 247 | dependencies = [ 248 | "autocfg", 249 | ] 250 | 251 | [[package]] 252 | name = "nix" 253 | version = "0.20.2" 254 | source = "registry+https://github.com/rust-lang/crates.io-index" 255 | checksum = "f5e06129fb611568ef4e868c14b326274959aa70ff7776e9d55323531c374945" 256 | dependencies = [ 257 | "bitflags", 258 | "cc", 259 | "cfg-if", 260 | "libc", 261 | "memoffset", 262 | ] 263 | 264 | [[package]] 265 | name = "nom" 266 | version = "5.1.2" 267 | source = "registry+https://github.com/rust-lang/crates.io-index" 268 | checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af" 269 | dependencies = [ 270 | "memchr", 271 | "version_check", 272 | ] 273 | 274 | [[package]] 275 | name = "page_size" 276 | version = "0.4.2" 277 | source = "registry+https://github.com/rust-lang/crates.io-index" 278 | checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd" 279 | dependencies = [ 280 | "libc", 281 | "winapi", 282 | ] 283 | 284 | [[package]] 285 | name = "peeking_take_while" 286 | version = "0.1.2" 287 | source = "registry+https://github.com/rust-lang/crates.io-index" 288 | checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" 289 | 290 | [[package]] 291 | name = "pkg-config" 292 | version = "0.3.19" 293 | source = "registry+https://github.com/rust-lang/crates.io-index" 294 | checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" 295 | 296 | [[package]] 297 | name = "proc-macro2" 298 | version = "1.0.25" 299 | source = "registry+https://github.com/rust-lang/crates.io-index" 300 | checksum = "37015eacd32e5cb0cb453d8888d5aad735e257cfc5b7aca77ad121a70ab326f9" 301 | dependencies = [ 302 | "unicode-xid", 303 | ] 304 | 305 | [[package]] 306 | name = "quote" 307 | version = "1.0.9" 308 | source = "registry+https://github.com/rust-lang/crates.io-index" 309 | checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" 310 | dependencies = [ 311 | "proc-macro2", 312 | ] 313 | 314 | [[package]] 315 | name = "rdma-fuse" 316 | version = "0.1.0" 317 | dependencies = [ 318 | "bincode", 319 | "clap", 320 | "env_logger", 321 | "fuser", 322 | "ibverbs", 323 | "libc", 324 | "log", 325 | "memmap2", 326 | "nix", 327 | ] 328 | 329 | [[package]] 330 | name = "regex" 331 | version = "1.4.5" 332 | source = "registry+https://github.com/rust-lang/crates.io-index" 333 | checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" 334 | dependencies = [ 335 | "aho-corasick", 336 | "memchr", 337 | "regex-syntax", 338 | ] 339 | 340 | [[package]] 341 | name = "regex-syntax" 342 | version = "0.6.23" 343 | source = "registry+https://github.com/rust-lang/crates.io-index" 344 | checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" 345 | 346 | [[package]] 347 | name = "rustc-hash" 348 | version = "1.1.0" 349 | source = "registry+https://github.com/rust-lang/crates.io-index" 350 | checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" 351 | 352 | [[package]] 353 | name = "serde" 354 | version = "1.0.125" 355 | source = "registry+https://github.com/rust-lang/crates.io-index" 356 | checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171" 357 | dependencies = [ 358 | "serde_derive", 359 | ] 360 | 361 | [[package]] 362 | name = "serde_derive" 363 | version = "1.0.125" 364 | source = "registry+https://github.com/rust-lang/crates.io-index" 365 | checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" 366 | dependencies = [ 367 | "proc-macro2", 368 | "quote", 369 | "syn", 370 | ] 371 | 372 | [[package]] 373 | name = "shlex" 374 | version = "1.0.0" 375 | source = "registry+https://github.com/rust-lang/crates.io-index" 376 | checksum = "42a568c8f2cd051a4d283bd6eb0343ac214c1b0f1ac19f93e1175b2dee38c73d" 377 | 378 | [[package]] 379 | name = "strsim" 380 | version = "0.8.0" 381 | source = "registry+https://github.com/rust-lang/crates.io-index" 382 | checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" 383 | 384 | [[package]] 385 | name = "syn" 386 | version = "1.0.67" 387 | source = "registry+https://github.com/rust-lang/crates.io-index" 388 | checksum = "6498a9efc342871f91cc2d0d694c674368b4ceb40f62b65a7a08c3792935e702" 389 | dependencies = [ 390 | "proc-macro2", 391 | "quote", 392 | "unicode-xid", 393 | ] 394 | 395 | [[package]] 396 | name = "synstructure" 397 | version = "0.12.4" 398 | source = "registry+https://github.com/rust-lang/crates.io-index" 399 | checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" 400 | dependencies = [ 401 | "proc-macro2", 402 | "quote", 403 | "syn", 404 | "unicode-xid", 405 | ] 406 | 407 | [[package]] 408 | name = "termcolor" 409 | version = "1.1.2" 410 | source = "registry+https://github.com/rust-lang/crates.io-index" 411 | checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" 412 | dependencies = [ 413 | "winapi-util", 414 | ] 415 | 416 | [[package]] 417 | name = "textwrap" 418 | version = "0.11.0" 419 | source = "registry+https://github.com/rust-lang/crates.io-index" 420 | checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" 421 | dependencies = [ 422 | "unicode-width", 423 | ] 424 | 425 | [[package]] 426 | name = "unicode-width" 427 | version = "0.1.8" 428 | source = "registry+https://github.com/rust-lang/crates.io-index" 429 | checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" 430 | 431 | [[package]] 432 | name = "unicode-xid" 433 | version = "0.2.1" 434 | source = "registry+https://github.com/rust-lang/crates.io-index" 435 | checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" 436 | 437 | [[package]] 438 | name = "users" 439 | version = "0.11.0" 440 | source = "registry+https://github.com/rust-lang/crates.io-index" 441 | checksum = "24cc0f6d6f267b73e5a2cadf007ba8f9bc39c6a6f9666f8cf25ea809a153b032" 442 | dependencies = [ 443 | "libc", 444 | "log", 445 | ] 446 | 447 | [[package]] 448 | name = "vec_map" 449 | version = "0.8.2" 450 | source = "registry+https://github.com/rust-lang/crates.io-index" 451 | checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" 452 | 453 | [[package]] 454 | name = "version_check" 455 | version = "0.9.3" 456 | source = "registry+https://github.com/rust-lang/crates.io-index" 457 | checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" 458 | 459 | [[package]] 460 | name = "which" 461 | version = "3.1.1" 462 | source = "registry+https://github.com/rust-lang/crates.io-index" 463 | checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724" 464 | dependencies = [ 465 | "libc", 466 | ] 467 | 468 | [[package]] 469 | name = "winapi" 470 | version = "0.3.9" 471 | source = "registry+https://github.com/rust-lang/crates.io-index" 472 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 473 | dependencies = [ 474 | "winapi-i686-pc-windows-gnu", 475 | "winapi-x86_64-pc-windows-gnu", 476 | ] 477 | 478 | [[package]] 479 | name = "winapi-i686-pc-windows-gnu" 480 | version = "0.4.0" 481 | source = "registry+https://github.com/rust-lang/crates.io-index" 482 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 483 | 484 | [[package]] 485 | name = "winapi-util" 486 | version = "0.1.5" 487 | source = "registry+https://github.com/rust-lang/crates.io-index" 488 | checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" 489 | dependencies = [ 490 | "winapi", 491 | ] 492 | 493 | [[package]] 494 | name = "winapi-x86_64-pc-windows-gnu" 495 | version = "0.4.0" 496 | source = "registry+https://github.com/rust-lang/crates.io-index" 497 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 498 | 499 | [[package]] 500 | name = "zerocopy" 501 | version = "0.3.0" 502 | source = "registry+https://github.com/rust-lang/crates.io-index" 503 | checksum = "6580539ad917b7c026220c4b3f2c08d52ce54d6ce0dc491e66002e35388fab46" 504 | dependencies = [ 505 | "byteorder", 506 | "zerocopy-derive", 507 | ] 508 | 509 | [[package]] 510 | name = "zerocopy-derive" 511 | version = "0.2.0" 512 | source = "registry+https://github.com/rust-lang/crates.io-index" 513 | checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" 514 | dependencies = [ 515 | "proc-macro2", 516 | "syn", 517 | "synstructure", 518 | ] 519 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rdma-fuse" 3 | version = "0.1.0" 4 | authors = ["Ian Wahbe "] 5 | edition = "2018" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | # custom for some extra derives 11 | fuser = { version = "0.7", git = "https://github.com/iwahbe/fuser.git", branch = "from-metadata-for-fileattr", features = ["serializable", "abi-7-19"]} 12 | libc = "0.2" 13 | nix = "0.20.2" 14 | clap = "2.33" 15 | log = "0.4" 16 | env_logger = "0.8" 17 | memmap2 = "0.2" 18 | ibverbs = "0.5" 19 | bincode = "1.3" 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## FUSE Filesystem Over RDMA 2 | 3 | Completed for Reed's Topics in Systems class. Professor is [Eitan Frachtenberg](https://www.reed.edu/faculty-profiles/profiles/frachtenberg-eitan.html). 4 | 5 | Ian Wahbe 6 | 7 | ### Warning 8 | 9 | This is not a stable system and should not be used in production. All software 10 | is provided as is and without warranty. 11 | 12 | ### Building 13 | 14 | The program should compile fine with `cargo build`. It does need rust's nightly 15 | compiler as well as FUSE and Linux header files. 16 | 17 | ### Background 18 | 19 | --- 20 | 21 | Hardware accelerators have evolved from small network and cryptography chips 22 | to full blown separate computers. As these accelerator computers are still 23 | subservient to the host computer, it's important there is easy and fast 24 | communication between host and accelerator. This can be accomplished via message 25 | passing over an internal network, where each process on the accelerator talks 26 | directly to a corresponding process on the host. This leads to extremely tight 27 | coupling, where each host and accelerator need custom message passing software 28 | written to interface between them. Because that accelerators are usually the 29 | slower general computer, most data is kept on the host, or an external drive 30 | connected to the host. To avoid this, I provide a base system to intermediate 31 | between host and accelerator. Using the Linux philosophy of "everything is a 32 | file", I have written a file system mirror that allows an accelerator to read 33 | and write to the host file system over the network. 34 | 35 | I build my file system mirror on two new and distinct pieces of technology. 36 | 37 | #### FUSE (Filesystem in Userspace) 38 | 39 | FUSE provides a generic kernel extension, and an interaction library written in 40 | C (I used a Rust wrapper). Without writing my own kernel extension, this is the 41 | only way to allow my filesystem to be completely transparent to the user once 42 | mounted. 43 | 44 | #### RDMA (remote direct memory access) 45 | 46 | RDMA provides the low latency communication of data, especially buffer reads and 47 | writes. I use RDMA's memory region abstraction, which allows the host and 48 | accelerator to share (with minimal overhead) a region of memory over a local 49 | network. This has the advantage of not requiring a system call to exchange 50 | information. Instead, RDMA uses a queue system. Each call to send the memory 51 | region is deposited in a device specific queue, and eventually sent by the OS. 52 | Likewise, we receive message by looking at a queue maintained by the OS (holding 53 | incoming memory writes). To read or write data from the memory region, you can 54 | just dereference the pointer to it, and treat the memory region as an allocated 55 | array. I found some similarity between RDMA and 56 | working with memory mapped files. 57 | 58 | Because RDMA does not use the standard network abstractions, it requires special 59 | hardware and software to use. I run RDMA over a 25 GB cable between our Nvidia 60 | Bluefield network card and the host machine (chimera), with both running linux. 61 | It requires configuration of the devices at both ends, as well as a kernel 62 | extension to handle the queues. 63 | 64 | Finally, RDMA has a major drawback. Unlike standard network connections, which 65 | use `port`s to virtualize their address space, RDMA hooks directly into its 66 | supporting hardware. This means we are restricted to a single connection at a 67 | time per RDMA device. While some cards and chips can have multiple devices, it 68 | stills supports a distinctly limited number of connections. 69 | 70 | 71 | 72 | 73 | 74 | ### Proposed Solution 75 | 76 | --- 77 | 78 | I built a transparent filesystem that operates over RDMA. When I say 79 | transparent, I meant that to the host, it looks like a server is accessing all 80 | files the client is accessing, and most difficult operations are performed on 81 | the host's native file system. For the client, I use FUSE to prevent the user 82 | from experiencing any difference in usage. 83 | 84 | #### Design considerations and choices for your implementation 85 | 86 | The technology choices I made stemmed from two main decisions. I needed to use 87 | FUSE and RDMA, and I wanted to write my project in Rust. There exists only 1 88 | maintained interface to FUSE for Rust, called 89 | [fuser](https://github.com/cberner/fuser) (FUSE Rust), which I used. You provide 90 | it a `struct` with an initialization method, teardown method, and methods that 91 | encompass the operations your file system supports. The signature of all methods 92 | look similar. The function takes as arguments its input, and reply is a `struct` 93 | with methods to take either output or errors. An example method, `open`, looks 94 | like this: 95 | 96 | ```rust 97 | fn open(&mut self, req: &Request<'_>, ino: Ino, flags: i32, reply: ReplyOpen) { 98 | // either 99 | reply.opened(file_handle, open_flags); 100 | // or 101 | reply.error(ERROR_NUMBER); 102 | } 103 | ``` 104 | 105 | --- 106 | 107 | I needed to pair this with an RDMA library. I chose 108 | [ibverbs](https://github.com/jonhoo/rust-ibverbs). It provided a solid set of 109 | abstractions for RDMA, but usage details are too complicated to go into here. 110 | Most of the finicky RDMA work was done in `rdma-fuse/src/lib.rs`, and is 111 | available for closer inspection. 112 | 113 | After library and language were chosen, most design decisions were made. I then 114 | proceeded to fill in the methods as required. 115 | 116 | #### Overview of algorithms / code / implementation 117 | 118 | Given FUSE, the main design followed. I built a server that sat on the host 119 | (chimera), and a client that runs on the accelerator. They begin by generating a 120 | TCP connection, which is used to initiate an RDMA handshake. After that, all 121 | communication occurs by reading and writing to the fixed size buffer. We store a 122 | fixed size `enum` (tagged union), and read the tag to determine which operation 123 | is requested. Each payload contains the information for both a request and a 124 | reply, allowing the client to validate its reply and allowing the server to 125 | perform minimal writes to shared memory. Each message corresponds to a file 126 | system syscall, and thus a FUSE request (except `Exit` and `Null`). When 127 | received by the server, it responds with an associated method on the `LocalData` 128 | `struct`. The client gets the response, unpacks it, and replies with the data 129 | given more or less blindly. 130 | 131 | The communication `enum` as declared goes as follows: 132 | 133 | ```rust 134 | pub enum Message { 135 | Exit, 136 | 137 | Startup { 138 | server: bool, 139 | }, 140 | 141 | Null, 142 | 143 | Lookup { 144 | errno: Option, 145 | parent: Ino, 146 | name: [u8; MAX_FILENAME_LENGTH], 147 | attr: Option, //to allow default 148 | generation: u64, 149 | }, 150 | 151 | GetAttr { 152 | errno: Option, 153 | ino: Ino, 154 | attr: Option, 155 | }, 156 | 157 | OpenDir { 158 | errno: Option, 159 | ino: Ino, 160 | flags: i32, 161 | fh: Fh, 162 | open_flags: u32, 163 | }, 164 | 165 | ReadDir { 166 | ino: Ino, 167 | fh: Fh, 168 | 169 | finished: bool, 170 | errno: Option, 171 | buf_ino: Ino, 172 | offset: i64, 173 | kind: FileType, 174 | name: [u8; MAX_FILENAME_LENGTH], 175 | }, 176 | 177 | ReleaseDir { 178 | fh: Fh, 179 | errno: Option, 180 | }, 181 | 182 | Open { 183 | errno: Option, 184 | ino: Ino, 185 | flags: i32, 186 | fh: Fh, 187 | open_flags: u32, 188 | }, 189 | 190 | Release { 191 | errno: Option, 192 | ino: Ino, 193 | fh: Fh, 194 | }, 195 | 196 | Read { 197 | errno: Option, 198 | fh: Fh, 199 | offset: i64, 200 | size: u32, 201 | buf: [u8; READ_WRITE_BUFFER_SIZE], 202 | }, 203 | 204 | Write { 205 | errno: Option, 206 | fh: Fh, 207 | offset: i64, 208 | data: [u8; READ_WRITE_BUFFER_SIZE], 209 | written: u32, 210 | }, 211 | 212 | Flush { 213 | errno: Option, 214 | fh: Fh, 215 | }, 216 | 217 | LSeek { 218 | errno: Option, 219 | fh: Fh, 220 | offset: i64, 221 | whence: i32, 222 | }, 223 | 224 | Create { 225 | errno: Option, 226 | parent: Ino, 227 | name: [u8; MAX_FILENAME_LENGTH], 228 | flags: i32, 229 | 230 | attr: Option, 231 | generation: u64, 232 | fh: Fh, 233 | open_flags: u32, 234 | }, 235 | 236 | Mkdir { 237 | errno: Option, 238 | parent: Ino, 239 | name: [u8; MAX_FILENAME_LENGTH], 240 | mode: u32, 241 | umask: u32, 242 | 243 | attr: Option, 244 | generation: u64, 245 | }, 246 | 247 | Unlink { 248 | errno: Option, 249 | parent: Ino, 250 | name: [u8; MAX_FILENAME_LENGTH], 251 | }, 252 | 253 | Rmdir { 254 | errno: Option, 255 | parent: Ino, 256 | name: [u8; MAX_FILENAME_LENGTH], 257 | }, 258 | 259 | Rename { 260 | errno: Option, 261 | parent: Ino, 262 | name: [u8; MAX_FILENAME_LENGTH], 263 | newparent: Ino, 264 | newname: [u8; MAX_FILENAME_LENGTH], 265 | }, 266 | } 267 | ``` 268 | 269 | We can describe the system as follows: 270 | 271 | 272 | 273 | ```json 274 | 275 | +----------+ +----------+ 276 | | RDMA(H) | <------------> | RDMA(C) | 277 | +----------+ +----------+ 278 | | | 279 | | | 280 | +-----------+ +------------+ 281 | | HOST FS | | CLIENT FS | 282 | +-----------+ +------------+ 283 | ``` 284 | 285 | Only the host side stores state. The client only remembers what type of call it 286 | is currently executing, and that is maintained by program stack. 287 | 288 | #### Similar Systems 289 | 290 | This sounds a lot like NFS, but is importantly different. Because the goal is to 291 | facilitate communication for computers which are directly adjacent (or attached) 292 | to each other, instead of a generic and fallable 1-N pairing, I can operate much 293 | more efficiently then NFS. Because this is a modern system, I take advantage of 294 | RDMA as my message passing interface. This both reduces latency, and the 295 | overhead. It does have major downsides compared to NFS as well. 296 | 297 | ##### Running 298 | 299 | To actually use the system, we do the following: 300 | 301 | ```sh 302 | # on the host 303 | ~/x86/bin/rdma-fuse remote --host target/ --ip 192.168.100.1:8393 304 | # on the client 305 | ~/arm/bin/rdma-fuse remote --client mnt/ --ip 192.168.100.1:8393 306 | ``` 307 | 308 | This mounts `target` on the host system to the empty folder `mnt` on the client 309 | system. It uses ip address `192.168.100.1` to communicate with port `8393`. 310 | Detailed usage information can be found by typing `rdma-fuse help`. 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 | ### Evaluation 319 | 320 | --- 321 | 322 | When evaluating the system, we compare it with two other cases. Eitan has 323 | mirrored the home folder of chimera on `bf1`, so we test that system, the native 324 | file system. These provide comparison for the RDMA system. I perform the tests 325 | using the `fs-bench` test provided by 326 | [ltp](https://github.com/linux-test-project/ltp.git). I have removed the 327 | ownership components of the test, as my RDMA/FUSE implementation does not 328 | support ownership. The linux test project contains instructions to build and run 329 | its tests. `fs-bench` is in `testcases/kernel/fs`, and contains its own 330 | instructions. 331 | 332 | `fs-bench` has four phases: 333 | 334 | 1. It creates files until the file system is out of memory. 335 | 2. It randomly accesses the files it creates. 336 | 3. It deletes and creates randomly. 337 | 4. It deletes all files it created. 338 | 339 | I modified the default test, removing changing the file owners between stages. 340 | My implementation does not support changing ownership of files. Running a 341 | benchmark is done simply with a command like this one: 342 | 343 | ```sh 344 | ianwahbe@bf1:/ltp-arm/testcases/kernel/fs/fs-bench/mnt$ ../fs-benchmark.sh |& tee ../result.txt 345 | ``` 346 | 347 | I ran the test on the all three systems, as well as an internal chimera with rdma. 348 | 349 | | | chimera | chimera rdma | bf1 | chimera bf1 rdma | 350 | | -------------------- | ------------- | -------------------- | ------------ | -------------------- | 351 | | technology | native fs | rdma (same computer) | native fs | rdma (bf1 - chimera) | 352 | | create files | 2270277/8m45s | 2283047/10m8s | 58123/2m33s | 57420/4m49s | 353 | | random access | 564363/1m45s | 564272/1m45s | 10452/0m0.6s | 10459/0m8s | 354 | | random create/delete | 250119/1m9s | 250289/1m6s | 4555/3m2s | 4523/0m29s | 355 | | remove | 0m19s | 0m18s | 0m13s | 0m11s | 356 | | total | 25m24s | 26m50s | 6m12s | 10m38s | 357 | 358 | I also attempted to run `fs-bench` over `NFS` as it was already installed. After 359 | an hour, NFS failed, and the benchmark started returning errors. 360 | 361 | I draw 2 conclusions from this data. 362 | 363 | 1. That FUSE and RDMA impose a roughly 10% cost over directly using the build in 364 | file system. We see this from comparing `chimera` and `chimera rdma`. This is 365 | latency from the system itself, not from data transfer. 366 | 2. There is a 90% increase cost from transferring the data back and forth 367 | between host and client. A rough doubling makes sense. Data needs to be moved 368 | from the input buffer (managed by FUSE), to an RDMA buffer. It is then sent to 369 | the host computer, copied into a memory mapped file, and a confirmation is 370 | sent back. This means that any read and write must occur twice. 371 | 372 | 373 | 374 | 375 | 376 | 377 | ### Conclusion 378 | 379 | --- 380 | 381 | I successfully implemented a file system mirroring system that operates over 382 | RDMA. It is incomparably fast versus NFS. I measured the results, and provide a 383 | replicable test. While the system is solid, it could be further optimized. Every 384 | RDMA communication exchanges a full buffer of size `READ_WRITE_BUFFER_SIZE` 385 | (8192). I would like to extend the system, to only exchange the necessary amount 386 | of space. There are also missing capabilities, like permissions and soft links. 387 | 388 | 389 | 390 | -------------------------------------------------------------------------------- /bf1_chimera_rdma_res.txt: -------------------------------------------------------------------------------- 1 | ## Start Test 2 | Thu May 13 23:43:06 UTC 2021 3 | 1620949386 4 | 5 | ## Create files 6 | 7 | Total create files: 57420 8 | 0000e12d: No space left on device 9 | Create files 10 | 11 | real 4m48.970s 12 | user 0m0.448s 13 | sys 0m41.495s 14 | 15 | ## tar all 16 | 17 | ## random access 18 | Success: 10459 19 | Fail: 42 20 | 21 | real 0m4.833s 22 | user 0m0.139s 23 | sys 0m1.884s 24 | 25 | ## Random delete and create 26 | Total create files: 4523 27 | Total delete files: 4697 28 | Total error : 1281 29 | 30 | real 0m29.314s 31 | user 0m0.081s 32 | sys 0m5.155s 33 | 34 | ## Remove all files and directories 35 | 36 | real 0m11.995s 37 | user 0m0.205s 38 | sys 0m10.988s 39 | 40 | ## Finish test 41 | 1620950024 42 | Thu May 13 23:53:44 UTC 2021 43 | TOTAL(seconds): 638 44 | -------------------------------------------------------------------------------- /bf1_res.txt: -------------------------------------------------------------------------------- 1 | ianwahbe@bf1:/ltp-arm/testcases/kernel/fs/fs-bench/bf1$ cat ../bf1-res.txt 2 | ## Start Test 3 | Wed May 12 02:07:10 UTC 2021 4 | 1620785230 5 | 6 | ## Create files 7 | 8 | Total create files: 58123 9 | 0000e3ee: No space left on device 10 | Create files 11 | 12 | real 2m33.505s 13 | user 0m0.396s 14 | sys 0m41.693s 15 | 16 | ## tar all 17 | 18 | ## random access 19 | Success: 10452 20 | Fail: 49 21 | 22 | real 0m1.624s 23 | user 0m0.142s 24 | sys 0m1.453s 25 | 26 | ## Random delete and create 27 | Total create files: 4555 28 | Total delete files: 4704 29 | Total error : 1242 30 | 31 | real 3m2.178s 32 | user 0m0.080s 33 | sys 0m6.600s 34 | 35 | ## Remove all files and directories 36 | 37 | real 0m12.966s 38 | user 0m0.134s 39 | sys 0m11.142s 40 | 41 | ## Finish test 42 | 1620785602 43 | Wed May 12 02:13:22 UTC 2021 44 | TOTAL(seconds): 372 45 | -------------------------------------------------------------------------------- /chimera_rdma_res.txt: -------------------------------------------------------------------------------- 1 | ## Start Test 2 | Tue 11 May 2021 10:52:36 PM PDT 3 | 1620798756 4 | 5 | ## Create files 6 | 7 | Total create files: 2283047 8 | 00231c20: No space left on device 9 | Create files 10 | 11 | real 10m8.012s 12 | user 0m2.352s 13 | sys 2m25.511s 14 | 15 | ## tar all 16 | 17 | ## random access 18 | Success: 564272 19 | Fail: 4244 20 | 21 | real 1m45.316s 22 | user 0m1.320s 23 | sys 0m21.639s 24 | 25 | ## Random delete and create 26 | Total create files: 250289 27 | Total delete files: 250395 28 | Total error : 67832 29 | 30 | real 1m6.619s 31 | user 0m0.657s 32 | sys 0m24.585s 33 | 34 | ## Remove all files and directories 35 | 36 | real 0m18.848s 37 | user 0m0.440s 38 | sys 0m17.076s 39 | 40 | ## Finish test 41 | 1620800366 42 | Tue 11 May 2021 11:19:26 PM PDT 43 | TOTAL(seconds): 1610 44 | -------------------------------------------------------------------------------- /chimera_res.txt: -------------------------------------------------------------------------------- 1 | ianwahbe@chimera:~/ltp/testcases/kernel/fs/fs-bench/chimera$ ../fs-bench-test.sh 2 | ## Start Test 3 | Tue 11 May 2021 06:47:01 PM PDT 4 | 1620784021 5 | 6 | ## Create files 7 | Create files 8 | 9 | Total create files: 2270277 10 | 0022e90c: No space left on device 11 | real 8m45.043s 12 | user 0m2.259s 13 | sys 2m20.411s 14 | 15 | ## tar all 16 | 17 | ## random access 18 | Success: 564363 19 | Fail: 4153 20 | real 1m45.468s 21 | user 0m1.324s 22 | sys 0m21.760s 23 | 24 | ## Random delete and create 25 | Total create files: 250119 26 | Total delete files: 250319 27 | Total error : 68078 28 | 29 | real 1m8.794s 30 | user 0m0.593s 31 | sys 0m26.139s 32 | 33 | ## Remove all files and directories 34 | 35 | real 0m18.877s 36 | user 0m0.412s 37 | sys 0m17.182s 38 | 39 | ## Finish test 40 | 1620785545 41 | Tue 11 May 2021 07:12:25 PM PDT 42 | TOTAL(seconds): 1524 43 | -------------------------------------------------------------------------------- /rdma-fuse.org: -------------------------------------------------------------------------------- 1 | #+TITLE: Rdma Fuse 2 | 3 | * Notes 4 | - Filesystem stat => get inode 5 | - Then direct map for all nodes but root (which must be 1 on fuse) 6 | 7 | * Figure out how to configure RDMA 8 | ** How to set ports when building q-pairs 9 | *** On the =chimera= 10 | Both got =port is not ACTIVE or ARMED= 11 | #+begin_src sh 12 | sudo mst start # to create devices 13 | 14 | #+end_src 15 | *** On the VM 16 | Both got =entity not found= 17 | *** On the card 18 | ip not configured 19 | ** Setting up on bfn1 20 | - Perftest (RDMA) testing suite 21 | - ib_test_write 22 | - ib_test_read 23 | - ib_* for all tests 24 | 25 | * [1/3] Converting to efficient transmission 26 | In an effort to cut down the penalty for sending a full buffer for every action, 27 | we move the previous enum into a manually handled tagged union. We then can send 28 | only the correct amount of data with each RDMA send verb. 29 | ** [2/2] Convert =enum= to tagged union 30 | *** DONE Convert 31 | *** DONE Test 32 | ** [2/2] Convert =RDMAConnection= to use multiple =u8= instead of a single =(Message, MessagePayload)= 33 | *** DONE Convert 34 | *** DONE Test 35 | ** [2/2] Send only the correct amount 36 | *** DONE Convert 37 | *** DONE Test 38 | 39 | * [0/2] Buffer Reads and Writes 40 | Currently reads and writes all =send= verbs to fixed size buffer. If the 41 | required send is too large, the program crashes. If it is too small, we send an 42 | overlarge amount of data. We should instead send only some fixed amount per 43 | =send=, and repeat sends as needed until the message is received. 44 | ** TODO Implement 45 | ** TODO Test 46 | Read / Write 47 | Bandwidth / latency 48 | Sequential: read 49 | Check 50 | #+begin_src sh 51 | time ./fio --name TEST --eta-newline=5s --filename=fio-tempfile.dat --rw=read --size=500m --io_size=10g --blocksize=1024k --ioengine=libaio --fsync=10000 --iodepth=32 --direct=1 --numjobs=1 --runtime=60 --group_reporting 52 | #+end_src 53 | -------------------------------------------------------------------------------- /remote_build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | # verify a valid build 5 | cargo build || exit 1 6 | 7 | ssh ianwahbe@chimera "rm -rf src/rdma-fuse" 8 | mv target /tmp/target-rdma-folder 9 | scp -r ../rdma-fuse ianwahbe@chimera:/home/ianwahbe/src/rdma-fuse 10 | mv /tmp/target-rdma-folder target 11 | ssh ianwahbe@chimera 12 | -------------------------------------------------------------------------------- /src/file.rs: -------------------------------------------------------------------------------- 1 | use crate::Fh; 2 | use memmap2; 3 | use std::os::unix::fs::MetadataExt; 4 | use std::path::Path; 5 | use std::{fs, io}; 6 | 7 | #[derive(Debug)] 8 | pub struct OpenFile { 9 | memory: Option, 10 | file: fs::File, 11 | len: u64, 12 | fh: Fh, 13 | } 14 | 15 | #[derive(Copy, Clone, Debug)] 16 | pub struct FileBuilder { 17 | create: bool, 18 | read: bool, 19 | write: bool, 20 | trunc: bool, 21 | } 22 | 23 | impl FileBuilder { 24 | pub fn new() -> Self { 25 | Self { 26 | create: false, 27 | read: false, 28 | write: false, 29 | trunc: false, 30 | } 31 | } 32 | 33 | pub fn create(mut self, create: bool) -> Self { 34 | self.create = create; 35 | self 36 | } 37 | 38 | pub fn read(mut self, read: bool) -> Self { 39 | self.read = read; 40 | self 41 | } 42 | 43 | pub fn write(mut self, write: bool) -> Self { 44 | self.write = write; 45 | self 46 | } 47 | 48 | /// Setts the flag to O_TRUNC 49 | pub fn trunc(mut self, trunc: bool) -> Self { 50 | self.trunc = trunc; 51 | self 52 | } 53 | 54 | /// Build a set of open flags from C flags 55 | pub fn from_flags(flags: i32) -> Self { 56 | use libc::*; 57 | Self::new() 58 | .read(flags & O_RDONLY != 0) 59 | .write(flags & O_WRONLY != 0) 60 | .create(flags & O_CREAT != 0) 61 | .trunc(flags & O_TRUNC != 0) 62 | } 63 | 64 | /// Creates a new openfile from unix flags 65 | pub fn path(self, path: &Path) -> io::Result { 66 | let file = fs::OpenOptions::new() 67 | .truncate(self.trunc) 68 | .read(self.read) 69 | .write(self.write) 70 | .create(self.create) 71 | .open(path)?; 72 | let meta = file.metadata()?; 73 | let fh = meta.ino() as _; 74 | Ok(OpenFile { 75 | memory: None, 76 | len: meta.len(), 77 | file, 78 | fh, 79 | }) 80 | } 81 | } 82 | 83 | impl OpenFile { 84 | pub fn metadata(&self) -> io::Result { 85 | self.file.metadata() 86 | } 87 | 88 | /// Truncate the file to size. size can be larger or smaller then the 89 | /// previous file size. 90 | pub fn truncate(&mut self, size: u64) -> io::Result<()> { 91 | self.memory.take(); 92 | let trunc = self.file.set_len(size); 93 | if trunc.is_err() { 94 | log::error!("Failed to truncate on fh {:?}", self.fh); 95 | trunc? 96 | } 97 | self.len = size; 98 | Ok(()) 99 | } 100 | 101 | // Expose a unique file handle 102 | pub fn fh(&self) -> Fh { 103 | self.fh 104 | } 105 | 106 | /// Get a reference to the backing memory. 107 | fn get_map(&mut self) -> io::Result<&mut memmap2::MmapMut> { 108 | log::info!("Get map was called on file {:?}", self.fh); 109 | if self.memory.is_none() { 110 | let meta = self.file.metadata()?; 111 | self.len = meta.len(); 112 | assert!(meta.len() > 0, "mmaped files must have positive length"); 113 | self.memory = Some(unsafe { 114 | match memmap2::MmapOptions::new().map_mut(&self.file) { 115 | Ok(k) => k, 116 | Err(e) => { 117 | log::error!( 118 | "Failed to mmap file {:?} with error {}. File size is {}", 119 | self.fh, 120 | e, 121 | meta.len() 122 | ); 123 | return Err(e); 124 | } 125 | } 126 | }); 127 | } 128 | Ok(self.memory.as_mut().unwrap()) 129 | } 130 | 131 | pub fn write(&mut self, buf: &[u8], offset: i64) -> io::Result { 132 | if buf.len() == 0 { 133 | return Ok(0); 134 | } 135 | if buf.len() as u64 + offset as u64 >= self.len { 136 | self.truncate(buf.len() as u64 + offset as u64 + 1)?; 137 | } 138 | let map = self.get_map()?; 139 | assert!( 140 | map.len() as usize > offset as usize + buf.len(), 141 | "We can't write off the end of a file" 142 | ); 143 | unsafe { 144 | let map: *mut u8 = map.as_mut_ptr().offset(offset as _); 145 | std::ptr::copy_nonoverlapping(buf.as_ptr(), map, buf.len()) 146 | }; 147 | Ok(buf.len()) 148 | } 149 | 150 | pub fn flush(&mut self) -> io::Result<()> { 151 | if let Some(mem) = self.memory.as_mut() { 152 | mem.flush()?; 153 | } 154 | Ok(()) 155 | } 156 | pub fn read(&mut self, buf: &mut [u8], offset: i64, size: usize) -> io::Result { 157 | if self.len == 0 { 158 | return Ok(0); 159 | } 160 | let map = self.get_map()?; 161 | let remainder = if map.len() >= offset as usize { 162 | map.len() - offset as usize 163 | } else { 164 | 0 165 | }; 166 | let map_size = buf.len().min(remainder).min(size); 167 | unsafe { 168 | std::ptr::copy_nonoverlapping( 169 | map.as_ptr().offset(offset as _), 170 | buf.as_mut_ptr(), 171 | map_size, 172 | ); 173 | } 174 | Ok(map_size) 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![feature(pin_static_ref)] 2 | #![feature(try_blocks)] 3 | #![feature(option_result_unwrap_unchecked)] 4 | 5 | mod file; 6 | mod local; 7 | mod remote; 8 | 9 | pub use local::LocalMount; 10 | pub use remote::{remote_server, RDMAFs, RDMA_MESSAGE_BUFFER_SIZE}; 11 | 12 | use bincode; 13 | use ibverbs::{CompletionQueue, Context, MemoryRegion, ProtectionDomain, QueuePair}; 14 | use std::mem::transmute; 15 | use std::pin::Pin; 16 | use std::{io, ops::Deref}; 17 | use std::{ 18 | io::{Read, Write}, 19 | ops::DerefMut, 20 | }; 21 | 22 | pub type Ino = u64; 23 | pub type Fh = u64; 24 | 25 | pub struct RDMAConnection 26 | where 27 | T: Copy + Default, 28 | { 29 | // This is unsafe as f**k 30 | // We need the fields to drop in this order to avoid ub 31 | // 32 | // To ensure that our static pointers remain valid, we pin them. This should 33 | // prevent the rust runtime from moving them when they mutate themselves. 34 | mem: MemoryRegion, 35 | qp: QueuePair<'static>, 36 | _pd: Pin>>, 37 | cq: Pin>>, 38 | _ctx: Pin>, 39 | next_id: u64, 40 | } 41 | 42 | impl RDMAConnection 43 | where 44 | T: Copy + Default, 45 | { 46 | // Creates a new `RDMAConnection`. This is a wrapper around a 47 | // `MemoryRegion`, with all the context needed to read and write to it. 48 | pub fn new(size: usize, mut connection: W) -> io::Result 49 | where 50 | W: Read + Write, 51 | { 52 | let ctx: Pin> = Box::pin(unsafe { 53 | transmute( 54 | ibverbs::devices()? 55 | .iter() 56 | .next() 57 | .ok_or_else(|| io::Error::from(io::ErrorKind::NotFound))? 58 | .open()?, 59 | ) 60 | }); 61 | 62 | // Unsafe: We cast the lifetime to static. This is safe because we embed 63 | // this all in the same struct. The struct garentees the drop order to 64 | // be correct. 65 | let cq: Pin>> = 66 | Box::pin(unsafe { transmute(ctx.as_ref().create_cq(16, 0)?) }); 67 | let pd: Pin>> = Box::pin(unsafe { 68 | transmute( 69 | ctx.alloc_pd() 70 | .map_err(|_| io::Error::from(io::ErrorKind::AddrNotAvailable))?, 71 | ) 72 | }); 73 | 74 | let qp_builder = unsafe { 75 | let cq: &'static CompletionQueue = &*(&*cq as *const CompletionQueue); 76 | transmute::<_, &'static ProtectionDomain<'static>>(pd.as_ref()) 77 | .create_qp(cq, cq, ibverbs::ibv_qp_type::IBV_QPT_RC) 78 | .build()? 79 | }; 80 | 81 | let qp = { 82 | let endpoint = qp_builder.endpoint(); 83 | let encode: Vec = bincode::serialize(&endpoint).unwrap(); 84 | connection.write_all(&encode)?; 85 | let endpoint: ibverbs::QueuePairEndpoint = 86 | bincode::deserialize_from(&mut connection).unwrap(); 87 | qp_builder.handshake(endpoint)? 88 | }; 89 | 90 | let mem = pd.allocate::(size)?; 91 | 92 | Ok(RDMAConnection { 93 | cq, 94 | _ctx: ctx, 95 | _pd: pd, 96 | qp, 97 | mem, 98 | next_id: 0, 99 | }) 100 | } 101 | 102 | /// Send a left aligned block of length `size`. The caller is responcible 103 | /// for garenteing that the data read from the memory region is valid if 104 | /// only `size` bytes are written to it. 105 | /// 106 | /// Safety: This function is unsafe. It is up to the caller to ensure that 107 | /// the type that it is sending fits in `size` bytes. 108 | pub unsafe fn send_sized(&mut self, size: usize) -> io::Result<()> { 109 | let id = self.next_id; 110 | self.next_id += 1; 111 | // Unsafe: we perform the unsafe send. This is safe because `self.mem` 112 | // is a correctly configured memory region. It is up to the sender to 113 | // ensure that recieve is called. 114 | self.qp.post_send(&mut self.mem, ..size, id)?; 115 | self.complete(id) 116 | } 117 | 118 | // Send the entire buffer back. 119 | pub fn send(&mut self) -> io::Result<()> { 120 | unsafe { self.send_sized(self.mem.len()) } 121 | } 122 | 123 | /// Recieve a left aligned block of length `size`. The caller is responcible 124 | /// for garenteing that the data read from the memory region is valid if 125 | /// only `size` bytes are written to it. 126 | /// 127 | /// Safety: This function is unsafe. It is up to the caller to ensure that 128 | /// the type that is expected to be recieved fits in `size` bytes. 129 | pub unsafe fn recv_sized(&mut self, size: usize) -> io::Result<()> { 130 | let id = self.next_id; 131 | self.next_id += 1; 132 | self.qp.post_receive(&mut self.mem, ..size, id)?; 133 | self.complete(id) 134 | } 135 | 136 | // recieve on the buffer. 137 | pub fn recv(&mut self) -> io::Result<()> { 138 | unsafe { self.recv_sized(self.mem.len()) } 139 | } 140 | 141 | // Waits on the completion of a send or recieve with a matching `id`. 142 | fn complete(&mut self, id: u64) -> io::Result<()> { 143 | let mut completions = [ibverbs::ibv_wc::default(); 16]; 144 | loop { 145 | let completed = self 146 | .cq 147 | .poll(&mut completions[..]) 148 | .map_err(|_| io::Error::from(io::ErrorKind::Interrupted))?; 149 | for wr in completed { 150 | if wr.wr_id() == id { 151 | return Ok(()); 152 | } 153 | } 154 | } 155 | } 156 | } 157 | 158 | impl Deref for RDMAConnection 159 | where 160 | T: Copy + Default, 161 | { 162 | type Target = [T]; 163 | 164 | fn deref(&self) -> &Self::Target { 165 | &self.mem 166 | } 167 | } 168 | 169 | impl DerefMut for RDMAConnection 170 | where 171 | T: Copy + Default, 172 | { 173 | fn deref_mut(&mut self) -> &mut Self::Target { 174 | &mut self.mem 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /src/local.rs: -------------------------------------------------------------------------------- 1 | use crate::file::{FileBuilder, OpenFile}; 2 | use fuser::*; 3 | use fuser::{Filesystem, KernelConfig, Request}; 4 | use libc::{c_int, statfs, ENOSYS}; 5 | use nix::errno::{errno, Errno}; 6 | use std::collections::HashMap; 7 | use std::convert::TryInto; 8 | use std::ffi::{CStr, CString, OsStr}; 9 | use std::fs; 10 | use std::time::SystemTime; 11 | use std::{ 12 | os::unix::{ffi::OsStrExt, fs::MetadataExt}, 13 | path::{Path, PathBuf}, 14 | }; 15 | 16 | use crate::{Fh, Ino}; 17 | 18 | /// A filesystem that reflects a local file system. 19 | pub struct LocalMount { 20 | /// Where the root of `Mount` is located in the host file system. 21 | root: PathBuf, 22 | /// The ino of the root of the reflected file system. 23 | root_ino: Ino, 24 | ino_paths: HashMap, 25 | open_files: HashMap, 26 | } 27 | 28 | impl LocalMount { 29 | pub fn new>(root: T) -> Self { 30 | LocalMount { 31 | root: root.into(), 32 | ino_paths: HashMap::new(), 33 | root_ino: 0, 34 | open_files: HashMap::new(), 35 | } 36 | } 37 | 38 | fn at_ino(&self, ino: &Ino) -> Option<&PathBuf> { 39 | if *ino == 1 || *ino == self.root_ino { 40 | Some(&self.root) 41 | } else { 42 | self.ino_paths.get(ino) 43 | } 44 | } 45 | 46 | fn register_new_file(&mut self, file: OpenFile) -> Fh { 47 | let fh = file.fh(); 48 | self.open_files.entry(fh).or_insert(file); 49 | fh 50 | } 51 | } 52 | 53 | impl Filesystem for LocalMount { 54 | fn init(&mut self, _req: &Request<'_>, _config: &mut KernelConfig) -> Result<(), c_int> { 55 | if !self.root.exists() || !self.root.is_dir() { 56 | return Err(1); 57 | } 58 | self.root = self.root.canonicalize().unwrap(); 59 | let root_data = fs::metadata(&self.root).unwrap(); 60 | self.root_ino = root_data.ino(); 61 | log::info!("File system mounted to reflect {:?}", self.root); 62 | Ok(()) 63 | } 64 | 65 | fn destroy(&mut self, _req: &Request<'_>) { 66 | self.open_files.clear(); 67 | log::info!("File system destroyed") 68 | } 69 | 70 | /// Open a file. 71 | /// Open flags (with the exception of O_CREAT, O_EXCL, O_NOCTTY and O_TRUNC) are 72 | /// available in flags. Filesystem may store an arbitrary file handle (pointer, index, 73 | /// etc) in fh, and use this in other all other file operations (read, write, flush, 74 | /// release, fsync). Filesystem may also implement stateless file I/O and not store 75 | /// anything in fh. There are also some flags (direct_io, keep_cache) which the 76 | /// filesystem may set, to change the way the file is opened. See fuse_file_info 77 | /// structure in for more details. 78 | fn open(&mut self, _req: &Request<'_>, ino: Ino, flags: i32, reply: ReplyOpen) { 79 | if let Some(path) = self.at_ino(&ino) { 80 | match FileBuilder::from_flags(flags) 81 | .read(true) 82 | .write(true) 83 | .path(path) 84 | { 85 | Ok(file) => { 86 | let fh = self.register_new_file(file); 87 | reply.opened(fh as _, flags as _); 88 | } 89 | Err(e) => reply.error(e.raw_os_error().unwrap_or(libc::EIO)), 90 | } 91 | } else { 92 | log::error!("Open failed with invalid ino {:?}", ino); 93 | reply.error(libc::EIO); 94 | } 95 | } 96 | 97 | /// Look up a directory entry by name and get its attributes. 98 | fn lookup(&mut self, _req: &Request<'_>, parent: Ino, name: &OsStr, reply: ReplyEntry) { 99 | let parent = if let Some(k) = self.at_ino(&parent) { 100 | k 101 | } else { 102 | reply.error(libc::ENOENT); 103 | log::error!( 104 | "Attempted lookup of parrent ino {:?}. File not found.", 105 | parent 106 | ); 107 | return; 108 | }; 109 | let new_file = parent.join(name); 110 | let data = if let Ok(k) = fs::metadata(&new_file) { 111 | k 112 | } else { 113 | reply.error(libc::ENOENT); 114 | return; 115 | }; 116 | reply.entry(&std::time::Duration::ZERO, &(&data).into(), 0); 117 | log::trace!( 118 | "Performed lookup on {:?} with parrent {:?}. Found new Ino {:?}", 119 | name, 120 | parent, 121 | new_file 122 | ); 123 | self.ino_paths.insert(data.ino(), new_file); 124 | } 125 | 126 | /// Get file attributes. 127 | fn getattr(&mut self, _req: &Request<'_>, ino: Ino, reply: ReplyAttr) { 128 | let handle; 129 | let buf = if ino == 1 { 130 | &self.root 131 | } else { 132 | handle = self.root.join(self.at_ino(&ino).unwrap()); 133 | &handle 134 | }; 135 | if let Ok(k) = fs::metadata(buf) { 136 | reply.attr(&std::time::Duration::ZERO, &(&k).into()); 137 | log::trace!("Replied with metadata of file {:?}", buf); 138 | } else { 139 | log::error!("Failed lookup on ino {:?} = {:?}", ino, buf); 140 | reply.error(libc::ENOENT); 141 | }; 142 | } 143 | 144 | /// Set file attributes. 145 | fn setattr( 146 | &mut self, 147 | _req: &Request<'_>, 148 | ino: u64, 149 | mode: Option, 150 | uid: Option, 151 | gid: Option, 152 | size: Option, 153 | atime: Option, 154 | mtime: Option, 155 | ctime: Option, 156 | fh: Option, 157 | crtime: Option, 158 | chgtime: Option, 159 | bkuptime: Option, 160 | flags: Option, 161 | reply: ReplyAttr, 162 | ) { 163 | let res = try { 164 | let path = self.at_ino(&ino).ok_or(libc::ENOENT)?; 165 | let mut attr: FileAttr = fs::metadata(path).as_ref().unwrap().try_into().unwrap(); 166 | macro_rules! maybe_set_attr { 167 | ($name: tt) => { 168 | if let Some($name) = $name { 169 | attr.$name = $name 170 | } 171 | }; 172 | ($name: tt, "time") => { 173 | if let Some($name) = $name { 174 | attr.$name = match $name { 175 | TimeOrNow::Now => SystemTime::now(), 176 | TimeOrNow::SpecificTime(t) => t, 177 | } 178 | } 179 | }; 180 | ($name: tt, "ignore") => { 181 | let _ = $name; 182 | }; 183 | } 184 | maybe_set_attr!(mode, "ignore"); 185 | maybe_set_attr!(uid); 186 | maybe_set_attr!(gid); 187 | maybe_set_attr!(size); 188 | maybe_set_attr!(atime, "time"); 189 | maybe_set_attr!(mtime, "time"); 190 | maybe_set_attr!(ctime); 191 | maybe_set_attr!(fh, "ignore"); 192 | maybe_set_attr!(crtime); 193 | maybe_set_attr!(chgtime, "ignore"); 194 | maybe_set_attr!(bkuptime, "ignore"); 195 | maybe_set_attr!(flags); 196 | attr 197 | }; 198 | match res { 199 | Ok(k) => reply.attr(&std::time::Duration::ZERO, &k), 200 | Err(e) => reply.error(e), 201 | } 202 | } 203 | 204 | /// Read symbolic link. 205 | fn readlink(&mut self, _req: &Request<'_>, ino: u64, reply: ReplyData) { 206 | let res = try { 207 | let path = self.at_ino(&ino).ok_or(libc::ENOENT)?; 208 | let path = CString::new(path.as_os_str().as_bytes()).unwrap(); 209 | static BUF: &[u8] = &[0; libc::PATH_MAX as usize]; 210 | let res = unsafe { libc::readlink(path.as_ptr(), BUF.as_ptr() as _, BUF.len()) }; 211 | match res { 212 | size @ _ if size >= 0 => &BUF[..size as usize], 213 | _ => Err(errno())?, 214 | } 215 | }; 216 | match res { 217 | Ok(k) => reply.data(k), 218 | Err(e) => reply.error(e), 219 | } 220 | } 221 | 222 | /// Create a directory. 223 | fn mkdir( 224 | &mut self, 225 | _req: &Request<'_>, 226 | parent: Ino, 227 | name: &OsStr, 228 | mode: u32, 229 | umask: u32, 230 | reply: ReplyEntry, 231 | ) { 232 | let res = try { 233 | let name = self.at_ino(&parent).ok_or(libc::ENOENT)?.join(name); 234 | let c_name = CString::new(name.as_os_str().as_bytes()).unwrap(); 235 | // NOTE: unsure about the bit-and 236 | let res = unsafe { libc::mkdir(c_name.as_ptr(), (mode & umask) as _) }; 237 | match res { 238 | 0 => fs::metadata(&name).as_ref().unwrap().try_into().unwrap(), 239 | _ => Err(errno())?, 240 | } 241 | }; 242 | match res { 243 | Ok(k) => reply.entry(&std::time::Duration::ZERO, &k, 0), 244 | Err(e) => reply.error(e), 245 | } 246 | } 247 | 248 | /// Remove a file. 249 | fn unlink(&mut self, _req: &Request<'_>, parent: u64, name: &OsStr, reply: ReplyEmpty) { 250 | if let Some(parent) = self.at_ino(&parent) { 251 | let fname = CString::new(parent.join(name).as_os_str().as_bytes()).unwrap(); 252 | let res = unsafe { libc::unlink(fname.as_ptr()) }; 253 | if res == 0 { 254 | reply.ok() 255 | } else { 256 | reply.error(errno()) 257 | } 258 | } else { 259 | log::error!("Unlink failed on invalid parent ino {:?}", parent); 260 | reply.error(libc::ENOENT); 261 | } 262 | } 263 | 264 | /// Remove a directory. 265 | fn rmdir(&mut self, _req: &Request<'_>, parent: Ino, name: &OsStr, reply: ReplyEmpty) { 266 | let res = try { 267 | let path = self.at_ino(&parent).ok_or(libc::ENOENT)?.join(name); 268 | let path = CString::new(path.as_os_str().as_bytes()).unwrap(); 269 | let res = unsafe { libc::rmdir(path.as_ptr()) }; 270 | match res { 271 | 0 => (), 272 | _ => Err(errno())?, 273 | } 274 | }; 275 | match res { 276 | Ok(_) => reply.ok(), 277 | Err(e) => reply.error(e), 278 | } 279 | } 280 | 281 | /// Create a symbolic link. 282 | fn symlink( 283 | &mut self, 284 | _req: &Request<'_>, 285 | parent: u64, 286 | name: &OsStr, 287 | link: &Path, 288 | reply: ReplyEntry, 289 | ) { 290 | let res = try { 291 | let path = self.at_ino(&parent).ok_or(libc::ENOENT)?.join(name); 292 | let path = CString::new(path.as_os_str().as_bytes()).unwrap(); 293 | let c_link = CString::new(link.as_os_str().as_bytes()).unwrap(); 294 | let res = unsafe { libc::symlink(path.as_ptr(), c_link.as_ptr()) }; 295 | match res { 296 | 0 => fs::metadata(link).as_ref().unwrap().into(), 297 | _ => Err(errno())?, 298 | } 299 | }; 300 | match res { 301 | Ok(k) => reply.entry(&std::time::Duration::ZERO, &k, 0), 302 | Err(e) => reply.error(e), 303 | } 304 | } 305 | 306 | /// Rename a file. 307 | fn rename( 308 | &mut self, 309 | _req: &Request<'_>, 310 | parent: Ino, 311 | name: &OsStr, 312 | newparent: Ino, 313 | newname: &OsStr, 314 | _flags: u32, 315 | reply: ReplyEmpty, 316 | ) { 317 | // TODO: revalidate the [ino <-> path] map 318 | let res = try { 319 | let old_name = CString::new(name.as_bytes()).unwrap(); 320 | let newname = CString::new(newname.as_bytes()).unwrap(); 321 | let res = unsafe { 322 | libc::renameat( 323 | parent as _, 324 | old_name.as_ptr(), 325 | newparent as _, 326 | newname.as_ptr(), 327 | ) 328 | }; 329 | match res { 330 | 0 => (), 331 | _ => Err(errno())?, 332 | } 333 | }; 334 | match res { 335 | Ok(_) => reply.ok(), 336 | Err(e) => reply.error(e), 337 | } 338 | } 339 | 340 | /// Create a hard link. 341 | fn link( 342 | &mut self, 343 | _req: &Request<'_>, 344 | ino: Ino, 345 | newparent: Ino, 346 | newname: &OsStr, 347 | reply: ReplyEntry, 348 | ) { 349 | let res = try { 350 | let old_name = CString::new( 351 | self.at_ino(&ino) 352 | .ok_or(libc::ENOENT)? 353 | .as_os_str() 354 | .as_bytes(), 355 | ) 356 | .unwrap(); 357 | let newparent = self.at_ino(&newparent).ok_or(libc::ENOENT)?; 358 | let path = newparent.join(newname); 359 | let path = path.as_os_str(); 360 | let newname = CString::new(path.as_bytes()).unwrap(); 361 | let res = unsafe { libc::link(old_name.as_ptr(), newname.as_ptr()) }; 362 | match res { 363 | 0 => fs::metadata(path).as_ref().unwrap().try_into().unwrap(), 364 | _ => Err(errno())?, 365 | } 366 | }; 367 | match res { 368 | Ok(k) => reply.entry(&std::time::Duration::ZERO, &k, 0), 369 | Err(e) => reply.error(e), 370 | } 371 | } 372 | 373 | /// Read data. 374 | /// Read should send exactly the number of bytes requested except on EOF or error, 375 | /// otherwise the rest of the data will be substituted with zeroes. An exception to 376 | /// this is when the file has been opened in 'direct_io' mode, in which case the 377 | /// return value of the read system call will reflect the return value of this 378 | /// operation. fh will contain the value set by the open method, or will be undefined 379 | /// if the open method didn't set any value. 380 | /// 381 | /// flags: these are the file flags, such as O_SYNC. Only supported with ABI >= 7.9 382 | /// lock_owner: only supported with ABI >= 7.9 383 | fn read( 384 | &mut self, 385 | _req: &Request<'_>, 386 | _ino: u64, 387 | fh: u64, 388 | offset: i64, 389 | size: u32, 390 | _flags: i32, 391 | _lock_owner: Option, 392 | reply: ReplyData, 393 | ) { 394 | let size = size as usize; 395 | if let Some(f) = self.open_files.get_mut(&(fh as _)) { 396 | let mut buf = vec![0; size]; 397 | match f.read(&mut buf, offset, size) { 398 | Ok(bytes_read) => { 399 | log::info!("Read {} bytes into file {}", bytes_read, fh); 400 | reply.data(&buf); 401 | } 402 | Err(e) => reply.error(e.raw_os_error().unwrap_or(1) as _), 403 | } 404 | } else { 405 | reply.error(libc::EBADF) 406 | } 407 | } 408 | 409 | /// Write data. 410 | /// Write should return exactly the number of bytes requested except on error. An 411 | /// exception to this is when the file has been opened in 'direct_io' mode, in 412 | /// which case the return value of the write system call will reflect the return 413 | /// value of this operation. fh will contain the value set by the open method, or 414 | /// will be undefined if the open method didn't set any value. 415 | /// 416 | /// write_flags: will contain FUSE_WRITE_CACHE, if this write is from the page cache. If set, 417 | /// the pid, uid, gid, and fh may not match the value that would have been sent if write cachin 418 | /// is disabled 419 | /// flags: these are the file flags, such as O_SYNC. Only supported with ABI >= 7.9 420 | /// lock_owner: only supported with ABI >= 7.9 421 | fn write( 422 | &mut self, 423 | _req: &Request<'_>, 424 | _ino: u64, 425 | fh: u64, 426 | offset: i64, 427 | data: &[u8], 428 | _write_flags: u32, 429 | _flags: i32, 430 | _lock_owner: Option, 431 | reply: ReplyWrite, 432 | ) { 433 | log::info!("Attempting to write to file at handle: {:?}", fh); 434 | if let Some(f) = self.open_files.get_mut(&fh) { 435 | match f.write(data, offset) { 436 | Ok(len) => reply.written(len as _), 437 | Err(e) => { 438 | log::error!("Failed to wtite bytes: {}", e); 439 | reply.error(e.raw_os_error().unwrap()); 440 | } 441 | } 442 | } else { 443 | log::error!( 444 | "Failed to write bytes to fh {:?}, could not find open file. There are {} open files.", 445 | fh, self.open_files.keys().len() 446 | ); 447 | reply.error(0); 448 | } 449 | } 450 | 451 | /// Flush method. 452 | /// This is called on each close() of the opened file. Since file descriptors can 453 | /// be duplicated (dup, dup2, fork), for one open call there may be many flush 454 | /// calls. Filesystems shouldn't assume that flush will always be called after some 455 | /// writes, or that if will be called at all. fh will contain the value set by the 456 | /// open method, or will be undefined if the open method didn't set any value. 457 | /// NOTE: the name of the method is misleading, since (unlike fsync) the filesystem 458 | /// is not forced to flush pending writes. One reason to flush data, is if the 459 | /// filesystem wants to return write errors. If the filesystem supports file locking 460 | /// operations (setlk, getlk) it should remove all locks belonging to 'lock_owner'. 461 | fn flush( 462 | &mut self, 463 | _req: &Request<'_>, 464 | _ino: u64, 465 | fh: u64, 466 | _lock_owner: u64, 467 | reply: ReplyEmpty, 468 | ) { 469 | if let Some(f) = self.open_files.get_mut(&fh) { 470 | if let Some(r) = f.flush().err() { 471 | reply.error(r.raw_os_error().unwrap()); 472 | } else { 473 | reply.ok(); 474 | } 475 | } else { 476 | reply.error(libc::ENOENT); 477 | } 478 | } 479 | 480 | /// Release an open file. 481 | /// Release is called when there are no more references to an open file: all file 482 | /// descriptors are closed and all memory mappings are unmapped. For every open 483 | /// call there will be exactly one release call. The filesystem may reply with an 484 | /// error, but error values are not returned to close() or munmap() which triggered 485 | /// the release. fh will contain the value set by the open method, or will be undefined 486 | /// if the open method didn't set any value. flags will contain the same flags as for 487 | /// open. 488 | fn release( 489 | &mut self, 490 | _req: &Request<'_>, 491 | ino: u64, 492 | fh: u64, 493 | _flags: i32, 494 | _lock_owner: Option, 495 | _flush: bool, 496 | reply: ReplyEmpty, 497 | ) { 498 | log::trace!("Released file {:?} with fh {:?}", self.at_ino(&ino), fh); 499 | self.open_files.remove(&fh); 500 | reply.ok(); 501 | } 502 | 503 | /// Synchronize file contents. 504 | /// If the datasync parameter is non-zero, then only the user data should be flushed, 505 | /// not the meta data. 506 | fn fsync( 507 | &mut self, 508 | _req: &Request<'_>, 509 | _ino: u64, 510 | fh: u64, 511 | _datasync: bool, 512 | reply: ReplyEmpty, 513 | ) { 514 | if let Some(f) = self.open_files.get_mut(&fh) { 515 | match f.flush() { 516 | Ok(_) => reply.ok(), 517 | Err(e) => reply.error(e.raw_os_error().unwrap()), 518 | } 519 | } 520 | } 521 | 522 | /// Open a directory. 523 | /// Filesystem may store an arbitrary file handle (pointer, index, etc) in fh, and 524 | /// use this in other all other directory stream operations (readdir, releasedir, 525 | /// fsyncdir). Filesystem may also implement stateless directory I/O and not store 526 | /// anything in fh, though that makes it impossible to implement standard conforming 527 | /// directory stream operations in case the contents of the directory can change 528 | /// between opendir and releasedir. 529 | fn opendir(&mut self, _req: &Request<'_>, ino: Ino, flags: i32, reply: ReplyOpen) { 530 | let buf = if let Some(buf) = self.at_ino(&ino) { 531 | CString::new(buf.as_os_str().as_bytes()).expect("buf should not contain a null pointer") 532 | } else { 533 | log::error!("opendir: Invalid ino {:?}", ino); 534 | return; 535 | }; 536 | let res = unsafe { libc::opendir(buf.as_ptr() as _) }; 537 | if res.is_null() { 538 | log::error!("opendir: libc call failed with ERRNO=?"); 539 | reply.error(errno()); 540 | } else { 541 | reply.opened(res as u64, flags as u32); 542 | } 543 | } 544 | 545 | /// Read directory. 546 | /// Send a buffer filled using buffer.fill(), with size not exceeding the 547 | /// requested size. Send an empty buffer on end of stream. fh will contain the 548 | /// value set by the opendir method, or will be undefined if the opendir method 549 | /// didn't set any value. 550 | fn readdir( 551 | &mut self, 552 | _req: &Request<'_>, 553 | ino: u64, 554 | fh: u64, 555 | _offset: i64, 556 | mut reply: ReplyDirectory, 557 | ) { 558 | loop { 559 | Errno::clear(); // Because it's not clear if readdir failed from it's output 560 | let dir_ent = unsafe { libc::readdir(fh as _) }; 561 | if dir_ent.is_null() { 562 | use libc::*; 563 | match errno() { 564 | EACCES | EBADF | EMFILE | ENFILE | ENOENT | ENOMEM | ENOTDIR => { 565 | reply.error(errno()); 566 | log::error!( 567 | "Encountered error {} reading directory {:?}, fh {:?}", 568 | std::io::Error::from_raw_os_error(errno()), 569 | self.at_ino(&ino) 570 | .map(|b| b.as_os_str()) 571 | .unwrap_or_else(|| OsStr::new("Unknown")), 572 | fh 573 | ); 574 | return; 575 | } 576 | _ => break, 577 | } 578 | } 579 | let dir_ent = unsafe { *dir_ent }; 580 | let file_len = unsafe { CStr::from_ptr(dir_ent.d_name.as_ptr() as _) } 581 | .to_bytes() 582 | .len(); 583 | let file = OsStr::from_bytes(unsafe { 584 | // We need to allow an extra step to convert u8 to i8 on some machines 585 | &*(&dir_ent.d_name[..file_len] as *const [_] as *const [u8]) 586 | }); 587 | log::trace!("File {:?} under dir {:?}", file, ino); 588 | 589 | if dir_ent.d_ino == 0 { 590 | continue; // file has been deleted, but has not yet been removed 591 | } 592 | 593 | // This conversion is not always necessary on all systems. 594 | // The seek data has different names on different OSs 595 | #[allow(clippy::useless_conversion)] 596 | #[cfg(target_os = "macos")] 597 | let seek = dir_ent 598 | .d_seekoff 599 | .try_into() 600 | .expect("File length does not fit into i64"); 601 | #[cfg(not(target_os = "macos"))] 602 | let seek = dir_ent.d_off; 603 | 604 | let full = reply.add( 605 | dir_ent.d_ino, 606 | seek, 607 | dir_ent.d_type.try_into().expect("Unknown file type"), 608 | file, 609 | ); 610 | self.ino_paths.insert( 611 | dir_ent.d_ino, 612 | self.at_ino(&ino).expect("Valid ino number").join(file), 613 | ); 614 | if full { 615 | break; 616 | } 617 | } 618 | reply.ok(); 619 | log::trace!("Read directory at ino: {}", ino); 620 | } 621 | 622 | /// Release an open directory. 623 | /// For every opendir call there will be exactly one releasedir call. fh will 624 | /// contain the value set by the opendir method, or will be undefined if the 625 | /// opendir method didn't set any value. 626 | fn releasedir( 627 | &mut self, 628 | _req: &Request<'_>, 629 | _ino: u64, 630 | fh: u64, 631 | _flags: i32, 632 | reply: ReplyEmpty, 633 | ) { 634 | let res = unsafe { libc::closedir(fh as _) }; 635 | if res == 0 { 636 | reply.ok(); 637 | } else { 638 | reply.error(errno()); 639 | } 640 | } 641 | 642 | /// Get file system statistics. 643 | fn statfs(&mut self, _req: &Request<'_>, ino: Ino, reply: ReplyStatfs) { 644 | let mut buf: statfs = unsafe { std::mem::MaybeUninit::uninit().assume_init() }; 645 | let path = if let Some(k) = self.at_ino(&ino) { 646 | k.as_os_str() 647 | } else { 648 | log::error!("statfs: Attempted to access invalid ino {:?}", ino); 649 | reply.error(errno()); 650 | return; 651 | }; 652 | log::trace!("Replying with file system stats called on file {:?}", path); 653 | let cstr = CString::new(path.as_bytes()).unwrap(); 654 | let e = unsafe { statfs(cstr.as_ptr(), &mut buf as _) }; 655 | if e != 0 { 656 | log::error!( 657 | "Error {:?} attempting to get file system statistics on path {:?}, ino: {:?}", 658 | e, 659 | path, 660 | ino 661 | ); 662 | reply.error(errno()); 663 | return; 664 | } 665 | reply.statfs( 666 | buf.f_blocks as _, 667 | buf.f_bfree as _, 668 | buf.f_bavail as _, 669 | buf.f_files as _, 670 | buf.f_ffree as _, 671 | buf.f_bsize as _, 672 | 255, 673 | buf.f_bsize as _, // Hardly ever used: 674 | // https://stackoverflow.com/questions/54823541/what-do-f-bsize-and-f-frsize-in-struct-statvfs-stand-for 675 | ); 676 | } 677 | 678 | /// Check file access permissions. 679 | /// This will be called for the access() system call. If the 'default_permissions' 680 | /// mount option is given, this method is not called. This method is not called 681 | /// under Linux kernel versions 2.4.x 682 | fn access(&mut self, _req: &Request<'_>, ino: u64, mask: i32, reply: ReplyEmpty) { 683 | if let Some(k) = self.at_ino(&ino) { 684 | log::trace!("Attempted permissions access of file {:?}", k); 685 | match unsafe { libc::access(k.as_os_str().as_bytes().as_ptr() as _, mask) } { 686 | 0 => reply.ok(), 687 | _ => reply.error(errno()), 688 | }; 689 | } else { 690 | log::error!("Failed to get permissions: invalid ino: {:?}", ino); 691 | reply.error(ENOSYS); 692 | }; 693 | } 694 | 695 | /// Create and open a file. 696 | /// If the file does not exist, first create it with the specified mode, and then 697 | /// open it. Open flags (with the exception of O_NOCTTY) are available in flags. 698 | /// Filesystem may store an arbitrary file handle (pointer, index, etc) in fh, 699 | /// and use this in other all other file operations (read, write, flush, release, 700 | /// fsync). There are also some flags (direct_io, keep_cache) which the 701 | /// filesystem may set, to change the way the file is opened. See fuse_file_info 702 | /// structure in for more details. If this method is not 703 | /// implemented or under Linux kernel versions earlier than 2.6.15, the mknod() 704 | /// and open() methods will be called instead. 705 | fn create( 706 | &mut self, 707 | _req: &Request<'_>, 708 | parent: Ino, 709 | name: &OsStr, 710 | _mode: u32, 711 | _umask: u32, 712 | flags: i32, 713 | reply: ReplyCreate, 714 | ) { 715 | if let Some(parent) = self.at_ino(&parent) { 716 | let path = parent.join(name); 717 | log::info!("create called on file {:?}", path); 718 | let file = match FileBuilder::from_flags(flags) 719 | .create(true) 720 | .read(true) 721 | .write(true) 722 | .path(&path) 723 | { 724 | Ok(k) => k, 725 | Err(e) => { 726 | log::error!("Failed to create file {:?} with error: {}", path, e); 727 | reply.error(e.raw_os_error().unwrap()); 728 | return; 729 | } 730 | }; 731 | let meta = file.metadata().expect("already called, so should work"); 732 | self.ino_paths.insert(meta.ino(), path); 733 | let fd = self.register_new_file(file); 734 | reply.created( 735 | &std::time::Duration::ZERO, 736 | &(&meta).into(), 737 | 0, 738 | fd as _, 739 | flags as _, 740 | ) 741 | } else { 742 | log::error!("Failed to create file {:?} with errno {:?}", name, errno()); 743 | reply.error(errno()); 744 | } 745 | } 746 | 747 | /// Reposition read/write file offset 748 | fn lseek( 749 | &mut self, 750 | _req: &Request<'_>, 751 | _ino: u64, 752 | fh: u64, 753 | offset: i64, 754 | whence: i32, 755 | reply: ReplyLseek, 756 | ) { 757 | let offset = unsafe { libc::lseek(fh as _, offset, whence) }; 758 | if offset == -1 { 759 | log::error!("lseek failed with error"); 760 | reply.error(errno()); 761 | } else { 762 | reply.offset(offset); 763 | } 764 | } 765 | } 766 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use clap::{App, AppSettings, Arg, ArgMatches, SubCommand}; 2 | use env_logger; 3 | use fuser::spawn_mount; 4 | use rdma_fuse::{remote_server, LocalMount, RDMAConnection, RDMAFs, RDMA_MESSAGE_BUFFER_SIZE}; 5 | use std::{io, io::stdin, net::ToSocketAddrs, path::PathBuf}; 6 | use std::{net::TcpStream, str::FromStr}; 7 | 8 | const DEFAULT_IP: &str = "127.0.0.1:8080"; 9 | const IP_COMMAND: &str = "ip"; 10 | 11 | fn main() -> io::Result<()> { 12 | env_logger::init(); 13 | let ip_arg = Arg::with_name(IP_COMMAND) 14 | .short("p") 15 | .long(IP_COMMAND) 16 | .takes_value(true) 17 | .help("What ip (Tcp) address threads will exchange RDMA enpoints over.") 18 | .validator(|s| match std::net::SocketAddr::from_str(&s) { 19 | Ok(_) => Ok(()), 20 | Err(e) => Err(format!("{}, value should be formated as 127.0.0.1:8000", e)), 21 | }); 22 | let matches = App::new("Passthrough FS") 23 | .version("0.1") 24 | .author("Ian wahbe") 25 | .setting(AppSettings::SubcommandRequiredElseHelp) 26 | .subcommand( 27 | SubCommand::with_name("local") 28 | .about( 29 | "Mount a local filesystem, mirroring another point on the current file system", 30 | ) 31 | .arg( 32 | Arg::with_name("rdma") 33 | .takes_value(false) 34 | .help("Even though this is the same system, use rdma anyway") 35 | .long("rdma"), 36 | ) 37 | .arg( 38 | Arg::with_name("mount at") 39 | .index(1) 40 | .required(true) 41 | .help("The empty directory on which to mount the virtual FS"), 42 | ) 43 | .arg( 44 | Arg::with_name("mount to") 45 | .index(2) 46 | .required(true) 47 | .help("The point on the host filesystem to mirror"), 48 | ) 49 | .arg(ip_arg.clone()), 50 | ) 51 | .subcommand( 52 | SubCommand::with_name("test") 53 | .about( 54 | "Run a test, confirming that RDMA exists, and that a connection can be formed", 55 | ) 56 | .arg( 57 | Arg::with_name("sender") 58 | .help("This thread will send an rdma message.") 59 | .long("sender") 60 | .takes_value(false) 61 | .conflicts_with("receiver"), 62 | ) 63 | .arg( 64 | Arg::with_name("receiver") 65 | .help("This thread will recieve an rdma message.") 66 | .long("receiver") 67 | .takes_value(false) 68 | .conflicts_with("sender"), 69 | ) 70 | .arg(ip_arg.clone()), 71 | ) 72 | .subcommand( 73 | SubCommand::with_name("remote") 74 | .about("Facilitate file system mirroring over RDMA") 75 | .arg( 76 | Arg::with_name("client") 77 | .help("Provide the local file system, talking to a remote host to get data") 78 | .long("client") 79 | .takes_value(true) 80 | .conflicts_with("host") 81 | .required(true), 82 | ) 83 | .arg( 84 | Arg::with_name("host") 85 | .help("Provides the remote end of an RDMA file sytem.") 86 | .long("host") 87 | .takes_value(true) 88 | .conflicts_with("client") 89 | .required(true), 90 | ) 91 | .arg(ip_arg), 92 | ) 93 | .get_matches(); 94 | if let Some(matches) = matches.subcommand_matches("test") { 95 | handle_test(matches)?; 96 | } 97 | 98 | if let Some(matches) = matches.subcommand_matches("local") { 99 | let mountpoint = matches 100 | .value_of("mount at") 101 | .expect("Clap ensures this is non-empty"); 102 | let mount_reflect = matches.value_of("mount to").expect("Mount failed"); 103 | 104 | if matches.is_present("rdma") { 105 | let port = matches.value_of(IP_COMMAND).unwrap_or(DEFAULT_IP); 106 | // We ape a remote rdma process over the default port 107 | let mut join = std::process::Command::new(std::env::args().next().unwrap()) 108 | .arg("remote") 109 | .arg("--client") 110 | .arg(mountpoint) 111 | .arg("--ip") 112 | .arg(port) 113 | .spawn()?; 114 | let res = std::process::Command::new(std::env::args().next().unwrap()) 115 | .arg("remote") 116 | .arg("--host") 117 | .arg(mount_reflect) 118 | .arg("--ip") 119 | .arg(port) 120 | .spawn()? 121 | .wait()?; 122 | let join = join.wait()?; 123 | std::process::exit(if join.success() && res.success() { 124 | 0 125 | } else { 126 | 1 127 | }); 128 | } else { 129 | // It's all a local process 130 | let _backround = spawn_mount(LocalMount::new(mount_reflect), &mountpoint, &[]).unwrap(); 131 | let mut s = String::new(); 132 | println!("Return on input"); 133 | stdin().read_line(&mut s).expect("Failed to read input"); 134 | } 135 | } 136 | 137 | if let Some(matches) = matches.subcommand_matches("remote") { 138 | handle_remote(matches)?; 139 | } 140 | Ok(()) 141 | } 142 | 143 | /// Initiates a rdma connection as either a sender or reciever. 144 | /// 145 | /// Communicates an enpoint across `port`, and either sends or receives based on 146 | /// `sender`. 147 | fn test_rdma(sender: bool, port: &mut std::net::TcpStream) -> io::Result<()> { 148 | let mut con: RDMAConnection = RDMAConnection::new(1, port)?; 149 | if sender { 150 | con[0] = 42; 151 | con.send()?; 152 | } else { 153 | con.recv()?; 154 | assert_eq!(42, con[0]); 155 | } 156 | Ok(()) 157 | } 158 | 159 | fn handle_test(matches: &ArgMatches) -> io::Result<()> { 160 | let port = 161 | std::net::SocketAddr::from_str(matches.value_of(IP_COMMAND).unwrap_or(DEFAULT_IP)).unwrap(); 162 | if matches.is_present("sender") { 163 | let mut socket = std::net::TcpStream::connect(&port); 164 | while socket.is_err() { 165 | std::thread::sleep(std::time::Duration::from_millis(50)); 166 | socket = std::net::TcpStream::connect(&port); 167 | } 168 | let mut socket = socket.unwrap(); 169 | let r = test_rdma(true, &mut socket); 170 | match &r { 171 | Ok(_) => println!("RDMA sent!"), 172 | Err(e) => { 173 | println!("Send failed: {}", e); 174 | // Exit will leak memory. This should not matter. 175 | std::process::exit(1) 176 | } 177 | } 178 | return r; 179 | } else if matches.is_present("receiver") { 180 | let mut socket = std::net::TcpListener::bind(&port)?.accept()?.0; 181 | let r = test_rdma(false, &mut socket); 182 | match &r { 183 | Ok(_) => println!("RDMA received!"), 184 | Err(e) => { 185 | println!("Recieve failed: {}", e); 186 | std::process::exit(1) 187 | } 188 | } 189 | return r; 190 | } else { 191 | let port = matches.value_of(IP_COMMAND).unwrap_or(DEFAULT_IP); 192 | let sender = std::process::Command::new(std::env::args().next().unwrap()) 193 | .arg("test") 194 | .arg("--sender") 195 | .arg("--ip") 196 | .arg(port) 197 | .spawn()?; 198 | let reciever = std::process::Command::new(std::env::args().next().unwrap()) 199 | .arg("test") 200 | .arg("--receiver") 201 | .arg("--ip") 202 | .arg(port) 203 | .spawn()?; 204 | let sender = sender.wait_with_output()?; 205 | let reciever = reciever.wait_with_output()?; 206 | if sender.status.success() && reciever.status.success() { 207 | return Ok(()); 208 | } else { 209 | std::process::exit(1); 210 | } 211 | } 212 | } 213 | 214 | fn connect_port(port: &T) -> io::Result 215 | where 216 | T: ToSocketAddrs, 217 | { 218 | let mut con; 219 | loop { 220 | con = std::net::TcpStream::connect(&port); 221 | let mut timeout = 2; 222 | match con { 223 | Ok(_) => break, 224 | Err(e) => match e.kind() { 225 | io::ErrorKind::TimedOut => { 226 | eprintln!("Connection timed out, retrying"); 227 | } 228 | io::ErrorKind::ConnectionRefused => { 229 | eprintln!("Connection refused: retrying"); 230 | timeout = 1; 231 | } 232 | _ => { 233 | eprintln!("Failed to connect: {}", e); 234 | return Err(e); 235 | } 236 | }, 237 | } 238 | std::thread::sleep(std::time::Duration::from_secs(timeout)); 239 | } 240 | con 241 | } 242 | 243 | fn handle_remote(matches: &ArgMatches) -> io::Result<()> { 244 | let port = 245 | std::net::SocketAddr::from_str(matches.value_of(IP_COMMAND).unwrap_or(DEFAULT_IP)).unwrap(); 246 | if let Some(mountpoint) = matches 247 | .value_of_lossy("host") 248 | .map(|s| PathBuf::from_str(&s).unwrap()) 249 | { 250 | let con = std::net::TcpListener::bind(&port)?.accept()?.0; 251 | let mut con = RDMAConnection::new(RDMA_MESSAGE_BUFFER_SIZE, con)?; 252 | remote_server(mountpoint, &mut con)?; 253 | } else { 254 | let con = connect_port(&port)?; 255 | let mountpoint = matches 256 | .value_of_lossy("client") 257 | .map(|s| PathBuf::from_str(&s).unwrap()) 258 | .unwrap(); 259 | // We are the client 260 | let con = RDMAFs::new(con)?; 261 | let backround = spawn_mount(con, mountpoint, &[]).unwrap(); 262 | let mut s = String::new(); 263 | println!("Return on input"); 264 | stdin().read_line(&mut s).expect("Failed to read input"); 265 | drop(backround); 266 | } 267 | Ok(()) 268 | } 269 | -------------------------------------------------------------------------------- /src/remote.rs: -------------------------------------------------------------------------------- 1 | use crate::file::{FileBuilder, OpenFile}; 2 | use crate::RDMAConnection; 3 | use crate::{Fh, Ino}; 4 | use fuser::*; 5 | use fuser::{Filesystem, KernelConfig, Request}; 6 | use libc::{c_int, ENOSYS, EREMOTEIO}; 7 | use nix::{ 8 | errno::{errno, Errno}, 9 | fcntl, 10 | }; 11 | use std::{ 12 | collections::HashMap, 13 | convert::TryInto, 14 | ffi::{CStr, CString, OsStr}, 15 | fs, io, 16 | mem::size_of, 17 | os::unix::{ffi::OsStrExt, fs::MetadataExt}, 18 | path::PathBuf, 19 | sync::atomic::AtomicBool, 20 | }; 21 | 22 | const MAX_FILENAME_LENGTH: usize = 255; 23 | const READ_WRITE_BUFFER_SIZE: usize = 2usize.pow(13); 24 | 25 | pub const RDMA_MESSAGE_BUFFER_SIZE: usize = size_of::(); 26 | 27 | #[repr(C)] 28 | struct BufferLayout { 29 | tag: Message, 30 | payload: MessagePayload, 31 | } 32 | 33 | macro_rules! exchange { 34 | ($msg: ident, $name: ident, $load: expr, $con: expr) => { 35 | *$con.tag() = Message::$msg; 36 | $con.payload().$name = $load; 37 | exchange!($msg, $con); 38 | }; 39 | ($type: ident, $con: expr) => { 40 | unsafe { 41 | let size = payload_size::<$type>($con.connection.as_mut_ptr()); 42 | $con.connection.send_sized(size_of::()).unwrap(); 43 | $con.connection 44 | .send_sized(size) 45 | .map_err(|e| e.raw_os_error().unwrap_or(EREMOTEIO)) 46 | .unwrap(); 47 | $con.connection 48 | .recv_sized(size) 49 | .map_err(|e| e.raw_os_error().unwrap_or(EREMOTEIO)) 50 | .unwrap(); 51 | } 52 | }; 53 | } 54 | 55 | fn payload_size(buf: *mut u8) -> usize { 56 | // In a perfect world, this would be const 57 | let base = buf as usize; 58 | let payload_offset = get_payload(buf) as *mut _ as *const u8 as usize; 59 | let offset: usize = payload_offset - base; 60 | offset + size_of::() 61 | } 62 | 63 | fn get_payload(buf: *mut u8) -> &'static mut MessagePayload { 64 | unsafe { &mut (*(buf as *mut BufferLayout)).payload } 65 | } 66 | 67 | fn get_tag(buf: *mut u8) -> &'static mut Message { 68 | unsafe { &mut (*(buf as *mut BufferLayout)).tag } 69 | } 70 | 71 | #[derive(Clone, Copy, PartialEq)] 72 | struct GetAttr { 73 | errno: Option, 74 | ino: Ino, 75 | attr: Option, 76 | } 77 | 78 | #[derive(Clone, Copy, PartialEq)] 79 | struct Lookup { 80 | errno: Option, 81 | parent: Ino, 82 | name: [u8; MAX_FILENAME_LENGTH], 83 | attr: Option, 84 | generation: u64, 85 | } 86 | 87 | #[derive(Clone, Copy, PartialEq)] 88 | struct Startup { 89 | server: bool, 90 | } 91 | 92 | #[derive(Clone, Copy, PartialEq)] 93 | struct OpenDir { 94 | errno: Option, 95 | ino: Ino, 96 | flags: i32, 97 | fh: Fh, 98 | open_flags: u32, 99 | } 100 | 101 | #[derive(Clone, Copy, PartialEq)] 102 | struct ReadDir { 103 | ino: Ino, 104 | fh: Fh, 105 | finished: bool, 106 | errno: Option, 107 | buf_ino: Ino, 108 | offset: i64, 109 | kind: FileType, 110 | name: [u8; MAX_FILENAME_LENGTH], 111 | } 112 | 113 | #[derive(Clone, Copy, PartialEq)] 114 | struct ReleaseDir { 115 | fh: Fh, 116 | errno: Option, 117 | } 118 | 119 | #[derive(Clone, Copy, PartialEq)] 120 | struct Open { 121 | errno: Option, 122 | ino: Ino, 123 | flags: i32, 124 | fh: Fh, 125 | open_flags: u32, 126 | } 127 | 128 | #[derive(Clone, Copy, PartialEq)] 129 | struct Release { 130 | errno: Option, 131 | ino: Ino, 132 | fh: Fh, 133 | } 134 | 135 | #[derive(Clone, Copy, PartialEq)] 136 | struct Flush { 137 | errno: Option, 138 | fh: Fh, 139 | } 140 | 141 | #[derive(Clone, Copy, PartialEq)] 142 | struct LSeek { 143 | errno: Option, 144 | fh: Fh, 145 | offset: i64, 146 | whence: i32, 147 | } 148 | 149 | #[derive(Clone, Copy, PartialEq)] 150 | struct Create { 151 | errno: Option, 152 | parent: Ino, 153 | name: [u8; MAX_FILENAME_LENGTH], 154 | flags: i32, 155 | attr: Option, 156 | generation: u64, 157 | fh: Fh, 158 | open_flags: u32, 159 | } 160 | 161 | #[derive(Clone, Copy, PartialEq)] 162 | struct Mkdir { 163 | errno: Option, 164 | parent: Ino, 165 | name: [u8; MAX_FILENAME_LENGTH], 166 | mode: u32, 167 | umask: u32, 168 | attr: Option, 169 | generation: u64, 170 | } 171 | 172 | #[derive(Clone, Copy, PartialEq)] 173 | struct Unlink { 174 | errno: Option, 175 | parent: Ino, 176 | name: [u8; MAX_FILENAME_LENGTH], 177 | } 178 | 179 | #[derive(Clone, Copy, PartialEq)] 180 | struct Rmdir { 181 | errno: Option, 182 | parent: Ino, 183 | name: [u8; MAX_FILENAME_LENGTH], 184 | } 185 | 186 | #[derive(Clone, Copy, PartialEq)] 187 | struct Rename { 188 | errno: Option, 189 | parent: Ino, 190 | name: [u8; MAX_FILENAME_LENGTH], 191 | newparent: Ino, 192 | newname: [u8; MAX_FILENAME_LENGTH], 193 | } 194 | 195 | #[derive(Clone, Copy, PartialEq)] 196 | struct Read { 197 | errno: Option, 198 | fh: Fh, 199 | offset: i64, 200 | size: u32, 201 | buf: [u8; READ_WRITE_BUFFER_SIZE], 202 | } 203 | 204 | #[derive(Clone, Copy, PartialEq)] 205 | struct Write { 206 | errno: Option, 207 | fh: Fh, 208 | offset: i64, 209 | data: [u8; READ_WRITE_BUFFER_SIZE], 210 | /// When a request is made, `written` holds the number of bytes to 211 | /// write. A reply contains the number of bytes written. 212 | written: u32, 213 | } 214 | 215 | #[derive(Clone, Copy, PartialEq)] 216 | struct FAllocate { 217 | errno: Option, 218 | fh: u64, 219 | offset: i64, 220 | length: i64, 221 | mode: i32, 222 | } 223 | 224 | #[derive(Clone, Copy)] 225 | #[repr(C)] 226 | union MessagePayload { 227 | startup: Startup, 228 | lookup: Lookup, 229 | get_attr: GetAttr, 230 | open_dir: OpenDir, 231 | read_dir: ReadDir, 232 | release_dir: ReleaseDir, 233 | open: Open, 234 | release: Release, 235 | read: Read, 236 | write: Write, 237 | flush: Flush, 238 | l_seek: LSeek, 239 | create: Create, 240 | mkdir: Mkdir, 241 | unlink: Unlink, 242 | rmdir: Rmdir, 243 | rename: Rename, 244 | fallocate: FAllocate, 245 | null: (), 246 | } 247 | 248 | impl Default for MessagePayload { 249 | fn default() -> Self { 250 | Self { null: () } 251 | } 252 | } 253 | 254 | /// The commands that an `RDMAFs` can issue to the server. Each command contains 255 | /// the information necessary for both a request and a reply. The client loads 256 | /// the request fields, and gets back a fully filled out reply. 257 | #[derive(Clone, Copy, PartialEq)] 258 | enum Message { 259 | // This way a default (zerod) buffer contains null. 260 | Null = 0, 261 | Exit, 262 | Startup, 263 | Lookup, 264 | GetAttr, 265 | OpenDir, 266 | ReadDir, 267 | ReleaseDir, 268 | Open, 269 | Release, 270 | Read, 271 | Write, 272 | Flush, 273 | LSeek, 274 | Create, 275 | Mkdir, 276 | Unlink, 277 | Rmdir, 278 | Rename, 279 | FAllocate, 280 | } 281 | 282 | impl Default for Message { 283 | fn default() -> Self { 284 | Message::Null 285 | } 286 | } 287 | 288 | pub(crate) static EXIT: AtomicBool = AtomicBool::new(false); 289 | 290 | /// A blocking call to the main event loop of the RDMA server. 291 | pub fn remote_server(root: PathBuf, connection: &mut RDMAConnection) -> io::Result<()> { 292 | assert_eq!( 293 | connection.len(), 294 | RDMA_MESSAGE_BUFFER_SIZE, 295 | "We need a correctly sized RDMA buffer" 296 | ); 297 | let mut tag = get_tag(connection.as_mut_ptr()); 298 | *tag = Message::Startup; 299 | let mut payload = get_payload(connection.as_mut_ptr()); 300 | payload.startup = Startup { server: true }; 301 | connection.send().unwrap(); 302 | connection.recv().unwrap(); 303 | assert!( 304 | Message::Startup == *tag, 305 | "Failed startup handshake (server)" 306 | ); 307 | 308 | assert!( 309 | Startup { server: false } == unsafe { payload.startup }, 310 | "Failed startup handshake (server)" 311 | ); 312 | println!("Handshake with the server completed"); 313 | 314 | let mut data = LocalData::new(root); 315 | loop { 316 | if EXIT.load(std::sync::atomic::Ordering::Relaxed) { 317 | eprintln!("Exiting server"); 318 | *tag = Message::Exit; 319 | connection.send().unwrap(); 320 | return Ok(()); 321 | } 322 | 323 | unsafe { 324 | connection.recv_sized(size_of::())?; 325 | } 326 | tag = get_tag(connection.as_mut_ptr()); 327 | macro_rules! recv { 328 | ($load: ident) => {{ 329 | let size = payload_size::<$load>(connection.as_mut_ptr()); 330 | unsafe { connection.recv_sized(size)? } 331 | }}; 332 | } 333 | match *tag { 334 | Message::Null | Message::Exit => {} 335 | Message::Startup => recv!(Startup), 336 | Message::Lookup => recv!(Lookup), 337 | Message::GetAttr => recv!(GetAttr), 338 | Message::OpenDir => recv!(OpenDir), 339 | Message::ReadDir => recv!(ReadDir), 340 | Message::ReleaseDir => recv!(ReleaseDir), 341 | Message::Open => recv!(Open), 342 | Message::Release => recv!(Release), 343 | Message::Read => recv!(Read), 344 | Message::Write => recv!(Write), 345 | Message::Flush => recv!(Flush), 346 | Message::LSeek => recv!(LSeek), 347 | Message::Create => recv!(Create), 348 | Message::Mkdir => recv!(Mkdir), 349 | Message::Unlink => recv!(Unlink), 350 | Message::Rmdir => recv!(Rmdir), 351 | Message::Rename => recv!(Rename), 352 | Message::FAllocate => recv!(FAllocate), 353 | } 354 | payload = get_payload(connection.as_mut_ptr()); 355 | macro_rules! send { 356 | ($load: ident) => {{ 357 | let size = payload_size::<$load>(connection.as_mut_ptr()); 358 | unsafe { connection.send_sized(size)? } 359 | }}; 360 | } 361 | match tag { 362 | Message::Exit => { 363 | println!("Recieved exit command. Goodbye!"); 364 | return Ok(()); 365 | } 366 | Message::Startup => { 367 | unsafe { assert!(!payload.startup.server) }; 368 | println!("Recieved unexpected startup command."); 369 | send!(Startup); 370 | } 371 | Message::Null => {} 372 | Message::Lookup => { 373 | let Lookup { 374 | parent, 375 | name, 376 | attr, 377 | generation, 378 | errno, 379 | } = unsafe { &mut payload.lookup }; 380 | match data.lookup(*parent, name) { 381 | Ok((fattr, gen)) => { 382 | *attr = Some(fattr); 383 | *generation = gen; 384 | } 385 | Err(e) => *errno = Some(e), 386 | } 387 | send!(Lookup); 388 | } 389 | Message::GetAttr => { 390 | let GetAttr { errno, ino, attr } = unsafe { &mut payload.get_attr }; 391 | match data.getattr(*ino) { 392 | Ok(k) => { 393 | *attr = Some(k); 394 | *errno = None; 395 | } 396 | Err(e) => { 397 | *errno = Some(e); 398 | *attr = None; 399 | } 400 | } 401 | send!(GetAttr); 402 | } 403 | Message::OpenDir => { 404 | let OpenDir { 405 | ino, 406 | flags, 407 | fh, 408 | open_flags, 409 | errno, 410 | } = unsafe { &mut payload.open_dir }; 411 | match data.opendir(*ino, *flags) { 412 | Ok((file_handle, flags)) => { 413 | *open_flags = flags; 414 | *fh = file_handle; 415 | *errno = None; 416 | } 417 | Err(e) => *errno = Some(e), 418 | } 419 | send!(OpenDir); 420 | } 421 | Message::ReleaseDir => { 422 | let ReleaseDir { fh, errno } = unsafe { &mut payload.release_dir }; 423 | *errno = data.releasedir(*fh).err(); 424 | send!(ReleaseDir); 425 | } 426 | 427 | Message::ReadDir => { 428 | let ReadDir { 429 | ino, 430 | fh, 431 | errno, 432 | buf_ino, 433 | offset, 434 | kind, 435 | name, 436 | finished, 437 | } = unsafe { &mut payload.read_dir }; 438 | match data.readdir(*ino, *fh) { 439 | Ok(Some((r_ino, r_offset, r_kind, r_name))) => { 440 | *errno = None; 441 | *buf_ino = r_ino; 442 | *offset = r_offset; 443 | *kind = r_kind; 444 | *name = r_name; 445 | } 446 | Ok(None) => *finished = true, 447 | Err(e) => *errno = Some(e), 448 | } 449 | send!(ReadDir); 450 | } 451 | 452 | Message::Open => { 453 | let Open { 454 | errno, 455 | ino, 456 | flags, 457 | fh, 458 | open_flags, 459 | } = unsafe { &mut payload.open }; 460 | match data.open(*ino, *flags) { 461 | Ok((file_handle, flags)) => { 462 | *errno = None; 463 | *fh = file_handle; 464 | *open_flags = flags; 465 | } 466 | Err(e) => *errno = Some(e), 467 | } 468 | send!(Open); 469 | } 470 | 471 | Message::Release => { 472 | let Release { errno, ino, fh } = unsafe { &mut payload.release }; 473 | *errno = data.release(*ino, *fh).err(); 474 | send!(Release); 475 | } 476 | 477 | Message::Read => { 478 | let Read { 479 | errno, 480 | fh, 481 | offset, 482 | size, 483 | buf, 484 | } = unsafe { &mut payload.read }; 485 | match data.read(*fh, *offset, *size, buf) { 486 | Ok(bytes_read) => *size = bytes_read as _, 487 | Err(e) => *errno = Some(e), 488 | } 489 | send!(Read); 490 | } 491 | 492 | Message::Write => { 493 | let Write { 494 | errno, 495 | fh, 496 | offset, 497 | data: buf, 498 | written, 499 | } = unsafe { &mut payload.write }; 500 | match data.write(*fh, *offset, &buf[..*written as _]) { 501 | Ok(k) => { 502 | *errno = None; 503 | *written = k; 504 | } 505 | Err(e) => *errno = Some(e), 506 | } 507 | send!(Write); 508 | } 509 | 510 | Message::Flush => { 511 | let Flush { errno, fh } = unsafe { &mut payload.flush }; 512 | *errno = data.flush(*fh).err(); 513 | send!(Flush); 514 | } 515 | 516 | Message::LSeek => { 517 | let LSeek { 518 | errno, 519 | fh, 520 | offset, 521 | whence, 522 | } = unsafe { &mut payload.l_seek }; 523 | match data.lseek(*fh, *offset, *whence) { 524 | Ok(k) => *offset = k, 525 | Err(e) => *errno = Some(e), 526 | } 527 | send!(LSeek); 528 | } 529 | 530 | Message::Create => { 531 | let Create { 532 | errno, 533 | parent, 534 | name, 535 | flags, 536 | attr, 537 | generation, 538 | fh, 539 | open_flags, 540 | } = unsafe { &mut payload.create }; 541 | 542 | match data.create(*parent, buf_to_osstr(name), *flags) { 543 | Ok((f_attr, gen, new_fh, flags)) => { 544 | *attr = Some(f_attr); 545 | *generation = gen; 546 | *fh = new_fh; 547 | *open_flags = flags; 548 | } 549 | Err(e) => *errno = Some(e), 550 | } 551 | send!(Create); 552 | } 553 | 554 | Message::Mkdir => { 555 | let Mkdir { 556 | errno, 557 | parent, 558 | name, 559 | mode, 560 | umask, 561 | attr, 562 | generation, 563 | } = unsafe { &mut payload.mkdir }; 564 | match data.mkdir(*parent, buf_to_osstr(name), *mode, *umask) { 565 | Ok((f_attr, f_gen)) => { 566 | *attr = Some(f_attr); 567 | *generation = f_gen; 568 | } 569 | Err(e) => *errno = Some(e), 570 | } 571 | send!(Mkdir); 572 | } 573 | 574 | Message::Unlink => { 575 | let Unlink { 576 | errno, 577 | parent, 578 | name, 579 | } = unsafe { &mut payload.unlink }; 580 | *errno = data.unlink(*parent, buf_to_osstr(name)).err(); 581 | send!(Unlink); 582 | } 583 | 584 | Message::Rmdir => { 585 | let Rmdir { 586 | errno, 587 | parent, 588 | name, 589 | } = unsafe { &mut payload.rmdir }; 590 | *errno = data.rmdir(*parent, buf_to_osstr(name)).err(); 591 | send!(Rmdir); 592 | } 593 | 594 | Message::Rename => { 595 | let Rename { 596 | errno, 597 | parent, 598 | name, 599 | newparent, 600 | newname, 601 | } = unsafe { &mut payload.rename }; 602 | *errno = data 603 | .rename( 604 | *parent, 605 | buf_to_osstr(name), 606 | *newparent, 607 | buf_to_osstr(newname), 608 | ) 609 | .err(); 610 | send!(Rename); 611 | } 612 | 613 | Message::FAllocate => { 614 | let FAllocate { 615 | errno, 616 | fh, 617 | offset, 618 | length, 619 | mode, 620 | } = unsafe { &mut payload.fallocate }; 621 | *errno = data 622 | .fallocate(*fh as _, *mode as _, *offset, *length as _) 623 | .err(); 624 | send!(FAllocate); 625 | } 626 | } 627 | } 628 | } 629 | 630 | pub struct RDMAFs { 631 | connection: RDMAConnection, 632 | initialized: bool, 633 | } 634 | 635 | /// Stores the data necessary for acting like a file system. 636 | pub struct LocalData { 637 | /// Where the root of `Mount` is located in the host file system. 638 | root: PathBuf, 639 | /// The ino of the root of the reflected file system. 640 | root_ino: Ino, 641 | ino_paths: HashMap, 642 | open_files: HashMap, 643 | } 644 | 645 | /// These are the local side operations that correspond to `local.rs`. 646 | impl LocalData { 647 | pub fn new(root: PathBuf) -> Self { 648 | let root = root.canonicalize().unwrap(); 649 | let root_data = fs::metadata(&root).unwrap(); 650 | let root_ino = root_data.ino(); 651 | let ino_paths: HashMap = std::iter::once((root_ino, root.clone())).collect(); 652 | Self { 653 | root, 654 | root_ino, 655 | ino_paths, 656 | open_files: HashMap::new(), 657 | } 658 | } 659 | 660 | fn fallocate(&mut self, fh: Fh, offset: i64, length: i64, mode: i32) -> Result<(), i32> { 661 | if let Some(file) = self.open_files.get_mut(&fh) { 662 | // There doesn't seem to be a specific flag for this. 663 | let mode = fcntl::FallocateFlags::from_bits(mode).ok_or(libc::EINVAL)?; 664 | match fcntl::fallocate(file.fh() as _, mode, offset as _, length as _) { 665 | Ok(_) => Ok(()), 666 | Err(e) => Err(e 667 | .as_errno() 668 | .map(|e| { 669 | io::Error::from(e).raw_os_error().expect( 670 | "This should be a valid Linux Errno, as it was taken from `nix`", 671 | ) 672 | }) 673 | .expect("Valid Linux Errno because it is derived from a `nix` errno")), 674 | } 675 | } else { 676 | Err(libc::ENOENT) 677 | } 678 | } 679 | 680 | /// Rename a file. 681 | fn rename( 682 | &mut self, 683 | parent: Ino, 684 | name: &OsStr, 685 | newparent: Ino, 686 | newname: &OsStr, 687 | ) -> Result<(), i32> { 688 | // TODO: revalidate the [ino <-> path] map 689 | let old_name = CString::new(name.as_bytes()).unwrap(); 690 | let newname = CString::new(newname.as_bytes()).unwrap(); 691 | let res = unsafe { 692 | libc::renameat( 693 | parent as _, 694 | old_name.as_ptr(), 695 | newparent as _, 696 | newname.as_ptr(), 697 | ) 698 | }; 699 | match res { 700 | 0 => Ok(()), 701 | _ => Err(errno()), 702 | } 703 | } 704 | 705 | /// Remove a directory. 706 | fn rmdir(&mut self, parent: Ino, name: &OsStr) -> Result<(), i32> { 707 | let path = self.at_ino(&parent).ok_or(libc::ENOENT)?.join(name); 708 | let path = CString::new(path.as_os_str().as_bytes()).unwrap(); 709 | let res = unsafe { libc::rmdir(path.as_ptr()) }; 710 | match res { 711 | 0 => Ok(()), 712 | _ => Err(errno()), 713 | } 714 | } 715 | 716 | /// Remove a file. 717 | fn unlink(&mut self, parent: u64, name: &OsStr) -> Result<(), i32> { 718 | if let Some(parent) = self.at_ino(&parent) { 719 | let fname = CString::new(parent.join(name).as_os_str().as_bytes()).unwrap(); 720 | let res = unsafe { libc::unlink(fname.as_ptr()) }; 721 | if res == 0 { 722 | Ok(()) 723 | } else { 724 | Err(errno()) 725 | } 726 | } else { 727 | log::error!("Unlink failed on invalid parent ino {:?}", parent); 728 | Err(libc::ENOENT) 729 | } 730 | } 731 | 732 | /// Create a directory. 733 | fn mkdir( 734 | &mut self, 735 | parent: Ino, 736 | name: &OsStr, 737 | mode: u32, 738 | umask: u32, 739 | ) -> Result<(FileAttr, u64), i32> { 740 | let res: Result = try { 741 | let name = self.at_ino(&parent).ok_or(libc::ENOENT)?.join(name); 742 | let c_name = CString::new(name.as_os_str().as_bytes()).unwrap(); 743 | // NOTE: unsure about the bit-and 744 | let res = unsafe { libc::mkdir(c_name.as_ptr(), (mode & umask) as _) }; 745 | match res { 746 | 0 => fs::metadata(&name).as_ref().unwrap().try_into().unwrap(), 747 | _ => Err(errno())?, 748 | } 749 | }; 750 | res.map(|k| (k, 0)) 751 | } 752 | 753 | fn create( 754 | &mut self, 755 | parent: Ino, 756 | name: &OsStr, 757 | flags: i32, 758 | ) -> Result<(FileAttr, u64, Fh, u32), i32> { 759 | if let Some(parent) = self.at_ino(&parent) { 760 | let path = parent.join(name); 761 | log::info!("create called on file {:?}", path); 762 | let file = match FileBuilder::from_flags(flags) 763 | .create(true) 764 | .read(true) 765 | .write(true) 766 | .path(&path) 767 | { 768 | Ok(k) => k, 769 | Err(e) => { 770 | log::error!("Failed to create file {:?} with error: {}", path, e); 771 | return Err(e.raw_os_error().unwrap()); 772 | } 773 | }; 774 | let meta = file.metadata().expect("already called, so should work"); 775 | self.ino_paths.insert(meta.ino(), path); 776 | let fd = self.register_new_file(file); 777 | Ok(((&meta).into(), 0, fd as _, flags as _)) 778 | } else { 779 | log::error!("Failed to create file {:?} with errno {:?}", name, errno()); 780 | Err(errno()) 781 | } 782 | } 783 | 784 | /// Reposition read/write file offset 785 | fn lseek(&mut self, fh: u64, offset: i64, whence: i32) -> Result { 786 | let offset = unsafe { libc::lseek(fh as _, offset, whence) }; 787 | if offset == -1 { 788 | log::error!("lseek failed with error"); 789 | Err(errno()) 790 | } else { 791 | Ok(offset) 792 | } 793 | } 794 | 795 | fn flush(&mut self, fh: Fh) -> Result<(), i32> { 796 | if let Some(f) = self.open_files.get_mut(&fh) { 797 | if let Some(r) = f.flush().err() { 798 | Err(r.raw_os_error().unwrap()) 799 | } else { 800 | Ok(()) 801 | } 802 | } else { 803 | Err(libc::ENOENT) 804 | } 805 | } 806 | 807 | fn write(&mut self, fh: Fh, offset: i64, data: &[u8]) -> Result { 808 | log::info!("Attempting to write to file at handle: {:?}", fh); 809 | if let Some(f) = self.open_files.get_mut(&fh) { 810 | match f.write(data, offset) { 811 | Ok(len) => Ok(len as _), 812 | Err(e) => { 813 | log::error!("Failed to wtite bytes: {}", e); 814 | Err(e.raw_os_error().unwrap()) 815 | } 816 | } 817 | } else { 818 | log::error!( 819 | "Failed to write bytes to fh {:?}, could not find open file. There are {} open files.", 820 | fh, self.open_files.keys().len() 821 | ); 822 | Err(0) 823 | } 824 | } 825 | 826 | fn read( 827 | &mut self, 828 | fh: u64, 829 | offset: i64, 830 | size: u32, 831 | buf: &mut [u8; READ_WRITE_BUFFER_SIZE], 832 | ) -> Result { 833 | let size = size as usize; 834 | assert!(size <= buf.len()); 835 | if let Some(f) = self.open_files.get_mut(&(fh as _)) { 836 | match f.read(&mut *buf, offset, size) { 837 | Ok(bytes_read) => { 838 | log::info!( 839 | "Read {} ({} requested) bytes into file {}", 840 | bytes_read, 841 | size, 842 | fh 843 | ); 844 | Ok(bytes_read) 845 | } 846 | Err(e) => Err(e.raw_os_error().unwrap_or(1) as _), 847 | } 848 | } else { 849 | Err(libc::EBADF) 850 | } 851 | } 852 | 853 | fn release(&mut self, ino: u64, fh: u64) -> Result<(), i32> { 854 | log::trace!("Released file {:?} with fh {:?}", self.at_ino(&ino), fh); 855 | self.open_files.remove(&fh); 856 | Ok(()) 857 | } 858 | 859 | fn open(&mut self, ino: Ino, flags: i32) -> Result<(Fh, u32), i32> { 860 | if let Some(path) = self.at_ino(&ino) { 861 | match FileBuilder::from_flags(flags) 862 | .read(true) 863 | .write(true) 864 | .path(path) 865 | { 866 | Ok(file) => { 867 | let fh = self.register_new_file(file); 868 | Ok((fh as _, flags as _)) 869 | } 870 | Err(e) => Err(e.raw_os_error().unwrap_or(libc::EIO)), 871 | } 872 | } else { 873 | log::error!("Open failed with invalid ino {:?}", ino); 874 | Err(libc::EIO) 875 | } 876 | } 877 | 878 | fn readdir( 879 | &mut self, 880 | ino: Ino, 881 | fh: Fh, 882 | ) -> Result, i32> { 883 | Errno::clear(); // Because it's not clear if readdir failed from it's output 884 | let dir_ent = unsafe { libc::readdir(fh as _) }; 885 | if dir_ent.is_null() { 886 | use libc::*; 887 | match errno() { 888 | EACCES | EBADF | EMFILE | ENFILE | ENOENT | ENOMEM | ENOTDIR => { 889 | log::error!( 890 | "Encountered error {} reading directory {:?}, fh {:?}", 891 | std::io::Error::from_raw_os_error(errno()), 892 | self.at_ino(&ino) 893 | .map(|b| b.as_os_str()) 894 | .unwrap_or_else(|| OsStr::new("Unknown")), 895 | fh 896 | ); 897 | return Err(errno()); 898 | } 899 | _ => return Ok(None), 900 | } 901 | } 902 | let dir_ent = unsafe { *dir_ent }; 903 | let file_len = unsafe { CStr::from_ptr(dir_ent.d_name.as_ptr() as _) } 904 | .to_bytes() 905 | .len(); 906 | // We need to allow an extra step to convert u8 to i8 on some machines 907 | let file_buf = unsafe { &*(&dir_ent.d_name[..file_len] as *const [_] as *const [u8]) }; 908 | let file = buf_to_osstr(file_buf); 909 | 910 | log::trace!("File {:?} under dir {:?}", file, ino); 911 | 912 | let seek = dir_ent.d_off; 913 | 914 | self.ino_paths.insert( 915 | dir_ent.d_ino, 916 | self.at_ino(&ino).expect("Valid ino number").join(file), 917 | ); 918 | let mut buf = [0; MAX_FILENAME_LENGTH]; 919 | &mut buf[..file_buf.len()].copy_from_slice(file_buf); 920 | Ok(Some(( 921 | dir_ent.d_ino, 922 | seek, 923 | dir_ent.d_type.try_into().expect("Unknown file type"), 924 | buf, 925 | ))) 926 | } 927 | 928 | fn releasedir(&mut self, fh: u64) -> Result<(), i32> { 929 | let res = unsafe { libc::closedir(fh as _) }; 930 | if res == 0 { 931 | Ok(()) 932 | } else { 933 | Err(errno()) 934 | } 935 | } 936 | 937 | fn opendir(&mut self, ino: Ino, flags: i32) -> Result<(Fh, u32), i32> { 938 | let buf = if let Some(buf) = self.at_ino(&ino) { 939 | CString::new(buf.as_os_str().as_bytes()).expect("buf should not contain a null pointer") 940 | } else { 941 | log::error!("opendir: Invalid ino {:?}", ino); 942 | return Err(0); 943 | }; 944 | let res = unsafe { libc::opendir(buf.as_ptr() as _) }; 945 | if res.is_null() { 946 | log::error!("opendir: libc call failed with ERRNO=?"); 947 | Err(errno()) 948 | } else { 949 | Ok((res as u64, flags as u32)) 950 | } 951 | } 952 | 953 | fn at_ino(&self, ino: &Ino) -> Option<&PathBuf> { 954 | if *ino == 1 || *ino == self.root_ino { 955 | Some(&self.root) 956 | } else { 957 | self.ino_paths.get(ino) 958 | } 959 | } 960 | 961 | fn register_new_file(&mut self, file: OpenFile) -> Fh { 962 | let fh = file.fh(); 963 | self.open_files.entry(fh).or_insert(file); 964 | fh 965 | } 966 | 967 | fn lookup(&mut self, parent: Ino, name: &[u8]) -> Result<(FileAttr, u64), i32> { 968 | let name = buf_to_osstr(name); 969 | let parent = if let Some(k) = self.at_ino(&parent) { 970 | k 971 | } else { 972 | log::error!( 973 | "Attempted lookup of parrent ino {:?}. File not found.", 974 | parent 975 | ); 976 | return Err(libc::ENOENT); 977 | }; 978 | let new_file = parent.join(name); 979 | let data = if let Ok(k) = fs::metadata(&new_file) { 980 | k 981 | } else { 982 | return Err(libc::ENOENT); 983 | }; 984 | log::trace!( 985 | "Performed lookup on {:?} with parrent {:?}. Found new Ino {:?}", 986 | name, 987 | parent, 988 | new_file 989 | ); 990 | self.ino_paths.insert(data.ino(), new_file); 991 | Ok(((&data).into(), 0)) 992 | } 993 | 994 | fn getattr(&mut self, ino: Ino) -> Result { 995 | let handle; 996 | let buf = if ino == 1 { 997 | &self.root 998 | } else { 999 | handle = self.root.join(self.at_ino(&ino).unwrap()); 1000 | &handle 1001 | }; 1002 | if let Ok(k) = fs::metadata(buf) { 1003 | log::trace!("Replied with metadata of file {:?}", buf); 1004 | Ok((&k).into()) 1005 | } else { 1006 | log::error!("Failed lookup on ino {:?} = {:?}", ino, buf); 1007 | Err(libc::ENOENT) 1008 | } 1009 | } 1010 | } 1011 | 1012 | impl RDMAFs { 1013 | pub fn new(connection: W) -> io::Result 1014 | where 1015 | W: io::Read + io::Write, 1016 | { 1017 | Ok(Self { 1018 | connection: RDMAConnection::new(RDMA_MESSAGE_BUFFER_SIZE, connection)?, 1019 | initialized: false, 1020 | }) 1021 | } 1022 | 1023 | fn tag<'a>(&'a mut self) -> &'a mut Message { 1024 | get_tag(self.connection.as_mut_ptr()) 1025 | } 1026 | fn payload<'a>(&'a mut self) -> &'a mut MessagePayload { 1027 | get_payload(self.connection.as_mut_ptr()) 1028 | } 1029 | } 1030 | 1031 | impl Drop for RDMAFs { 1032 | fn drop(&mut self) { 1033 | if self.initialized { 1034 | *self.tag() = Message::Exit; 1035 | unsafe { self.connection.send_sized(size_of::()).unwrap() }; 1036 | } 1037 | } 1038 | } 1039 | impl Filesystem for RDMAFs { 1040 | fn init(&mut self, _req: &Request<'_>, config: &mut KernelConfig) -> Result<(), c_int> { 1041 | config.set_max_write(READ_WRITE_BUFFER_SIZE as _).unwrap(); 1042 | self.connection 1043 | .recv() 1044 | .map_err(|e| e.raw_os_error().unwrap_or(EREMOTEIO))?; 1045 | assert!( 1046 | *self.tag() == Message::Startup, 1047 | "Failed startup handshake (client)" 1048 | ); 1049 | assert!( 1050 | unsafe { self.payload().startup } == Startup { server: true }, 1051 | "Failed startup handshake (client)(payload)" 1052 | ); 1053 | *self.tag() = Message::Startup; 1054 | self.payload().startup = Startup { server: false }; 1055 | self.connection 1056 | .send() 1057 | .map_err(|e| e.raw_os_error().unwrap_or(EREMOTEIO))?; 1058 | self.initialized = true; 1059 | Ok(()) 1060 | } 1061 | 1062 | fn destroy(&mut self, _req: &Request<'_>) {} 1063 | 1064 | fn fallocate( 1065 | &mut self, 1066 | _req: &Request<'_>, 1067 | _ino: u64, 1068 | fh: u64, 1069 | offset: i64, 1070 | length: i64, 1071 | mode: i32, 1072 | reply: ReplyEmpty, 1073 | ) { 1074 | exchange! { 1075 | FAllocate, 1076 | fallocate, 1077 | FAllocate { 1078 | fh, offset, length, mode, errno: None, 1079 | }, 1080 | self 1081 | } 1082 | match *self.tag() { 1083 | Message::FAllocate => { 1084 | let errno = unsafe { &self.payload().fallocate.errno }; 1085 | if let Some(errno) = errno { 1086 | reply.error(*errno); 1087 | } else { 1088 | reply.ok(); 1089 | } 1090 | } 1091 | Message::Null => reply.error(ENOSYS), 1092 | _ => panic!("Unexpected lookup"), 1093 | } 1094 | } 1095 | 1096 | fn lookup(&mut self, _req: &Request<'_>, parent: u64, name: &OsStr, reply: ReplyEntry) { 1097 | let name = name.as_bytes(); 1098 | assert!(name.len() < MAX_FILENAME_LENGTH); 1099 | let mut buf = [0; MAX_FILENAME_LENGTH]; 1100 | &mut buf[..name.len()].copy_from_slice(name); 1101 | exchange!( 1102 | Lookup, 1103 | lookup, 1104 | Lookup { 1105 | parent, 1106 | name: buf, 1107 | attr: None, 1108 | generation: 0, 1109 | errno: None, 1110 | }, 1111 | self 1112 | ); 1113 | match *self.tag() { 1114 | Message::Lookup => { 1115 | let Lookup { 1116 | errno, 1117 | attr, 1118 | generation, 1119 | .. 1120 | } = unsafe { &self.payload().lookup }; 1121 | if let Some(errno) = *errno { 1122 | reply.error(errno); 1123 | } else { 1124 | reply.entry( 1125 | &std::time::Duration::ZERO, 1126 | &attr.expect("A reply should contain the attr (lookup)"), 1127 | *generation, 1128 | ); 1129 | } 1130 | } 1131 | Message::Null => reply.error(ENOSYS), 1132 | _ => panic!("Expected lookup"), 1133 | } 1134 | } 1135 | 1136 | fn getattr(&mut self, _req: &Request<'_>, ino: u64, reply: ReplyAttr) { 1137 | exchange!( 1138 | GetAttr, 1139 | get_attr, 1140 | GetAttr { 1141 | ino, 1142 | attr: None, 1143 | errno: None, 1144 | }, 1145 | self 1146 | ); 1147 | match *self.tag() { 1148 | Message::GetAttr => { 1149 | let GetAttr { attr, errno, .. } = unsafe { &self.payload().get_attr }; 1150 | if let Some(errno) = *errno { 1151 | reply.error(errno); 1152 | } else { 1153 | reply.attr( 1154 | &std::time::Duration::ZERO, 1155 | &attr.expect("A reply should contain the attr (getattr)"), 1156 | ) 1157 | } 1158 | } 1159 | Message::Null => reply.error(ENOSYS), 1160 | _ => panic!("Expected getattr"), 1161 | } 1162 | } 1163 | 1164 | fn setattr( 1165 | &mut self, 1166 | _req: &Request<'_>, 1167 | _ino: u64, 1168 | _mode: Option, 1169 | _uid: Option, 1170 | _gid: Option, 1171 | _size: Option, 1172 | _atime: Option, 1173 | _mtime: Option, 1174 | _ctime: Option, 1175 | _fh: Option, 1176 | _crtime: Option, 1177 | _chgtime: Option, 1178 | _bkuptime: Option, 1179 | _flags: Option, 1180 | reply: ReplyAttr, 1181 | ) { 1182 | reply.error(ENOSYS); 1183 | } 1184 | 1185 | fn readlink(&mut self, _req: &Request<'_>, _ino: u64, reply: ReplyData) { 1186 | reply.error(ENOSYS); 1187 | } 1188 | 1189 | fn mkdir( 1190 | &mut self, 1191 | _req: &Request<'_>, 1192 | parent: u64, 1193 | name: &OsStr, 1194 | mode: u32, 1195 | umask: u32, 1196 | reply: ReplyEntry, 1197 | ) { 1198 | let mut buf = [0; MAX_FILENAME_LENGTH]; 1199 | buf[0..name.len()].copy_from_slice(name.as_bytes()); 1200 | 1201 | exchange!( 1202 | Mkdir, 1203 | mkdir, 1204 | Mkdir { 1205 | errno: None, 1206 | parent, 1207 | name: buf, 1208 | mode, 1209 | umask, 1210 | attr: None, 1211 | generation: 0, 1212 | }, 1213 | self 1214 | ); 1215 | 1216 | match *self.tag() { 1217 | Message::Mkdir => { 1218 | let Mkdir { 1219 | errno, 1220 | attr, 1221 | generation, 1222 | .. 1223 | } = unsafe { self.payload().mkdir }; 1224 | if let Some(errno) = errno { 1225 | reply.error(errno); 1226 | } else { 1227 | reply.entry( 1228 | &std::time::Duration::ZERO, 1229 | &attr.expect("File attr info filled"), 1230 | generation, 1231 | ); 1232 | } 1233 | } 1234 | Message::Null => reply.error(ENOSYS), 1235 | _ => panic!("Expected open"), 1236 | } 1237 | } 1238 | 1239 | fn unlink(&mut self, _req: &Request<'_>, parent: u64, name: &OsStr, reply: ReplyEmpty) { 1240 | let mut buf = [0; MAX_FILENAME_LENGTH]; 1241 | buf[0..name.len()].copy_from_slice(name.as_bytes()); 1242 | 1243 | exchange! { 1244 | Unlink, unlink, Unlink { 1245 | errno: None, 1246 | parent, 1247 | name: buf, 1248 | }, 1249 | self 1250 | }; 1251 | 1252 | match *self.tag() { 1253 | Message::Unlink => { 1254 | let errno = unsafe { self.payload().unlink }.errno; 1255 | if let Some(errno) = errno { 1256 | reply.error(errno); 1257 | } else { 1258 | reply.ok(); 1259 | } 1260 | } 1261 | Message::Null => reply.error(ENOSYS), 1262 | _ => panic!("Expected Unlink"), 1263 | } 1264 | } 1265 | 1266 | fn rmdir(&mut self, _req: &Request<'_>, parent: u64, name: &OsStr, reply: ReplyEmpty) { 1267 | let mut buf = [0; MAX_FILENAME_LENGTH]; 1268 | buf[0..name.len()].copy_from_slice(name.as_bytes()); 1269 | exchange! { 1270 | Rmdir, rmdir, Rmdir {errno: None, parent, name: buf}, 1271 | self 1272 | }; 1273 | match *self.tag() { 1274 | Message::Rmdir => { 1275 | let errno = unsafe { self.payload().rmdir }.errno; 1276 | if let Some(errno) = errno { 1277 | reply.error(errno); 1278 | } else { 1279 | reply.ok(); 1280 | } 1281 | } 1282 | Message::Null => reply.error(ENOSYS), 1283 | _ => panic!("Expected Rmdir"), 1284 | } 1285 | } 1286 | 1287 | fn symlink( 1288 | &mut self, 1289 | _req: &Request<'_>, 1290 | _parent: u64, 1291 | _name: &OsStr, 1292 | _link: &std::path::Path, 1293 | reply: ReplyEntry, 1294 | ) { 1295 | reply.error(ENOSYS); 1296 | } 1297 | 1298 | fn rename( 1299 | &mut self, 1300 | _req: &Request<'_>, 1301 | parent: u64, 1302 | name: &OsStr, 1303 | newparent: u64, 1304 | newname: &OsStr, 1305 | _flags: u32, 1306 | reply: ReplyEmpty, 1307 | ) { 1308 | let mut buf = [0; MAX_FILENAME_LENGTH]; 1309 | buf[0..name.len()].copy_from_slice(name.as_bytes()); 1310 | let mut newbuf = [0; MAX_FILENAME_LENGTH]; 1311 | newbuf[0..newname.len()].copy_from_slice(newname.as_bytes()); 1312 | 1313 | exchange! { 1314 | Rename, 1315 | rename, 1316 | Rename { 1317 | errno: None, 1318 | parent, 1319 | name: buf, 1320 | newparent, 1321 | newname: newbuf, 1322 | }, self 1323 | }; 1324 | match *self.tag() { 1325 | Message::Rename => { 1326 | let errno = unsafe { self.payload().rename }.errno; 1327 | if let Some(errno) = errno { 1328 | reply.error(errno); 1329 | } else { 1330 | reply.ok(); 1331 | } 1332 | } 1333 | Message::Null => reply.error(ENOSYS), 1334 | _ => panic!("Expected Rename"), 1335 | } 1336 | } 1337 | 1338 | fn link( 1339 | &mut self, 1340 | _req: &Request<'_>, 1341 | _ino: u64, 1342 | _newparent: u64, 1343 | _newname: &OsStr, 1344 | reply: ReplyEntry, 1345 | ) { 1346 | reply.error(ENOSYS); 1347 | } 1348 | 1349 | fn open(&mut self, _req: &Request<'_>, ino: u64, flags: i32, reply: ReplyOpen) { 1350 | exchange!( 1351 | Open, 1352 | open, 1353 | Open { 1354 | ino, 1355 | flags, 1356 | fh: 0, 1357 | open_flags: 0, 1358 | errno: None, 1359 | }, 1360 | self 1361 | ); 1362 | match *self.tag() { 1363 | Message::Open => { 1364 | let Open { 1365 | fh, 1366 | open_flags, 1367 | errno, 1368 | .. 1369 | } = unsafe { self.payload().open }; 1370 | if let Some(errno) = errno { 1371 | reply.error(errno); 1372 | } else { 1373 | reply.opened(fh, open_flags); 1374 | } 1375 | } 1376 | Message::Null => reply.error(ENOSYS), 1377 | _ => panic!("Expected open"), 1378 | } 1379 | } 1380 | 1381 | fn read( 1382 | &mut self, 1383 | _req: &Request<'_>, 1384 | _ino: u64, 1385 | fh: u64, 1386 | offset: i64, 1387 | size: u32, 1388 | _flags: i32, 1389 | _lock_owner: Option, 1390 | reply: ReplyData, 1391 | ) { 1392 | exchange!( 1393 | Read, 1394 | read, 1395 | Read { 1396 | errno: None, 1397 | fh, 1398 | offset, 1399 | size, 1400 | buf: [0; READ_WRITE_BUFFER_SIZE], 1401 | }, 1402 | self 1403 | ); 1404 | match self.tag() { 1405 | Message::Read => { 1406 | let Read { 1407 | errno, buf, size, .. 1408 | } = unsafe { &self.payload().read }; 1409 | if let Some(errno) = *errno { 1410 | reply.error(errno); 1411 | } else { 1412 | reply.data(&buf[..*size as usize]); 1413 | } 1414 | } 1415 | Message::Null => reply.error(ENOSYS), 1416 | _ => panic!("Expected Read"), 1417 | } 1418 | } 1419 | 1420 | fn write( 1421 | &mut self, 1422 | _req: &Request<'_>, 1423 | _ino: u64, 1424 | fh: u64, 1425 | offset: i64, 1426 | data: &[u8], 1427 | _write_flags: u32, 1428 | _flags: i32, 1429 | _lock_owner: Option, 1430 | reply: ReplyWrite, 1431 | ) { 1432 | assert!(data.len() <= READ_WRITE_BUFFER_SIZE); 1433 | let mut buf = [0; READ_WRITE_BUFFER_SIZE]; 1434 | buf[0..data.len()].copy_from_slice(data); 1435 | exchange!( 1436 | Write, 1437 | write, 1438 | Write { 1439 | errno: None, 1440 | fh, 1441 | offset, 1442 | data: buf, 1443 | written: data.len() as _, 1444 | }, 1445 | self 1446 | ); 1447 | match self.tag() { 1448 | Message::Write => { 1449 | let Write { errno, written, .. } = unsafe { &self.payload().write }; 1450 | if let Some(errno) = *errno { 1451 | reply.error(errno); 1452 | } else { 1453 | reply.written(*written); 1454 | } 1455 | } 1456 | Message::Null => reply.error(ENOSYS), 1457 | _ => panic!("Expected Write"), 1458 | } 1459 | } 1460 | 1461 | fn flush( 1462 | &mut self, 1463 | _req: &Request<'_>, 1464 | _ino: u64, 1465 | fh: u64, 1466 | _lock_owner: u64, 1467 | reply: ReplyEmpty, 1468 | ) { 1469 | exchange!(Flush, flush, Flush { errno: None, fh }, self); 1470 | match *self.tag() { 1471 | Message::Flush => { 1472 | let errno = unsafe { self.payload().flush }.errno; 1473 | if let Some(errno) = errno { 1474 | reply.error(errno); 1475 | } else { 1476 | reply.ok(); 1477 | } 1478 | } 1479 | Message::Null => reply.error(ENOSYS), 1480 | _ => panic!("Expected Flush"), 1481 | } 1482 | } 1483 | 1484 | fn release( 1485 | &mut self, 1486 | _req: &Request<'_>, 1487 | ino: u64, 1488 | fh: u64, 1489 | _flags: i32, 1490 | _lock_owner: Option, 1491 | _flush: bool, 1492 | reply: ReplyEmpty, 1493 | ) { 1494 | exchange!( 1495 | Release, 1496 | release, 1497 | Release { 1498 | ino, 1499 | fh, 1500 | errno: None, 1501 | }, 1502 | self 1503 | ); 1504 | match *self.tag() { 1505 | Message::Release => { 1506 | let errno = unsafe { self.payload().release }.errno; 1507 | if let Some(errno) = errno { 1508 | reply.error(errno); 1509 | } else { 1510 | reply.ok(); 1511 | } 1512 | } 1513 | Message::Null => reply.error(ENOSYS), 1514 | _ => panic!("Expected Release"), 1515 | } 1516 | } 1517 | 1518 | fn opendir(&mut self, _req: &Request<'_>, ino: u64, flags: i32, reply: ReplyOpen) { 1519 | exchange!( 1520 | OpenDir, 1521 | open_dir, 1522 | OpenDir { 1523 | ino, 1524 | flags, 1525 | fh: 0, 1526 | open_flags: 0, 1527 | errno: None, 1528 | }, 1529 | self 1530 | ); 1531 | match *self.tag() { 1532 | Message::OpenDir => { 1533 | let OpenDir { 1534 | fh, 1535 | open_flags, 1536 | errno, 1537 | .. 1538 | } = unsafe { self.payload().open_dir }; 1539 | if let Some(errno) = errno { 1540 | reply.error(errno); 1541 | } else { 1542 | reply.opened(fh, open_flags); 1543 | } 1544 | } 1545 | Message::Null => reply.error(ENOSYS), 1546 | _ => panic!("Expected opendir"), 1547 | } 1548 | } 1549 | 1550 | // Protocal: Send a ReadDir request with correct ino, fh. We expect to 1551 | // recieve a ReadDir back, and filled out. The process is statless, at the 1552 | // communication level. We don't send any more requests when we are done 1553 | // with the `fh`. 1554 | fn readdir( 1555 | &mut self, 1556 | _req: &Request<'_>, 1557 | ino: u64, 1558 | fh: u64, 1559 | _offset: i64, 1560 | mut reply: ReplyDirectory, 1561 | ) { 1562 | *self.tag() = Message::ReadDir; 1563 | self.payload().read_dir = ReadDir { 1564 | ino, 1565 | fh, 1566 | finished: false, 1567 | errno: None, 1568 | buf_ino: 0, 1569 | offset: 0, 1570 | kind: FileType::RegularFile, 1571 | name: [0; MAX_FILENAME_LENGTH], 1572 | }; 1573 | loop { 1574 | exchange!(ReadDir, self); 1575 | match self.tag() { 1576 | Message::Null => { 1577 | reply.error(ENOSYS); 1578 | return; 1579 | } 1580 | 1581 | Message::ReadDir => { 1582 | let ReadDir { 1583 | errno, 1584 | buf_ino, 1585 | offset, 1586 | kind, 1587 | name, 1588 | finished, 1589 | .. 1590 | } = unsafe { &mut self.payload().read_dir }; 1591 | if *finished { 1592 | break; 1593 | } else if let Some(errno) = *errno { 1594 | reply.error(errno); 1595 | return; 1596 | } else if *buf_ino != 0 1597 | && reply.add(*buf_ino, *offset, *kind, buf_to_osstr(name)) 1598 | { 1599 | break; 1600 | } 1601 | } 1602 | _ => panic!("Expected ReadDir"), 1603 | } 1604 | } 1605 | reply.ok(); 1606 | } 1607 | 1608 | fn releasedir( 1609 | &mut self, 1610 | _req: &Request<'_>, 1611 | _ino: u64, 1612 | fh: u64, 1613 | _flags: i32, 1614 | reply: ReplyEmpty, 1615 | ) { 1616 | exchange!( 1617 | ReleaseDir, 1618 | release_dir, 1619 | ReleaseDir { errno: None, fh }, 1620 | self 1621 | ); 1622 | match *self.tag() { 1623 | Message::Null => reply.error(ENOSYS), 1624 | Message::ReleaseDir => { 1625 | let errno = unsafe { self.payload().release_dir.errno }; 1626 | if let Some(errno) = errno { 1627 | reply.error(errno); 1628 | } else { 1629 | reply.ok(); 1630 | } 1631 | } 1632 | _ => panic!("Expected ReleaseDir"), 1633 | } 1634 | } 1635 | 1636 | fn statfs(&mut self, _req: &Request<'_>, _ino: u64, reply: ReplyStatfs) { 1637 | reply.statfs(0, 0, 0, 0, 0, 512, 255, 0); 1638 | } 1639 | 1640 | fn access(&mut self, _req: &Request<'_>, _ino: u64, _mask: i32, reply: ReplyEmpty) { 1641 | reply.ok(); // TODO: implement real access 1642 | } 1643 | 1644 | fn create( 1645 | &mut self, 1646 | _req: &Request<'_>, 1647 | parent: Ino, 1648 | name: &OsStr, 1649 | _mode: u32, 1650 | _umask: u32, 1651 | flags: i32, 1652 | reply: ReplyCreate, 1653 | ) { 1654 | let mut buf = [0; MAX_FILENAME_LENGTH]; 1655 | buf[0..name.len()].copy_from_slice(name.as_bytes()); 1656 | exchange!( 1657 | Create, 1658 | create, 1659 | Create { 1660 | errno: None, 1661 | parent, 1662 | name: buf, 1663 | flags, 1664 | attr: None, 1665 | generation: 0, 1666 | fh: 0, 1667 | open_flags: 0, 1668 | }, 1669 | self 1670 | ); 1671 | match *self.tag() { 1672 | Message::Null => reply.error(ENOSYS), 1673 | Message::Create => { 1674 | let Create { 1675 | errno, 1676 | attr, 1677 | generation, 1678 | fh, 1679 | open_flags, 1680 | .. 1681 | } = unsafe { self.payload().create }; 1682 | if let Some(errno) = errno { 1683 | reply.error(errno); 1684 | } else { 1685 | reply.created( 1686 | &std::time::Duration::ZERO, 1687 | &attr.expect("A filled attribute for create"), 1688 | generation, 1689 | fh, 1690 | open_flags, 1691 | ); 1692 | } 1693 | } 1694 | _ => panic!("Expected Create"), 1695 | } 1696 | } 1697 | 1698 | fn lseek( 1699 | &mut self, 1700 | _req: &Request<'_>, 1701 | _ino: u64, 1702 | fh: u64, 1703 | offset: i64, 1704 | whence: i32, 1705 | reply: ReplyLseek, 1706 | ) { 1707 | exchange!( 1708 | LSeek, 1709 | l_seek, 1710 | LSeek { 1711 | errno: None, 1712 | fh, 1713 | offset, 1714 | whence, 1715 | }, 1716 | self 1717 | ); 1718 | match *self.tag() { 1719 | Message::Null => reply.error(ENOSYS), 1720 | Message::LSeek => { 1721 | let LSeek { errno, offset, .. } = unsafe { self.payload().l_seek }; 1722 | if let Some(errno) = errno { 1723 | reply.error(errno); 1724 | } else { 1725 | reply.offset(offset); 1726 | } 1727 | } 1728 | _ => panic!("Expected LSeek"), 1729 | } 1730 | } 1731 | } 1732 | 1733 | fn buf_to_osstr(b: &[u8]) -> &OsStr { 1734 | OsStr::from_bytes( 1735 | &b[..b 1736 | .iter() 1737 | .position(|e| *e == 0) 1738 | .unwrap_or(MAX_FILENAME_LENGTH) 1739 | .min(b.len())], 1740 | ) 1741 | } 1742 | --------------------------------------------------------------------------------