├── src ├── empty_file_for_testing.rs ├── unix.rs ├── windows.rs └── lib.rs ├── .gitignore ├── rust-toolchain.toml ├── Cargo.toml ├── contributing.md ├── examples ├── head.rs ├── sha256sum_filebuffer.rs └── sha256sum_naive.rs ├── .appveyor.yml ├── changelog.md ├── readme.md ├── Cargo.lock └── license /src/empty_file_for_testing.rs: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Cargo files 2 | /target 3 | 4 | # Editor files 5 | *.swp 6 | *.swo 7 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | # Note, this is the Minimum Supported Rust Version. We build with this by 3 | # default to ensure we don't accidentally break it, but of course you should be 4 | # able to build with more recent versions. 5 | channel = "1.40.0" 6 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "filebuffer" 3 | version = "1.0.0" 4 | authors = ["Ruud van Asseldonk "] 5 | license = "Apache-2.0" 6 | readme = "readme.md" 7 | keywords = ["non-blocking", "mmap", "file", "input", "prefetch"] 8 | categories = ["asynchronous", "filesystem", "memory-management"] 9 | description = "Fast and simple file reading" 10 | repository = "https://github.com/ruuda/filebuffer" 11 | documentation = "https://docs.rs/filebuffer" 12 | edition = "2018" 13 | 14 | [target.'cfg(unix)'.dependencies] 15 | libc = "0.2.86" 16 | 17 | [target.'cfg(windows)'.dependencies] 18 | winapi = { version = "0.3", features = ["handleapi", "memoryapi", "processthreadsapi", "sysinfoapi", "winnt"] } 19 | 20 | [dev-dependencies] 21 | rust-crypto = "0.2.36" 22 | -------------------------------------------------------------------------------- /contributing.md: -------------------------------------------------------------------------------- 1 | Contributing 2 | ============ 3 | 4 | Contributions in the form of bug reports, feature requests, or pull requests are 5 | welcome. For pull requests, please consider: 6 | 7 | * Write [a proper commit message][proper-commit] and keep the history clean. 8 | * Avoid unrelated formatting changes, they make it harder to identify 9 | functional changes in the diff. 10 | * You agree to license your contribution under the Apache 2.0 license. 11 | 12 | Code of conduct 13 | --------------- 14 | 15 | * Be nice. 16 | * Please do not discuss politics in the issue tracker, 17 | the issue tracker is for technical issues. 18 | * Do not post LLM-generated content. 19 | 20 | [proper-commit]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html 21 | -------------------------------------------------------------------------------- /examples/head.rs: -------------------------------------------------------------------------------- 1 | // Filebuffer -- Fast and simple file reading 2 | // Copyright 2016 Ruud van Asseldonk 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // A copy of the License has been included in the root of the repository. 7 | 8 | // This example implements the `head` program in Rust using the Filebuffer library. 9 | // Input files are assumed to be valid UTF-8. 10 | 11 | use std::env; 12 | use std::str; 13 | use filebuffer::FileBuffer; 14 | 15 | extern crate filebuffer; 16 | 17 | fn main() { 18 | for fname in env::args().skip(1) { 19 | println!("==> {} <==", &fname); 20 | let fbuffer = FileBuffer::open(&fname).expect("failed to open file"); 21 | let lines = str::from_utf8(&fbuffer).expect("not valid UTF-8").lines(); 22 | for line in lines.take(10) { 23 | println!("{}", line); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /.appveyor.yml: -------------------------------------------------------------------------------- 1 | environment: 2 | matrix: 3 | - target: 1.40.0-x86_64-pc-windows-msvc 4 | - target: 1.40.0-i686-pc-windows-msvc 5 | - target: 1.60.0-x86_64-pc-windows-msvc 6 | - target: 1.60.0-i686-pc-windows-msvc 7 | - target: beta-x86_64-pc-windows-msvc 8 | - target: beta-i686-pc-windows-msvc 9 | - target: nightly-x86_64-pc-windows-msvc 10 | - target: nightly-i686-pc-windows-msvc 11 | 12 | install: 13 | # Download the Rust and Cargo installer. 14 | - ps: Start-FileDownload "https://static.rust-lang.org/dist/rust-${env:target}.msi" 15 | 16 | # Install Rust and Cargo and wait for installation to finish by using Write-Output. 17 | - ps: msiexec /package "rust-${env:target}.msi" /quiet /norestart | Write-Output 18 | 19 | # Pick up the new Path variable after the installer modified it. 20 | - ps: $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") 21 | 22 | # Print versions for future reference. 23 | - rustc --version 24 | - cargo --version 25 | 26 | build_script: 27 | - cargo build 28 | 29 | test_script: 30 | - cargo test 31 | -------------------------------------------------------------------------------- /examples/sha256sum_filebuffer.rs: -------------------------------------------------------------------------------- 1 | // Filebuffer -- Fast and simple file reading 2 | // Copyright 2016 Ruud van Asseldonk 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // A copy of the License has been included in the root of the repository. 7 | 8 | // This example implements the `sha256sum` program in Rust using the Filebuffer library. Compare 9 | // with `sha256sum_naive` which uses the IO primitives in the standard library. 10 | 11 | use std::env; 12 | use crypto::digest::Digest; 13 | use crypto::sha2::Sha256; 14 | use filebuffer::FileBuffer; 15 | 16 | extern crate crypto; 17 | extern crate filebuffer; 18 | 19 | fn main() { 20 | for fname in env::args().skip(1) { 21 | let fbuffer = FileBuffer::open(&fname).expect("failed to open file"); 22 | let mut hasher = Sha256::new(); 23 | hasher.input(&fbuffer); 24 | 25 | // Match the output format of `sha256sum`, which has two spaces between the hash and name. 26 | println!("{} {}", hasher.result_str(), fname); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /examples/sha256sum_naive.rs: -------------------------------------------------------------------------------- 1 | // Filebuffer -- Fast and simple file reading 2 | // Copyright 2016 Ruud van Asseldonk 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // A copy of the License has been included in the root of the repository. 7 | 8 | // This example implements the `sha256sum` program in Rust using the IO primitives in the 9 | // standard library. Compare with `sha256sum_filebuffer` which uses the Filebuffer library. 10 | 11 | use std::env; 12 | use std::fs; 13 | use std::io; 14 | use std::io::BufRead; 15 | use crypto::digest::Digest; 16 | use crypto::sha2::Sha256; 17 | 18 | extern crate crypto; 19 | 20 | fn main() { 21 | for fname in env::args().skip(1) { 22 | let file = fs::File::open(&fname).expect("failed to open file"); 23 | let mut reader = io::BufReader::new(file); 24 | let mut hasher = Sha256::new(); 25 | 26 | loop { 27 | let consumed_len = { 28 | let buffer = reader.fill_buf().expect("failed to read from file"); 29 | if buffer.len() == 0 { 30 | // End of file. 31 | break; 32 | } 33 | hasher.input(buffer); 34 | buffer.len() 35 | }; 36 | reader.consume(consumed_len); 37 | } 38 | 39 | // Match the output format of `sha256sum`, which has two spaces between the hash and name. 40 | println!("{} {}", hasher.result_str(), fname); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /changelog.md: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | 1.0.0 5 | ----- 6 | 7 | Released 2024-05-18. 8 | 9 | * **Compatibility:** The minimum supported Rust version is now 1.40, up from 10 | 1.25 previously. 11 | * Ensure compatibility with Rust 1.78 (which introduced a panic in 12 | `slice::from_raw_parts` that affects mapping an empty file). 13 | * Use Rust 2018 edition, upgrade usage of `try!` to the `?` operator. 14 | 15 | 0.4.0 16 | ----- 17 | 18 | Released 2018-04-29. 19 | 20 | * Bump `winapi` dependency to 0.3. 21 | * Ensures compatibility with Rust 1.8 through 1.25 stable. 22 | 23 | 0.3.0 24 | ----- 25 | 26 | Released 2017-08-03. 27 | 28 | * Add support for Mac OS X. 29 | * Implement `AsRef<[u8]>` for `Filebuffer`. 30 | * Ensures compatibility with Rust 1.8 through 1.19 stable. 31 | 32 | Thanks to Craig M. Brandenburg for contributing to this release. 33 | 34 | 0.2.0 35 | ----- 36 | 37 | Released 2017-05-20. 38 | 39 | * Derive `fmt::Debug` for public types. 40 | * Depend on libc only on Unix-like environments, and on kernel32-sys only on 41 | Windows. This requires Rust 1.8 or later, so this is a breaking change. 42 | * Ensures compatibility with Rust 1.8 through 1.17 stable. 43 | 44 | Thanks to Colin Wallace for contributing to this release. 45 | 46 | 0.1.1 47 | ----- 48 | 49 | Released 2017-02-01. 50 | 51 | * Ensures compatibility with Rust 1.4 through 1.14 stable. 52 | * Host documentation on docs.rs (thanks, docs.rs authors!). 53 | * Update crate metadata. 54 | 55 | 0.1.0 56 | ----- 57 | 58 | Released 2016-01-31. 59 | 60 | Initial release with Windows and Linux support. 61 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | Filebuffer 2 | ========== 3 | Fast and simple file reading for Rust. 4 | 5 | [![Crates.io version][crate-img]][crate] 6 | [![Changelog][changelog-img]][changelog] 7 | [![Documentation][docs-img]][docs] 8 | 9 | Filebuffer can map files into memory. This is often faster than using the 10 | primitives in `std::io`, and also more convenient. Furthermore this crate 11 | offers prefetching and checking whether file data is resident in physical 12 | memory (so access will not incur a page fault). This enables non-blocking 13 | file reading. 14 | 15 | Example 16 | ------- 17 | Below is an implementation of the `sha256sum` program that is both faster and 18 | simpler than the naive `std::io` equivalent. (See `sha256sum_filebuffer` and 19 | `sha256sum_naive` in the examples directory.) 20 | 21 | ```rust 22 | use std::env; 23 | use crypto::digest::Digest; 24 | use crypto::sha2::Sha256; 25 | use filebuffer::FileBuffer; 26 | 27 | extern crate crypto; 28 | extern crate filebuffer; 29 | 30 | fn main() { 31 | for fname in env::args().skip(1) { 32 | let fbuffer = FileBuffer::open(&fname).expect("failed to open file"); 33 | let mut hasher = Sha256::new(); 34 | hasher.input(&fbuffer); 35 | println!("{} {}", hasher.result_str(), fname); 36 | } 37 | } 38 | ``` 39 | 40 | License 41 | ------- 42 | Filebuffer is licensed under the [Apache 2.0][apache2] license. It may be used 43 | in free software as well as closed-source applications, both for commercial and 44 | non-commercial use under the conditions given in the license. If you want to use 45 | Filebuffer in your GPLv2-licensed software, you can add an [exception][except] 46 | to your copyright notice. 47 | 48 | [crate-img]: https://img.shields.io/crates/v/filebuffer.svg 49 | [crate]: https://crates.io/crates/filebuffer 50 | [changelog-img]: https://img.shields.io/badge/changelog-online-blue.svg 51 | [changelog]: https://github.com/ruuda/filebuffer/blob/master/changelog.md#changelog 52 | [docs-img]: https://img.shields.io/badge/docs-online-blue.svg 53 | [docs]: https://docs.rs/filebuffer 54 | [apache2]: https://www.apache.org/licenses/LICENSE-2.0 55 | [except]: https://www.gnu.org/licenses/gpl-faq.html#GPLIncompatibleLibs 56 | -------------------------------------------------------------------------------- /src/unix.rs: -------------------------------------------------------------------------------- 1 | // Filebuffer -- Fast and simple file reading 2 | // Copyright 2016 Ruud van Asseldonk 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // A copy of the License has been included in the root of the repository. 7 | 8 | //! This mod contains the platform-specific implementations of functions based on the libc crate 9 | //! that is available on Unix-ish platforms. 10 | 11 | use std::fs; 12 | use std::io; 13 | use std::mem; 14 | use std::os::unix::io::AsRawFd; 15 | use std::ptr; 16 | 17 | extern crate libc; 18 | 19 | #[derive(Debug)] 20 | pub struct PlatformData; 21 | 22 | pub fn map_file(file: fs::File) -> io::Result<(*const u8, usize, PlatformData)> { 23 | let fd = file.as_raw_fd(); 24 | let length = file.metadata()?.len(); 25 | 26 | if length > usize::max_value() as u64 { 27 | return Err(io::Error::new(io::ErrorKind::Other, "file is larger than address space")); 28 | } 29 | 30 | // Don't try to map anything if the file is empty. 31 | if length == 0 { 32 | return Ok((ptr::null(), 0, PlatformData)); 33 | } 34 | 35 | let result = unsafe { 36 | libc::mmap(ptr::null_mut(), length as usize, libc::PROT_READ, libc::MAP_PRIVATE, fd, 0) 37 | }; 38 | 39 | if result == libc::MAP_FAILED { 40 | Err(io::Error::last_os_error()) 41 | } else { 42 | Ok((result as *const u8, length as usize, PlatformData)) 43 | } 44 | } 45 | 46 | pub fn unmap_file(buffer: *const u8, length: usize) { 47 | let result = unsafe { libc::munmap(buffer as *mut libc::c_void, length) }; 48 | 49 | // `munmap` only fails due to incorrect usage, which is a program error, not a runtime failure. 50 | assert!(result == 0); 51 | } 52 | 53 | /// Writes whether the pages in the range starting at `buffer` with a length of `length` bytes 54 | /// are resident in physical memory into `residency`. The size of `residency` must be at least 55 | /// `length / page_size`. Both `buffer` and `length` must be a multiple of the page size. 56 | pub fn get_resident(buffer: *const u8, length: usize, residency: &mut [bool]) { 57 | use std::thread; 58 | 59 | let result = unsafe { 60 | // Note: the libc on BSD descendants uses a signed char for residency_char while 61 | // glibc uses an unsigned one, which is why we use an type-inferred cast here. 62 | let residency_char = residency.as_mut_ptr() as *mut _; 63 | assert_eq!(1, mem::size_of_val(&*residency_char)); 64 | libc::mincore(buffer as *mut libc::c_void, length, residency_char) 65 | }; 66 | 67 | // Any error code except EAGAIN indicates a programming error. 68 | assert!(result == libc::EAGAIN || result == 0); 69 | 70 | // In the rare occasion that the kernel is busy, yield so we don't spam the kernel with 71 | // `mincore` calls, then try again. 72 | if result == libc::EAGAIN { 73 | thread::yield_now(); 74 | get_resident(buffer, length, residency) 75 | } 76 | } 77 | 78 | /// Requests the kernel to make the specified range of bytes resident in physical memory. `buffer` 79 | /// must be page-aligned. 80 | pub fn prefetch(buffer: *const u8, length: usize) { 81 | let result = unsafe { 82 | libc::madvise(buffer as *mut libc::c_void, length, libc::MADV_WILLNEED) 83 | }; 84 | 85 | // Any returned error code indicates a programming error, not a runtime error. 86 | assert_eq!(0, result); 87 | } 88 | 89 | pub fn get_page_size() -> usize { 90 | let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) as usize }; 91 | 92 | // Assert that the page size is a power of two, which is assumed when the page size is used. 93 | assert!(page_size != 0); 94 | assert_eq!(0, page_size & (page_size - 1)); 95 | 96 | page_size 97 | } 98 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | [[package]] 4 | name = "filebuffer" 5 | version = "1.0.0" 6 | dependencies = [ 7 | "libc", 8 | "rust-crypto", 9 | "winapi", 10 | ] 11 | 12 | [[package]] 13 | name = "fuchsia-cprng" 14 | version = "0.1.1" 15 | source = "registry+https://github.com/rust-lang/crates.io-index" 16 | checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" 17 | 18 | [[package]] 19 | name = "gcc" 20 | version = "0.3.55" 21 | source = "registry+https://github.com/rust-lang/crates.io-index" 22 | checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" 23 | 24 | [[package]] 25 | name = "libc" 26 | version = "0.2.87" 27 | source = "registry+https://github.com/rust-lang/crates.io-index" 28 | checksum = "265d751d31d6780a3f956bb5b8022feba2d94eeee5a84ba64f4212eedca42213" 29 | 30 | [[package]] 31 | name = "rand" 32 | version = "0.3.23" 33 | source = "registry+https://github.com/rust-lang/crates.io-index" 34 | checksum = "64ac302d8f83c0c1974bf758f6b041c6c8ada916fbb44a609158ca8b064cc76c" 35 | dependencies = [ 36 | "libc", 37 | "rand 0.4.6", 38 | ] 39 | 40 | [[package]] 41 | name = "rand" 42 | version = "0.4.6" 43 | source = "registry+https://github.com/rust-lang/crates.io-index" 44 | checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" 45 | dependencies = [ 46 | "fuchsia-cprng", 47 | "libc", 48 | "rand_core 0.3.1", 49 | "rdrand", 50 | "winapi", 51 | ] 52 | 53 | [[package]] 54 | name = "rand_core" 55 | version = "0.3.1" 56 | source = "registry+https://github.com/rust-lang/crates.io-index" 57 | checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" 58 | dependencies = [ 59 | "rand_core 0.4.2", 60 | ] 61 | 62 | [[package]] 63 | name = "rand_core" 64 | version = "0.4.2" 65 | source = "registry+https://github.com/rust-lang/crates.io-index" 66 | checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" 67 | 68 | [[package]] 69 | name = "rdrand" 70 | version = "0.4.0" 71 | source = "registry+https://github.com/rust-lang/crates.io-index" 72 | checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" 73 | dependencies = [ 74 | "rand_core 0.3.1", 75 | ] 76 | 77 | [[package]] 78 | name = "rust-crypto" 79 | version = "0.2.36" 80 | source = "registry+https://github.com/rust-lang/crates.io-index" 81 | checksum = "f76d05d3993fd5f4af9434e8e436db163a12a9d40e1a58a726f27a01dfd12a2a" 82 | dependencies = [ 83 | "gcc", 84 | "libc", 85 | "rand 0.3.23", 86 | "rustc-serialize", 87 | "time", 88 | ] 89 | 90 | [[package]] 91 | name = "rustc-serialize" 92 | version = "0.3.25" 93 | source = "registry+https://github.com/rust-lang/crates.io-index" 94 | checksum = "fe834bc780604f4674073badbad26d7219cadfb4a2275802db12cbae17498401" 95 | 96 | [[package]] 97 | name = "time" 98 | version = "0.1.44" 99 | source = "registry+https://github.com/rust-lang/crates.io-index" 100 | checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" 101 | dependencies = [ 102 | "libc", 103 | "wasi", 104 | "winapi", 105 | ] 106 | 107 | [[package]] 108 | name = "wasi" 109 | version = "0.10.0+wasi-snapshot-preview1" 110 | source = "registry+https://github.com/rust-lang/crates.io-index" 111 | checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" 112 | 113 | [[package]] 114 | name = "winapi" 115 | version = "0.3.9" 116 | source = "registry+https://github.com/rust-lang/crates.io-index" 117 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 118 | dependencies = [ 119 | "winapi-i686-pc-windows-gnu", 120 | "winapi-x86_64-pc-windows-gnu", 121 | ] 122 | 123 | [[package]] 124 | name = "winapi-i686-pc-windows-gnu" 125 | version = "0.4.0" 126 | source = "registry+https://github.com/rust-lang/crates.io-index" 127 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 128 | 129 | [[package]] 130 | name = "winapi-x86_64-pc-windows-gnu" 131 | version = "0.4.0" 132 | source = "registry+https://github.com/rust-lang/crates.io-index" 133 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 134 | -------------------------------------------------------------------------------- /src/windows.rs: -------------------------------------------------------------------------------- 1 | // Filebuffer -- Fast and simple file reading 2 | // Copyright 2016 Ruud van Asseldonk 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // A copy of the License has been included in the root of the repository. 7 | 8 | //! This mod contains the platform-specific implementations for Windows based on the winapi crate. 9 | 10 | use std::fs; 11 | use std::io; 12 | use std::mem; 13 | use std::os::windows::io::AsRawHandle; 14 | use std::ptr; 15 | 16 | extern crate winapi; 17 | 18 | #[derive(Debug)] 19 | pub struct PlatformData { 20 | // On Windows, the file must be kept open for the lifetime of the mapping. 21 | #[allow(dead_code)] // The field is not dead, the destructor is effectful. 22 | file: fs::File, 23 | mapping_handle: winapi::um::winnt::HANDLE, 24 | } 25 | 26 | impl Drop for PlatformData { 27 | fn drop (&mut self) { 28 | if self.mapping_handle != ptr::null_mut() { 29 | let success = unsafe { winapi::um::handleapi::CloseHandle(self.mapping_handle) }; 30 | assert!(success != 0); 31 | } 32 | } 33 | } 34 | 35 | pub fn map_file(file: fs::File) -> io::Result<(*const u8, usize, PlatformData)> { 36 | let file_handle = file.as_raw_handle(); 37 | let length = file.metadata()?.len(); 38 | 39 | if length > usize::max_value() as u64 { 40 | return Err(io::Error::new(io::ErrorKind::Other, "file is larger than address space")); 41 | } 42 | 43 | let mut platform_data = PlatformData { 44 | file: file, 45 | mapping_handle: ptr::null_mut(), 46 | }; 47 | 48 | // Don't try to map anything if the file is empty. 49 | if length == 0 { 50 | return Ok((ptr::null(), 0, platform_data)); 51 | } 52 | 53 | // Memory-mapping a file on Windows is a two-step process: first we create a file mapping 54 | // object, and then we create a view of that mapping in the virtual address space. 55 | platform_data.mapping_handle = unsafe { 56 | winapi::um::memoryapi::CreateFileMappingW( 57 | file_handle as *mut winapi::ctypes::c_void, 58 | ptr::null_mut(), // Use default security policy. 59 | winapi::um::winnt::PAGE_READONLY, // The memory will be read-only. 60 | 0, 0, // The mapping size is the size of the file. 61 | ptr::null_mut() // The mapping does not have a name. 62 | ) 63 | }; 64 | 65 | if platform_data.mapping_handle == ptr::null_mut() { 66 | return Err(io::Error::last_os_error()); 67 | } 68 | 69 | let result = unsafe { 70 | winapi::um::memoryapi::MapViewOfFile( 71 | platform_data.mapping_handle, 72 | winapi::um::memoryapi::FILE_MAP_READ, // The memory mapping will be read-only. 73 | 0, 0, // Start offset of the mapping is 0. 74 | length as winapi::shared::basetsd::SIZE_T // Map the entire file. 75 | ) 76 | }; 77 | 78 | if result == ptr::null_mut() { 79 | Err(io::Error::last_os_error()) 80 | } else { 81 | Ok((result as *const u8, length as usize, platform_data)) 82 | } 83 | } 84 | 85 | pub fn unmap_file(buffer: *const u8, _length: usize) { 86 | let success = unsafe { 87 | winapi::um::memoryapi::UnmapViewOfFile(buffer as *mut winapi::ctypes::c_void) 88 | }; 89 | assert!(success != 0); 90 | } 91 | 92 | /// See also `unix::get_resident`. 93 | pub fn get_resident(_buffer: *const u8, _length: usize, residency: &mut [bool]) { 94 | // As far as I am aware, Windows does not expose a way to query whether pages are resident. 95 | // There is no equivalent of `mincore()`. The closest thing is `VirtualQuery()`, but the 96 | // `state` value in the `MEMORY_BASIC_INFORMATION` struct that it fills does not indicate 97 | // whether the page is resident. 98 | 99 | // Lie and pretend everything is resident. 100 | for x in residency { 101 | *x = true; 102 | } 103 | } 104 | 105 | /// See also `unix::prefetch`. 106 | pub fn prefetch(buffer: *const u8, length: usize) { 107 | let mut entry = winapi::um::memoryapi::WIN32_MEMORY_RANGE_ENTRY { 108 | VirtualAddress: buffer as *mut winapi::ctypes::c_void, 109 | NumberOfBytes: length as winapi::shared::basetsd::SIZE_T, 110 | }; 111 | 112 | unsafe { 113 | let current_process_handle = winapi::um::processthreadsapi::GetCurrentProcess(); 114 | winapi::um::memoryapi::PrefetchVirtualMemory( 115 | current_process_handle, // Prefetch for the current process. 116 | 1, &mut entry, // An array of length 1 that contains `entry`. 117 | 0 // Reserved flag that must be 0. 118 | ); 119 | } 120 | 121 | // The return value of `PrefetchVirtualMemory` is ignored. MSDN says the function may fail if 122 | // the system is under memory pressure. (It is not entirely clear whether "fail" means 123 | // "returns a nonzero value", but I assume it does.) 124 | } 125 | 126 | pub fn get_page_size() -> usize { 127 | // Fill the `SYSTEM_INFO` struct with zeroes. It will be filled by 128 | // `GetSystemInfo` later but Rust requires it to be initialized. 129 | let mut sysinfo: winapi::um::sysinfoapi::SYSTEM_INFO = unsafe { mem::zeroed() }; 130 | unsafe { winapi::um::sysinfoapi::GetSystemInfo(&mut sysinfo); } 131 | sysinfo.dwPageSize as usize 132 | } 133 | -------------------------------------------------------------------------------- /license: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | // Filebuffer -- Fast and simple file reading 2 | // Copyright 2016 Ruud van Asseldonk 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // A copy of the License has been included in the root of the repository. 7 | 8 | //! Filebuffer, a library for fast and simple file reading. 9 | //! 10 | //! # Examples 11 | //! 12 | //! Map a file into memory and access it as a slice of bytes. This is simple and will generally 13 | //! outperform `Read::read_to_end()`. 14 | //! 15 | //! ``` 16 | //! use filebuffer::FileBuffer; 17 | //! let fbuffer = FileBuffer::open("src/lib.rs").unwrap(); 18 | //! assert_eq!(&fbuffer[3..45], &b"Filebuffer -- Fast and simple file reading"[..]); 19 | //! ``` 20 | 21 | #![warn(missing_docs)] 22 | 23 | use std::cmp; 24 | use std::io; 25 | use std::fs; 26 | use std::ops::Deref; 27 | use std::path::Path; 28 | use std::ptr; 29 | use std::slice; 30 | 31 | #[cfg(unix)] 32 | mod unix; 33 | 34 | #[cfg(windows)] 35 | mod windows; 36 | 37 | #[cfg(unix)] 38 | use unix::{PlatformData, get_page_size, map_file, unmap_file, prefetch}; 39 | 40 | #[cfg(all(unix))] 41 | use unix::get_resident; 42 | 43 | #[cfg(windows)] 44 | use windows::{PlatformData, get_resident, get_page_size, map_file, unmap_file, prefetch}; 45 | 46 | /// A memory-mapped file. 47 | /// 48 | /// # Safety 49 | /// 50 | /// **On Unix-ish platforms, external modifications to the file made after the file buffer was 51 | /// opened can show up in this file buffer.** In particular, if a file is truncated after opening, 52 | /// accessing the removed part causes undefined behavior. On Windows it is possible to prevent this 53 | /// by opening the file in exclusive mode, but that functionality is not available in stable Rust 54 | /// currently. (Filebuffer will be updated after stabilization.) 55 | /// 56 | /// It is recommended to ensure that other applications do not write to the file when it is mapped, 57 | /// possibly by marking the file read-only. (Though even this is no guarantee.) 58 | #[derive(Debug)] 59 | pub struct FileBuffer { 60 | page_size: usize, 61 | buffer: *const u8, 62 | length: usize, 63 | 64 | #[allow(dead_code)] // This field is not dead, it might have an effectful destructor. 65 | platform_data: PlatformData, 66 | } 67 | 68 | /// Rounds `size` up to the nearest multiple of `power_of_two`. 69 | fn round_up_to(size: usize, power_of_two: usize) -> usize { 70 | (size + (power_of_two - 1)) & !(power_of_two - 1) 71 | } 72 | 73 | #[test] 74 | fn verify_round_up_to() { 75 | assert_eq!(1024, round_up_to(23, 1024)); 76 | assert_eq!(1024, round_up_to(1024, 1024)); 77 | assert_eq!(2048, round_up_to(1025, 1024)); 78 | } 79 | 80 | /// Rounds `size` down to the nearest multiple of `power_of_two`. 81 | fn round_down_to(size: usize, power_of_two: usize) -> usize { 82 | size & !(power_of_two - 1) 83 | } 84 | 85 | #[test] 86 | fn verify_round_down_to() { 87 | assert_eq!(0, round_down_to(23, 1024)); 88 | assert_eq!(1024, round_down_to(1024, 1024)); 89 | assert_eq!(1024, round_down_to(1025, 1024)); 90 | } 91 | 92 | impl FileBuffer { 93 | /// Maps the file at `path` into memory. 94 | pub fn open>(path: P) -> io::Result { 95 | // Open the `fs::File` so we get all of std's error handling for free, then use it to 96 | // extract the file descriptor. The file is closed again when `map_file` returns on 97 | // Unix-ish platforms, but `mmap` only requires the descriptor to be open for the `mmap` 98 | // call, so this is fine. On Windows, the file must be kept open for the lifetime of the 99 | // mapping, so `map_file` moves the file into the platform data. 100 | let mut open_opts = fs::OpenOptions::new(); 101 | open_opts.read(true); 102 | 103 | // TODO: On Windows, set `share_mode()` to read-only. This requires the 104 | // `open_options_ext` feature that is currently unstable, but it is 105 | // required to ensure that a different process does not suddenly modify 106 | // the contents of the file. See also Rust issue 27720. 107 | 108 | let file = open_opts.open(path)?; 109 | let (buffer, length, platform_data) = map_file(file)?; 110 | let fbuffer = FileBuffer { 111 | page_size: get_page_size(), 112 | buffer: buffer, 113 | length: length, 114 | platform_data: platform_data 115 | }; 116 | Ok(fbuffer) 117 | } 118 | 119 | /// Returns the number of bytes resident in physical memory, starting from `offset`. 120 | /// 121 | /// The slice `[offset..offset + resident_len]` can be accessed without causing page faults or 122 | /// disk access. Note that this is only a snapshot, and the kernel might decide to evict pages 123 | /// or make them resident at any time. 124 | /// 125 | /// The returned resident length is at most `length`. 126 | /// 127 | /// # Panics 128 | /// 129 | /// Panics if the specified range lies outside of the buffer. 130 | /// 131 | /// # Remarks 132 | /// 133 | /// Windows does not expose a mechanism to query which pages are resident in physical 134 | /// memory. Therefore this function optimistically claims that the entire range is resident 135 | /// on Windows. 136 | pub fn resident_len(&self, offset: usize, length: usize) -> usize { 137 | // The specified offset and length must lie within the buffer. 138 | assert!(offset + length <= self.length); 139 | 140 | // This is a no-op for empty files. 141 | if self.buffer == ptr::null() { return 0; } 142 | 143 | let aligned_offset = round_down_to(offset, self.page_size); 144 | let aligned_length = round_up_to(length + (offset - aligned_offset), self.page_size); 145 | let num_pages = aligned_length / self.page_size; 146 | 147 | // There is a tradeoff here: to store residency information, we need an array of booleans. 148 | // The requested range can potentially be very large and it is only known at runtime. We 149 | // could allocate a vector here, but that requires a heap allocation just to get residency 150 | // information (which might in turn cause a page fault). Instead, check at most 32 pages at 151 | // once. This means more syscalls for large ranges, but it saves us the heap allocation, 152 | // and for ranges up to 32 pages (128 KiB typically) there is only one syscall. 153 | let mut residency = [false; 32]; 154 | let mut pages_checked = 0; 155 | let mut pages_resident = 0; 156 | 157 | while pages_checked < num_pages { 158 | let pages_to_check = cmp::min(32, num_pages - pages_checked); 159 | let check_offset = (aligned_offset + pages_checked * self.page_size) as isize; 160 | let check_buffer = unsafe { self.buffer.offset(check_offset) }; 161 | let check_length = pages_to_check * self.page_size; 162 | get_resident(check_buffer, check_length, &mut residency); 163 | 164 | // Count the number of resident pages. 165 | match residency[..pages_to_check].iter().position(|resident| !resident) { 166 | Some(non_resident) => { 167 | // The index of the non-resident page is the number of resident pages. 168 | pages_resident += non_resident; 169 | break; 170 | } 171 | None => { 172 | pages_resident += pages_to_check; 173 | pages_checked += pages_to_check; 174 | } 175 | } 176 | } 177 | 178 | let resident_length = pages_resident * self.page_size + aligned_offset - offset; 179 | 180 | // Never return more than the requested length. The resident length might be larger than 181 | // the length of the buffer, because it is rounded up to the page size. 182 | cmp::min(length, resident_length) 183 | } 184 | 185 | /// Returns the system page size. 186 | /// 187 | /// When the kernel makes the file resident in physical memory, it does so with page 188 | /// granularity. (In practice this happens in larger chunks, but still in multiples of 189 | /// the page size.) Therefore, when processing the file in chunks, this is a good chunk 190 | /// length. 191 | pub fn chunk_len_hint(&self) -> usize { 192 | self.page_size 193 | } 194 | 195 | /// Advises the kernel to make a slice of the file resident in physical memory. 196 | /// 197 | /// This method does not block, meaning that when the function returns, the slice is not 198 | /// necessarily resident. After this function returns, the kernel may read the requested slice 199 | /// from disk and make it resident. Note that this is only an advice, the kernel need not honor 200 | /// it. 201 | /// 202 | /// To check whether the slice is resident at a later time, use `resident_len()`. 203 | /// 204 | /// # Panics 205 | /// 206 | /// Panics if the specified range lies outside of the buffer. 207 | pub fn prefetch(&self, offset: usize, length: usize) { 208 | // TODO: This function should use `collections::range::RangeArgument` once stabilized. 209 | // The specified offset and length must lie within the buffer. 210 | assert!(offset + length <= self.length); 211 | 212 | // This is a no-op for empty files. 213 | if self.buffer == ptr::null() { return; } 214 | 215 | let aligned_offset = round_down_to(offset, self.page_size); 216 | let aligned_length = round_up_to(length + (offset - aligned_offset), self.page_size); 217 | 218 | let buffer = unsafe { self.buffer.offset(aligned_offset as isize) }; 219 | prefetch(buffer, aligned_length); 220 | } 221 | 222 | /// Leaks the file buffer as a byte slice. 223 | /// 224 | /// This prevents the buffer from being unmapped, keeping the file mapped until the program 225 | /// ends. This is not as bad as it sounds, because the kernel is free to evict pages from 226 | /// physical memory in case of memory pressure. Because the file is mapped read-only, it can 227 | /// always be read from disk again. 228 | /// 229 | /// If the file buffer is going to be open for the entire duration of the program anyway, this 230 | /// method can avoid some lifetime issues. Still, it is good practice to close the file buffer 231 | /// if possible. This method should be a last resort. 232 | pub fn leak(mut self) -> &'static [u8] { 233 | let buffer = if self.buffer == ptr::null() { 234 | &[] 235 | } else { 236 | unsafe { slice::from_raw_parts(self.buffer, self.length) } 237 | }; 238 | 239 | // Prevent `drop()` from freeing the buffer. 240 | self.buffer = ptr::null(); 241 | self.length = 0; 242 | 243 | buffer 244 | } 245 | } 246 | 247 | // There is no possibility of data races when passing `&FileBuffer` across threads, 248 | // because the buffer is read-only. `&FileBuffer` has no interior mutability. 249 | unsafe impl Sync for FileBuffer {} 250 | 251 | // It is safe to move a `FileBuffer` into a different thread. 252 | unsafe impl Send for FileBuffer {} 253 | 254 | impl Drop for FileBuffer { 255 | fn drop(&mut self) { 256 | if self.buffer != ptr::null() { unmap_file(self.buffer, self.length); } 257 | } 258 | } 259 | 260 | impl Deref for FileBuffer { 261 | type Target = [u8]; 262 | 263 | fn deref(&self) -> &[u8] { 264 | if self.buffer == ptr::null() { 265 | &[] 266 | } else { 267 | unsafe { slice::from_raw_parts(self.buffer, self.length) } 268 | } 269 | } 270 | } 271 | 272 | impl AsRef<[u8]> for FileBuffer { 273 | fn as_ref(&self) -> &[u8] { 274 | self.deref() 275 | } 276 | } 277 | 278 | #[test] 279 | fn open_file() { 280 | let fbuffer = FileBuffer::open("src/lib.rs"); 281 | assert!(fbuffer.is_ok()); 282 | } 283 | 284 | #[test] 285 | fn make_resident() { 286 | let fbuffer = FileBuffer::open("src/lib.rs").unwrap(); 287 | 288 | // Touch the first page to make it resident. 289 | assert_eq!(&fbuffer[3..13], &b"Filebuffer"[..]); 290 | 291 | // Now at least that part should be resident. 292 | assert_eq!(fbuffer.resident_len(3, 10), 10); 293 | } 294 | 295 | #[test] 296 | fn prefetch_is_not_harmful() { 297 | let fbuffer = FileBuffer::open("src/lib.rs").unwrap(); 298 | 299 | // It is impossible to test that this actually works without root access to instruct the kernel 300 | // to drop its caches, but at least we can verify that calling `prefetch` is not harmful. 301 | fbuffer.prefetch(0, fbuffer.len()); 302 | 303 | // Reading from the file should still work as normal. 304 | assert_eq!(&fbuffer[3..13], &b"Filebuffer"[..]); 305 | } 306 | 307 | #[test] 308 | fn drop_after_leak() { 309 | let mut bytes = &[0u8][..]; 310 | assert_eq!(bytes[0], 0); 311 | { 312 | let fbuffer = FileBuffer::open("src/lib.rs").unwrap(); 313 | bytes = fbuffer.leak(); 314 | } 315 | assert_eq!(&bytes[3..13], &b"Filebuffer"[..]); 316 | } 317 | 318 | #[test] 319 | fn fbuffer_can_be_moved_into_thread() { 320 | use std::thread; 321 | 322 | let fbuffer = FileBuffer::open("src/lib.rs").unwrap(); 323 | thread::spawn(move || { 324 | assert_eq!(&fbuffer[3..13], &b"Filebuffer"[..]); 325 | }); 326 | } 327 | 328 | #[test] 329 | fn fbuffer_can_be_shared_among_threads() { 330 | use std::sync; 331 | use std::thread; 332 | 333 | let fbuffer = FileBuffer::open("src/lib.rs").unwrap(); 334 | let buffer1 = sync::Arc::new(fbuffer); 335 | let buffer2 = buffer1.clone(); 336 | thread::spawn(move || { 337 | assert_eq!(&buffer2[3..13], &b"Filebuffer"[..]); 338 | }); 339 | assert_eq!(&buffer1[17..45], &b"Fast and simple file reading"[..]); 340 | } 341 | 342 | #[test] 343 | fn open_empty_file_is_fine() { 344 | FileBuffer::open("src/empty_file_for_testing.rs").unwrap(); 345 | } 346 | 347 | #[test] 348 | fn empty_file_prefetch_is_fine() { 349 | let fbuffer = FileBuffer::open("src/empty_file_for_testing.rs").unwrap(); 350 | fbuffer.prefetch(0, 0); 351 | } 352 | 353 | #[test] 354 | fn empty_file_deref_is_fine() { 355 | let fbuffer = FileBuffer::open("src/empty_file_for_testing.rs").unwrap(); 356 | assert_eq!(fbuffer.iter().any(|_| true), false); 357 | } 358 | 359 | #[test] 360 | fn empty_file_has_zero_resident_len() { 361 | let fbuffer = FileBuffer::open("src/empty_file_for_testing.rs").unwrap(); 362 | assert_eq!(fbuffer.resident_len(0, 0), 0); 363 | } 364 | 365 | #[test] 366 | fn page_size_at_least_4096() { 367 | // There is no reason why the page size cannot be smaller, it is just that in practice there 368 | // is no platform with a smaller page size, so this tests that `get_page_size()` returns 369 | // a plausible value. 370 | assert!(get_page_size() >= 4096); 371 | } 372 | --------------------------------------------------------------------------------