├── .github ├── dependabot.yml └── workflows │ ├── check-all-targets.yml │ ├── check.yml │ ├── lints.yml │ ├── readme.yml │ └── security.yml ├── .gitignore ├── CHANGELOG.md ├── COPYING ├── Cargo.toml ├── README.md ├── README.tpl ├── ash ├── Cargo.toml └── src │ └── lib.rs ├── erupt ├── Cargo.toml └── src │ └── lib.rs ├── examples ├── Cargo.toml └── src │ ├── ash.rs │ ├── erupt.rs │ ├── mock.rs │ └── transient_reuse.rs ├── gpu-alloc ├── Cargo.toml └── src │ ├── allocator.rs │ ├── block.rs │ ├── buddy.rs │ ├── config.rs │ ├── error.rs │ ├── freelist.rs │ ├── heap.rs │ ├── lib.rs │ ├── slab.rs │ ├── usage.rs │ └── util.rs ├── license ├── APACHE └── MIT ├── mock ├── Cargo.toml └── src │ └── lib.rs └── types ├── Cargo.toml └── src ├── device.rs ├── lib.rs └── types.rs /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "cargo" 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "daily" 12 | - package-ecosystem: "github-actions" 13 | directory: "/" # Location of package manifests 14 | schedule: 15 | interval: "daily" 16 | -------------------------------------------------------------------------------- /.github/workflows/check-all-targets.yml: -------------------------------------------------------------------------------- 1 | name: Check all targets 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | 7 | env: 8 | CARGO_TERM_COLOR: always 9 | 10 | jobs: 11 | check-targets: 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | target: 16 | - i686-pc-windows-gnu 17 | - i686-pc-windows-msvc 18 | - i686-unknown-linux-gnu 19 | - x86_64-apple-darwin 20 | - x86_64-pc-windows-gnu 21 | - x86_64-pc-windows-msvc 22 | - x86_64-unknown-linux-gnu 23 | - wasm32-unknown-unknown 24 | 25 | steps: 26 | - uses: actions/checkout@v3 27 | - name: Install stable toolchain 28 | uses: actions-rs/toolchain@v1 29 | with: 30 | profile: minimal 31 | toolchain: stable 32 | target: ${{ matrix.target }} 33 | - name: Run cargo check 34 | uses: actions-rs/cargo@v1 35 | with: 36 | command: check 37 | args: --all --all-features 38 | -------------------------------------------------------------------------------- /.github/workflows/check.yml: -------------------------------------------------------------------------------- 1 | name: Check 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | types: [ opened, edited ] 8 | branches: [ main ] 9 | paths: 10 | - '**.rs' 11 | - '**/Cargo.toml' 12 | 13 | env: 14 | CARGO_TERM_COLOR: always 15 | 16 | jobs: 17 | test: 18 | runs-on: ubuntu-latest 19 | strategy: 20 | matrix: 21 | rust-toolchain: [stable, nightly] 22 | steps: 23 | - uses: actions/checkout@v3 24 | - name: Install ${{ matrix.rust-toolchain }} toolchain 25 | uses: actions-rs/toolchain@v1 26 | with: 27 | profile: minimal 28 | toolchain: ${{ matrix.rust-toolchain }} 29 | - name: Run cargo test 30 | uses: actions-rs/cargo@v1 31 | with: 32 | command: test 33 | args: --all --all-features 34 | test-1_60: 35 | runs-on: ubuntu-latest 36 | steps: 37 | - uses: actions/checkout@v3 38 | - name: Install 1.60.0 toolchain 39 | uses: actions-rs/toolchain@v1 40 | with: 41 | profile: minimal 42 | toolchain: '1.60.0' 43 | - name: Run cargo test 44 | uses: actions-rs/cargo@v1 45 | with: 46 | command: test 47 | args: --all --lib --all-features 48 | -------------------------------------------------------------------------------- /.github/workflows/lints.yml: -------------------------------------------------------------------------------- 1 | name: Lints 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | types: [ opened, edited ] 8 | branches: [ main ] 9 | paths: 10 | - '**.rs' 11 | - '**/Cargo.toml' 12 | 13 | env: 14 | CARGO_TERM_COLOR: always 15 | 16 | jobs: 17 | fmt: 18 | runs-on: ubuntu-latest 19 | steps: 20 | - uses: actions/checkout@v3 21 | - name: Install nightly toolchain with rustfmt available 22 | uses: actions-rs/toolchain@v1 23 | with: 24 | profile: minimal 25 | toolchain: nightly 26 | components: rustfmt 27 | - name: Run cargo fmt 28 | uses: actions-rs/cargo@v1 29 | with: 30 | command: fmt 31 | args: --all -- --check 32 | 33 | clippy: 34 | runs-on: ubuntu-latest 35 | steps: 36 | - uses: actions/checkout@v3 37 | - name: Install nightly toolchain with clippy available 38 | uses: actions-rs/toolchain@v1 39 | with: 40 | profile: minimal 41 | toolchain: nightly 42 | components: clippy 43 | - name: Run cargo clippy 44 | uses: actions-rs/clippy-check@v1 45 | with: 46 | token: ${{ secrets.GITHUB_TOKEN }} 47 | args: --all --lib --all-features -- -D warnings 48 | -------------------------------------------------------------------------------- /.github/workflows/readme.yml: -------------------------------------------------------------------------------- 1 | name: Generated readme check 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | types: [ opened, edited ] 8 | branches: [ main ] 9 | paths: 10 | - 'README.md' 11 | - 'README.tpl' 12 | - 'gpu-alloc/src/lib.rs' 13 | 14 | env: 15 | CARGO_TERM_COLOR: always 16 | 17 | jobs: 18 | readme: 19 | runs-on: ubuntu-latest 20 | steps: 21 | - uses: actions/checkout@v3 22 | - name: Install cargo-readme 23 | uses: actions-rs/install@v0.1 24 | with: 25 | crate: cargo-readme 26 | use-tool-cache: true 27 | - name: Check that readme is up-to-date 28 | run: 'cd gpu-alloc && [[ ! $(cargo readme --template ../README.tpl | diff - ../README.md) ]]' 29 | -------------------------------------------------------------------------------- /.github/workflows/security.yml: -------------------------------------------------------------------------------- 1 | name: Security audit 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | paths: 9 | - '**/Cargo.toml' 10 | 11 | env: 12 | CARGO_TERM_COLOR: always 13 | 14 | jobs: 15 | security_audit: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v3 19 | - uses: actions-rs/audit-check@v1 20 | with: 21 | token: ${{ secrets.GITHUB_TOKEN }} 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | **/*.rs.bk 3 | Cargo.lock 4 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## [Unreleased] 8 | 9 | ### Added 10 | - Support for `ash` API. 11 | 12 | ### Fixed 13 | - Erupt checks for correct extension to determine buffer device feature availability. 14 | 15 | ## [0.4.7] - 2021-05-22 16 | 17 | ### Fixed 18 | - Add missing block allocation counting for free-list allocator. 19 | 20 | ## [0.4.6] - 2021-05-21 21 | 22 | ### Fixed 23 | - Fixed freeing tail block in its region for FreeBlockAllocator. 24 | 25 | ## [0.4.5] - 2021-05-03 26 | 27 | ### Fixed 28 | - Fixed checking region-block overlap 29 | 30 | ## [0.4.4] - 2021-04-28 31 | ### Changed 32 | - `freelist` feature to use free-list based allocator instead of arena allocator now enabled by default 33 | 34 | ### Fixed 35 | - Fix free-list based allocator 36 | 37 | ## [0.4.3] - 2021-04-20 38 | ### Changed 39 | - Drop check error message is suppressed during unwinding 40 | 41 | ## [0.4.2] - 2021-03-30 42 | ### Fixed 43 | - Allocator type gets back Send and Sync implementations 44 | 45 | ## [0.4.1] - 2021-03-30 [YANKED] 46 | ### Added 47 | - Free-list based allocator to replace arena allocator for better memory reuse. 48 | Active only when feature `freelist` is enabled. 49 | 50 | ### Fixed 51 | - SemVer adhesion 52 | 53 | ## [0.3.1] - 2021-03-27 [YANKED] 54 | ### Fixed 55 | - Typos in public API 56 | 57 | ## [0.3.0] 58 | ### Changed 59 | - Mapping and unmapping now requires mutable reference, 60 | preventing aliasing and double-mapping problems. 61 | - Simple heap budget checking removed as the device may report 62 | significantly smaller heap size. 63 | 64 | 65 | ## [0.2.0] - 2020-11-13 66 | ### Changed 67 | - Allowed to map memory though shared memory block. 68 | 69 | ## [0.1.1] - 2020-11-10 70 | ### Added 71 | - Graphics API agnostic general purpose gpu memory allocator. 72 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | Copyright 2020 The gpu-alloc Project Developers 2 | 3 | Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 6 | copied, modified, or distributed except according to those terms. 7 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = ["types", "gpu-alloc", "mock", "erupt", "examples", "ash"] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gpu-alloc 2 | 3 | [![crates](https://img.shields.io/crates/v/gpu-alloc.svg?style=for-the-badge&label=gpu-alloc)](https://crates.io/crates/gpu-alloc) 4 | [![docs](https://img.shields.io/badge/docs.rs-gpu--alloc-66c2a5?style=for-the-badge&labelColor=555555&logoColor=white)](https://docs.rs/gpu-alloc) 5 | [![actions](https://img.shields.io/github/workflow/status/zakarumych/gpu-alloc/Rust/main?style=for-the-badge)](https://github.com/zakarumych/gpu-alloc/actions?query=workflow%3ARust) 6 | [![MIT/Apache](https://img.shields.io/badge/license-MIT%2FApache-blue.svg?style=for-the-badge)](COPYING) 7 | ![loc](https://img.shields.io/tokei/lines/github/zakarumych/gpu-alloc?style=for-the-badge) 8 | 9 | 10 | Implementation agnostic memory allocator for Vulkan like APIs. 11 | 12 | This crate is intended to be used as part of safe API implementations.\ 13 | Use with caution. There are unsafe functions all over the place. 14 | 15 | ## Usage 16 | 17 | Start with fetching `DeviceProperties` from `gpu-alloc-` crate for the backend of choice.\ 18 | Then create `GpuAllocator` instance and use it for all device memory allocations.\ 19 | `GpuAllocator` will take care for all necessary bookkeeping like memory object count limit, 20 | heap budget and memory mapping. 21 | 22 | #### Backends implementations 23 | 24 | Backend supporting crates should not depend on this crate.\ 25 | Instead they should depend on `gpu-alloc-types` which is much more stable, 26 | allowing to upgrade `gpu-alloc` version without `gpu-alloc-` upgrade. 27 | 28 | 29 | Supported Rust Versions 30 | 31 | The minimum supported version is 1.40. 32 | The current version is not guaranteed to build on Rust versions earlier than the minimum supported version. 33 | 34 | `gpu-alloc-erupt` crate requires version 1.48 or higher due to dependency on `erupt` crate. 35 | 36 | ## License 37 | 38 | Licensed under either of 39 | 40 | * Apache License, Version 2.0, ([license/APACHE](license/APACHE) or http://www.apache.org/licenses/LICENSE-2.0) 41 | * MIT license ([license/MIT](license/MIT) or http://opensource.org/licenses/MIT) 42 | 43 | at your option. 44 | 45 | ## Contributions 46 | 47 | Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. 48 | 49 | ## Donate 50 | 51 | [![Become a patron](https://c5.patreon.com/external/logo/become_a_patron_button.png)](https://www.patreon.com/zakarum) 52 | -------------------------------------------------------------------------------- /README.tpl: -------------------------------------------------------------------------------- 1 | # gpu-alloc 2 | 3 | [![crates](https://img.shields.io/crates/v/gpu-alloc.svg?style=for-the-badge&label=gpu-alloc)](https://crates.io/crates/gpu-alloc) 4 | [![docs](https://img.shields.io/badge/docs.rs-gpu--alloc-66c2a5?style=for-the-badge&labelColor=555555&logoColor=white)](https://docs.rs/gpu-alloc) 5 | [![actions](https://img.shields.io/github/workflow/status/zakarumych/gpu-alloc/Rust/main?style=for-the-badge)](https://github.com/zakarumych/gpu-alloc/actions?query=workflow%3ARust) 6 | [![MIT/Apache](https://img.shields.io/badge/license-MIT%2FApache-blue.svg?style=for-the-badge)](COPYING) 7 | ![loc](https://img.shields.io/tokei/lines/github/zakarumych/gpu-alloc?style=for-the-badge) 8 | 9 | {{readme}} 10 | 11 | Supported Rust Versions 12 | 13 | The minimum supported version is 1.40. 14 | The current version is not guaranteed to build on Rust versions earlier than the minimum supported version. 15 | 16 | `gpu-alloc-erupt` crate requires version 1.48 or higher due to dependency on `erupt` crate. 17 | 18 | ## License 19 | 20 | Licensed under either of 21 | 22 | * Apache License, Version 2.0, ([license/APACHE](license/APACHE) or http://www.apache.org/licenses/LICENSE-2.0) 23 | * MIT license ([license/MIT](license/MIT) or http://opensource.org/licenses/MIT) 24 | 25 | at your option. 26 | 27 | ## Contributions 28 | 29 | Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. 30 | 31 | ## Donate 32 | 33 | [![Become a patron](https://c5.patreon.com/external/logo/become_a_patron_button.png)](https://www.patreon.com/zakarum) 34 | -------------------------------------------------------------------------------- /ash/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "gpu-alloc-ash" 3 | version = "0.7.0" 4 | authors = ["Zakarum "] 5 | edition = "2018" 6 | description = "`ash` backend for `gpu-alloc`" 7 | documentation = "https://docs.rs/gpu-alloc-ash" 8 | readme = "../README.md" 9 | homepage = "https://github.com/zakarumych/gpu-alloc" 10 | repository = "https://github.com/zakarumych/gpu-alloc" 11 | license = "MIT OR Apache-2.0" 12 | keywords = ["gpu", "vulkan", "allocation", "no-std"] 13 | categories = ["graphics", "memory-management", "no-std", "game-development"] 14 | 15 | [dependencies] 16 | gpu-alloc-types = { path = "../types", version = "=0.3.0" } 17 | tracing = { version = "0.1", features = ["attributes"], optional = true } 18 | ash = { version = "0.38", default-features = false } 19 | tinyvec = { version = "1.0", default-features = false, features = ["alloc"] } 20 | -------------------------------------------------------------------------------- /ash/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Ash backend for `gpu-alloc` 3 | //! 4 | //! # Usage example 5 | //! 6 | //! ```ignore 7 | //! use { 8 | //! ash::{vk, DefaultEntryLoader, DeviceLoader, InstanceLoader}, 9 | //! gpu_alloc::{Config, GpuAllocator, Request, UsageFlags}, 10 | //! gpu_alloc_ash::{device_properties, AshMemoryDevice}, 11 | //! std::ffi::CStr, 12 | //! }; 13 | //! 14 | //! fn main() -> eyre::Result<()> { 15 | //! color_eyre::install()?; 16 | //! 17 | //! let entry = DefaultEntryLoader::new()?; 18 | //! 19 | //! let instance = InstanceLoader::new( 20 | //! &entry, 21 | //! &vk::InstanceCreateInfo::default() 22 | //! .into_builder() 23 | //! .application_info( 24 | //! &vk::ApplicationInfo::default() 25 | //! .into_builder() 26 | //! .engine_name(CStr::from_bytes_with_nul(b"GpuAlloc\0").unwrap()) 27 | //! .engine_version(1) 28 | //! .application_name(CStr::from_bytes_with_nul(b"GpuAllocApp\0").unwrap()) 29 | //! .application_version(1) 30 | //! .api_version(entry.instance_version()), 31 | //! ), 32 | //! None, 33 | //! )?; 34 | //! 35 | //! let physical_devices = unsafe { instance.enumerate_physical_devices(None) }.result()?; 36 | //! let physical_device = physical_devices[0]; 37 | //! 38 | //! let props = unsafe { device_properties(&instance, physical_device) }?; 39 | //! 40 | //! let device = DeviceLoader::new( 41 | //! &instance, 42 | //! physical_device, 43 | //! &vk::DeviceCreateInfoBuilder::new().queue_create_infos(&[ 44 | //! vk::DeviceQueueCreateInfoBuilder::new() 45 | //! .queue_family_index(0) 46 | //! .queue_priorities(&[0f32]), 47 | //! ]), 48 | //! None, 49 | //! )?; 50 | //! 51 | //! let config = Config::i_am_potato(); 52 | //! 53 | //! let mut allocator = GpuAllocator::new(config, props); 54 | //! 55 | //! let mut block = unsafe { 56 | //! allocator.alloc( 57 | //! AshMemoryDevice::wrap(&device), 58 | //! Request { 59 | //! size: 10, 60 | //! align_mask: 1, 61 | //! usage: UsageFlags::HOST_ACCESS, 62 | //! memory_types: !0, 63 | //! }, 64 | //! ) 65 | //! }?; 66 | //! 67 | //! unsafe { 68 | //! block.write_bytes( 69 | //! AshMemoryDevice::wrap(&device), 70 | //! 0, 71 | //! &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 72 | //! ) 73 | //! }?; 74 | //! 75 | //! unsafe { allocator.dealloc(AshMemoryDevice::wrap(&device), block) } 76 | //! 77 | //! // the `ash::Device` also implements `AsRef` 78 | //! // you can pass a reference of `ash::Device` directly as argument 79 | //! let mut block = unsafe { 80 | //! allocator.alloc( 81 | //! &device, 82 | //! Request { 83 | //! size: 10, 84 | //! align_mask: 1, 85 | //! usage: UsageFlags::HOST_ACCESS, 86 | //! memory_types: !0, 87 | //! }, 88 | //! ) 89 | //! }?; 90 | //! 91 | //! unsafe { 92 | //! block.write_bytes( 93 | //! &device, 94 | //! 0, 95 | //! &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 96 | //! ) 97 | //! }?; 98 | //! 99 | //! unsafe { allocator.dealloc(&device, block) } 100 | //! 101 | //! Ok(()) 102 | //! } 103 | //! ``` 104 | //! 105 | 106 | use { 107 | ash::{vk, Device, Instance}, 108 | gpu_alloc_types::{ 109 | AllocationFlags, DeviceMapError, DeviceProperties, MappedMemoryRange, MemoryDevice, 110 | MemoryHeap, MemoryPropertyFlags, MemoryType, OutOfMemory, 111 | }, 112 | std::ptr::NonNull, 113 | tinyvec::TinyVec, 114 | }; 115 | 116 | #[repr(transparent)] 117 | pub struct AshMemoryDevice { 118 | device: Device, 119 | } 120 | 121 | impl AshMemoryDevice { 122 | pub fn wrap(device: &Device) -> &Self { 123 | unsafe { 124 | // Safe because `Self` is `repr(transparent)` 125 | // with only field being `DeviceLoader`. 126 | &*(device as *const Device as *const Self) 127 | } 128 | } 129 | } 130 | 131 | impl AsRef for Device { 132 | #[inline(always)] 133 | fn as_ref(&self) -> &AshMemoryDevice { 134 | AshMemoryDevice::wrap(self) 135 | } 136 | } 137 | 138 | // AsRef does not have a blanket implementation. need to add this impl so that 139 | // old user code (i.e. explicit wrap) still compiles without any change 140 | impl AsRef for AshMemoryDevice { 141 | #[inline(always)] 142 | fn as_ref(&self) -> &AshMemoryDevice { 143 | self 144 | } 145 | } 146 | 147 | impl MemoryDevice for AshMemoryDevice { 148 | #[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))] 149 | unsafe fn allocate_memory( 150 | &self, 151 | size: u64, 152 | memory_type: u32, 153 | flags: AllocationFlags, 154 | ) -> Result { 155 | assert!((flags & !(AllocationFlags::DEVICE_ADDRESS)).is_empty()); 156 | 157 | let mut info = vk::MemoryAllocateInfo::default() 158 | .allocation_size(size) 159 | .memory_type_index(memory_type); 160 | 161 | let mut info_flags; 162 | 163 | if flags.contains(AllocationFlags::DEVICE_ADDRESS) { 164 | info_flags = vk::MemoryAllocateFlagsInfo::default() 165 | .flags(vk::MemoryAllocateFlags::DEVICE_ADDRESS); 166 | info = info.push_next(&mut info_flags); 167 | } 168 | 169 | match self.device.allocate_memory(&info, None) { 170 | Ok(memory) => Ok(memory), 171 | Err(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY) => Err(OutOfMemory::OutOfDeviceMemory), 172 | Err(vk::Result::ERROR_OUT_OF_HOST_MEMORY) => Err(OutOfMemory::OutOfHostMemory), 173 | Err(vk::Result::ERROR_TOO_MANY_OBJECTS) => panic!("Too many objects"), 174 | Err(err) => panic!("Unexpected Vulkan error: `{}`", err), 175 | } 176 | } 177 | 178 | #[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))] 179 | unsafe fn deallocate_memory(&self, memory: vk::DeviceMemory) { 180 | self.device.free_memory(memory, None); 181 | } 182 | 183 | #[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))] 184 | unsafe fn map_memory( 185 | &self, 186 | memory: &mut vk::DeviceMemory, 187 | offset: u64, 188 | size: u64, 189 | ) -> Result, DeviceMapError> { 190 | match self 191 | .device 192 | .map_memory(*memory, offset, size, vk::MemoryMapFlags::empty()) 193 | { 194 | Ok(ptr) => { 195 | Ok(NonNull::new(ptr as *mut u8) 196 | .expect("Pointer to memory mapping must not be null")) 197 | } 198 | Err(vk::Result::ERROR_OUT_OF_DEVICE_MEMORY) => Err(DeviceMapError::OutOfDeviceMemory), 199 | Err(vk::Result::ERROR_OUT_OF_HOST_MEMORY) => Err(DeviceMapError::OutOfHostMemory), 200 | Err(vk::Result::ERROR_MEMORY_MAP_FAILED) => Err(DeviceMapError::MapFailed), 201 | Err(err) => panic!("Unexpected Vulkan error: `{}`", err), 202 | } 203 | } 204 | 205 | #[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))] 206 | unsafe fn unmap_memory(&self, memory: &mut vk::DeviceMemory) { 207 | self.device.unmap_memory(*memory); 208 | } 209 | 210 | #[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))] 211 | unsafe fn invalidate_memory_ranges( 212 | &self, 213 | ranges: &[MappedMemoryRange<'_, vk::DeviceMemory>], 214 | ) -> Result<(), OutOfMemory> { 215 | self.device 216 | .invalidate_mapped_memory_ranges( 217 | &ranges 218 | .iter() 219 | .map(|range| { 220 | vk::MappedMemoryRange::default() 221 | .memory(*range.memory) 222 | .offset(range.offset) 223 | .size(range.size) 224 | }) 225 | .collect::>(), 226 | ) 227 | .map_err(|err| match err { 228 | vk::Result::ERROR_OUT_OF_DEVICE_MEMORY => OutOfMemory::OutOfDeviceMemory, 229 | vk::Result::ERROR_OUT_OF_HOST_MEMORY => OutOfMemory::OutOfHostMemory, 230 | err => panic!("Unexpected Vulkan error: `{}`", err), 231 | }) 232 | } 233 | 234 | #[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))] 235 | unsafe fn flush_memory_ranges( 236 | &self, 237 | ranges: &[MappedMemoryRange<'_, vk::DeviceMemory>], 238 | ) -> Result<(), OutOfMemory> { 239 | self.device 240 | .flush_mapped_memory_ranges( 241 | &ranges 242 | .iter() 243 | .map(|range| { 244 | vk::MappedMemoryRange::default() 245 | .memory(*range.memory) 246 | .offset(range.offset) 247 | .size(range.size) 248 | }) 249 | .collect::>(), 250 | ) 251 | .map_err(|err| match err { 252 | vk::Result::ERROR_OUT_OF_DEVICE_MEMORY => OutOfMemory::OutOfDeviceMemory, 253 | vk::Result::ERROR_OUT_OF_HOST_MEMORY => OutOfMemory::OutOfHostMemory, 254 | err => panic!("Unexpected Vulkan error: `{}`", err), 255 | }) 256 | } 257 | } 258 | 259 | /// Returns `DeviceProperties` from ash's `InstanceLoader` for specified `PhysicalDevice`, required to create `GpuAllocator`. 260 | /// 261 | /// # Safety 262 | /// 263 | /// `physical_device` must be queried from `Instance` associated with this `instance`. 264 | /// Even if returned properties' field `buffer_device_address` is set to true, 265 | /// feature `PhysicalDeviceBufferDeviceAddressFeatures::buffer_derive_address` must be enabled explicitly on device creation 266 | /// and extension "VK_KHR_buffer_device_address" for Vulkan prior 1.2. 267 | /// Otherwise the field must be set to false before passing to `GpuAllocator::new`. 268 | pub unsafe fn device_properties( 269 | instance: &Instance, 270 | version: u32, 271 | physical_device: vk::PhysicalDevice, 272 | ) -> Result, vk::Result> { 273 | use ash::vk::PhysicalDeviceFeatures2; 274 | 275 | let limits = instance 276 | .get_physical_device_properties(physical_device) 277 | .limits; 278 | 279 | let memory_properties = instance.get_physical_device_memory_properties(physical_device); 280 | 281 | let buffer_device_address = 282 | if vk::api_version_major(version) >= 1 && vk::api_version_minor(version) >= 2 { 283 | let mut features = PhysicalDeviceFeatures2::default(); 284 | let mut bda_features = vk::PhysicalDeviceBufferDeviceAddressFeatures::default(); 285 | features.p_next = 286 | &mut bda_features as *mut vk::PhysicalDeviceBufferDeviceAddressFeatures as *mut _; 287 | instance.get_physical_device_features2(physical_device, &mut features); 288 | bda_features.buffer_device_address != 0 289 | } else { 290 | false 291 | }; 292 | 293 | Ok(DeviceProperties { 294 | max_memory_allocation_count: limits.max_memory_allocation_count, 295 | max_memory_allocation_size: u64::max_value(), // FIXME: Can query this information if instance is v1.1 296 | non_coherent_atom_size: limits.non_coherent_atom_size, 297 | memory_types: memory_properties.memory_types 298 | [..memory_properties.memory_type_count as usize] 299 | .iter() 300 | .map(|memory_type| MemoryType { 301 | props: memory_properties_from_ash(memory_type.property_flags), 302 | heap: memory_type.heap_index, 303 | }) 304 | .collect(), 305 | memory_heaps: memory_properties.memory_heaps 306 | [..memory_properties.memory_heap_count as usize] 307 | .iter() 308 | .map(|&memory_heap| MemoryHeap { 309 | size: memory_heap.size, 310 | }) 311 | .collect(), 312 | buffer_device_address, 313 | }) 314 | } 315 | 316 | pub fn memory_properties_from_ash(props: vk::MemoryPropertyFlags) -> MemoryPropertyFlags { 317 | let mut result = MemoryPropertyFlags::empty(); 318 | if props.contains(vk::MemoryPropertyFlags::DEVICE_LOCAL) { 319 | result |= MemoryPropertyFlags::DEVICE_LOCAL; 320 | } 321 | if props.contains(vk::MemoryPropertyFlags::HOST_VISIBLE) { 322 | result |= MemoryPropertyFlags::HOST_VISIBLE; 323 | } 324 | if props.contains(vk::MemoryPropertyFlags::HOST_COHERENT) { 325 | result |= MemoryPropertyFlags::HOST_COHERENT; 326 | } 327 | if props.contains(vk::MemoryPropertyFlags::HOST_CACHED) { 328 | result |= MemoryPropertyFlags::HOST_CACHED; 329 | } 330 | if props.contains(vk::MemoryPropertyFlags::LAZILY_ALLOCATED) { 331 | result |= MemoryPropertyFlags::LAZILY_ALLOCATED; 332 | } 333 | result 334 | } 335 | 336 | pub fn memory_properties_to_ash(props: MemoryPropertyFlags) -> vk::MemoryPropertyFlags { 337 | let mut result = vk::MemoryPropertyFlags::empty(); 338 | if props.contains(MemoryPropertyFlags::DEVICE_LOCAL) { 339 | result |= vk::MemoryPropertyFlags::DEVICE_LOCAL; 340 | } 341 | if props.contains(MemoryPropertyFlags::HOST_VISIBLE) { 342 | result |= vk::MemoryPropertyFlags::HOST_VISIBLE; 343 | } 344 | if props.contains(MemoryPropertyFlags::HOST_COHERENT) { 345 | result |= vk::MemoryPropertyFlags::HOST_COHERENT; 346 | } 347 | if props.contains(MemoryPropertyFlags::HOST_CACHED) { 348 | result |= vk::MemoryPropertyFlags::HOST_CACHED; 349 | } 350 | if props.contains(MemoryPropertyFlags::LAZILY_ALLOCATED) { 351 | result |= vk::MemoryPropertyFlags::LAZILY_ALLOCATED; 352 | } 353 | result 354 | } 355 | -------------------------------------------------------------------------------- /erupt/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "gpu-alloc-erupt" 3 | version = "0.9.0" 4 | authors = ["Zakarum "] 5 | edition = "2018" 6 | description = "`erupt` backend for `gpu-alloc`" 7 | documentation = "https://docs.rs/gpu-alloc-erupt" 8 | readme = "../README.md" 9 | homepage = "https://github.com/zakarumych/gpu-alloc" 10 | repository = "https://github.com/zakarumych/gpu-alloc" 11 | license = "MIT OR Apache-2.0" 12 | keywords = ["gpu", "vulkan", "allocation", "no-std"] 13 | categories = ["graphics", "memory-management", "no-std", "game-development"] 14 | 15 | [dependencies] 16 | gpu-alloc-types = { path = "../types", version = "=0.3.0" } 17 | tracing = { version = "0.1", features = ["attributes"], optional = true } 18 | erupt = { version = "0.23.0", default-features = false, features = ["loading"] } 19 | tinyvec = { version = "1.0", default-features = false, features = ["alloc"] } 20 | -------------------------------------------------------------------------------- /erupt/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! # Erupt backend for `gpu-alloc` 3 | //! 4 | //! # Usage example 5 | //! 6 | //! ```ignore 7 | //! use { 8 | //! erupt::{vk1_0, DeviceLoader, EntryLoader, InstanceLoader}, 9 | //! gpu_alloc::{Config, GpuAllocator, Request, UsageFlags}, 10 | //! gpu_alloc_erupt::{device_properties, EruptMemoryDevice}, 11 | //! std::ffi::CStr, 12 | //! }; 13 | //! 14 | //! fn main() -> eyre::Result<()> { 15 | //! color_eyre::install()?; 16 | //! 17 | //! let entry = EntryLoader::new()?; 18 | //! 19 | //! let instance = unsafe { 20 | //! InstanceLoader::new( 21 | //! &entry, 22 | //! &vk1_0::InstanceCreateInfo::default() 23 | //! .into_builder() 24 | //! .application_info( 25 | //! &vk1_0::ApplicationInfo::default() 26 | //! .into_builder() 27 | //! .engine_name(CStr::from_bytes_with_nul(b"GpuAlloc\0").unwrap()) 28 | //! .engine_version(1) 29 | //! .application_name(CStr::from_bytes_with_nul(b"GpuAllocApp\0").unwrap()) 30 | //! .application_version(1) 31 | //! .api_version(entry.instance_version()), 32 | //! ), 33 | //! None, 34 | //! ) 35 | //! }?; 36 | //! 37 | //! let physical_devices = unsafe { instance.enumerate_physical_devices(None) }.result()?; 38 | //! let physical_device = physical_devices[0]; 39 | //! 40 | //! let props = unsafe { device_properties(&instance, physical_device) }?; 41 | //! 42 | //! let device = unsafe { 43 | //! DeviceLoader::new( 44 | //! &instance, 45 | //! physical_device, 46 | //! &vk1_0::DeviceCreateInfoBuilder::new().queue_create_infos(&[ 47 | //! vk1_0::DeviceQueueCreateInfoBuilder::new() 48 | //! .queue_family_index(0) 49 | //! .queue_priorities(&[0f32]), 50 | //! ]), 51 | //! None, 52 | //! ) 53 | //! }?; 54 | //! 55 | //! let config = Config::i_am_potato(); 56 | //! 57 | //! let mut allocator = GpuAllocator::new(config, props); 58 | //! 59 | //! let mut block = unsafe { 60 | //! allocator.alloc( 61 | //! EruptMemoryDevice::wrap(&device), 62 | //! Request { 63 | //! size: 10, 64 | //! align_mask: 1, 65 | //! usage: UsageFlags::HOST_ACCESS, 66 | //! memory_types: !0, 67 | //! }, 68 | //! ) 69 | //! }?; 70 | //! 71 | //! unsafe { 72 | //! block.write_bytes( 73 | //! EruptMemoryDevice::wrap(&device), 74 | //! 0, 75 | //! &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 76 | //! ) 77 | //! }?; 78 | //! 79 | //! unsafe { allocator.dealloc(EruptMemoryDevice::wrap(&device), block) } 80 | //! 81 | //! // the `erupt::DeviceLoader` also implements `AsRef` 82 | //! // you can pass a reference of `erupt::DeviceLoader` directly as argument 83 | //! let mut block = unsafe { 84 | //! allocator.alloc( 85 | //! &device, 86 | //! Request { 87 | //! size: 10, 88 | //! align_mask: 1, 89 | //! usage: UsageFlags::HOST_ACCESS, 90 | //! memory_types: !0, 91 | //! }, 92 | //! ) 93 | //! }?; 94 | //! 95 | //! unsafe { 96 | //! block.write_bytes( 97 | //! &device, 98 | //! 0, 99 | //! &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 100 | //! ) 101 | //! }?; 102 | //! 103 | //! unsafe { allocator.dealloc(&device, block) } 104 | //! 105 | //! Ok(()) 106 | //! } 107 | //! ``` 108 | //! 109 | 110 | use std::ptr::NonNull; 111 | 112 | use erupt::{vk::MemoryMapFlags, vk1_0, vk1_1, DeviceLoader, ExtendableFrom, InstanceLoader}; 113 | use gpu_alloc_types::{ 114 | AllocationFlags, DeviceMapError, DeviceProperties, MappedMemoryRange, MemoryDevice, MemoryHeap, 115 | MemoryPropertyFlags, MemoryType, OutOfMemory, 116 | }; 117 | use tinyvec::TinyVec; 118 | 119 | #[repr(transparent)] 120 | pub struct EruptMemoryDevice { 121 | device: DeviceLoader, 122 | } 123 | 124 | impl EruptMemoryDevice { 125 | pub fn wrap(device: &DeviceLoader) -> &Self { 126 | unsafe { 127 | // Safe because `Self` is `repr(transparent)` 128 | // with only field being `DeviceLoader`. 129 | &*(device as *const DeviceLoader as *const Self) 130 | } 131 | } 132 | } 133 | 134 | impl AsRef for DeviceLoader { 135 | #[inline(always)] 136 | fn as_ref(&self) -> &EruptMemoryDevice { 137 | EruptMemoryDevice::wrap(self) 138 | } 139 | } 140 | 141 | impl AsRef for EruptMemoryDevice { 142 | #[inline(always)] 143 | fn as_ref(&self) -> &EruptMemoryDevice { 144 | self 145 | } 146 | } 147 | 148 | impl MemoryDevice for EruptMemoryDevice { 149 | #[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))] 150 | unsafe fn allocate_memory( 151 | &self, 152 | size: u64, 153 | memory_type: u32, 154 | flags: AllocationFlags, 155 | ) -> Result { 156 | assert!((flags & !(AllocationFlags::DEVICE_ADDRESS)).is_empty()); 157 | 158 | let mut info = vk1_0::MemoryAllocateInfoBuilder::new() 159 | .allocation_size(size) 160 | .memory_type_index(memory_type); 161 | 162 | let mut info_flags; 163 | 164 | if flags.contains(AllocationFlags::DEVICE_ADDRESS) { 165 | info_flags = vk1_1::MemoryAllocateFlagsInfoBuilder::new() 166 | .flags(vk1_1::MemoryAllocateFlags::DEVICE_ADDRESS); 167 | info = info.extend_from(&mut info_flags); 168 | } 169 | 170 | match self.device.allocate_memory(&info, None).result() { 171 | Ok(memory) => Ok(memory), 172 | Err(vk1_0::Result::ERROR_OUT_OF_DEVICE_MEMORY) => Err(OutOfMemory::OutOfDeviceMemory), 173 | Err(vk1_0::Result::ERROR_OUT_OF_HOST_MEMORY) => Err(OutOfMemory::OutOfHostMemory), 174 | Err(vk1_0::Result::ERROR_TOO_MANY_OBJECTS) => panic!("Too many objects"), 175 | Err(err) => panic!("Unexpected Vulkan error: `{}`", err), 176 | } 177 | } 178 | 179 | #[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))] 180 | unsafe fn deallocate_memory(&self, memory: vk1_0::DeviceMemory) { 181 | self.device.free_memory(memory, None); 182 | } 183 | 184 | #[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))] 185 | unsafe fn map_memory( 186 | &self, 187 | memory: &mut vk1_0::DeviceMemory, 188 | offset: u64, 189 | size: u64, 190 | ) -> Result, DeviceMapError> { 191 | match self 192 | .device 193 | .map_memory(*memory, offset, size, MemoryMapFlags::empty()) 194 | .result() 195 | { 196 | Ok(ptr) => { 197 | Ok(NonNull::new(ptr as *mut u8) 198 | .expect("Pointer to memory mapping must not be null")) 199 | } 200 | Err(vk1_0::Result::ERROR_OUT_OF_DEVICE_MEMORY) => { 201 | Err(DeviceMapError::OutOfDeviceMemory) 202 | } 203 | Err(vk1_0::Result::ERROR_OUT_OF_HOST_MEMORY) => Err(DeviceMapError::OutOfHostMemory), 204 | Err(vk1_0::Result::ERROR_MEMORY_MAP_FAILED) => Err(DeviceMapError::MapFailed), 205 | Err(err) => panic!("Unexpected Vulkan error: `{}`", err), 206 | } 207 | } 208 | 209 | #[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))] 210 | unsafe fn unmap_memory(&self, memory: &mut vk1_0::DeviceMemory) { 211 | self.device.unmap_memory(*memory); 212 | } 213 | 214 | #[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))] 215 | unsafe fn invalidate_memory_ranges( 216 | &self, 217 | ranges: &[MappedMemoryRange<'_, vk1_0::DeviceMemory>], 218 | ) -> Result<(), OutOfMemory> { 219 | self.device 220 | .invalidate_mapped_memory_ranges( 221 | &ranges 222 | .iter() 223 | .map(|range| { 224 | vk1_0::MappedMemoryRangeBuilder::new() 225 | .memory(*range.memory) 226 | .offset(range.offset) 227 | .size(range.size) 228 | }) 229 | .collect::>(), 230 | ) 231 | .result() 232 | .map_err(|err| match err { 233 | vk1_0::Result::ERROR_OUT_OF_DEVICE_MEMORY => OutOfMemory::OutOfDeviceMemory, 234 | vk1_0::Result::ERROR_OUT_OF_HOST_MEMORY => OutOfMemory::OutOfHostMemory, 235 | err => panic!("Unexpected Vulkan error: `{}`", err), 236 | }) 237 | } 238 | 239 | #[cfg_attr(feature = "tracing", tracing::instrument(skip(self)))] 240 | unsafe fn flush_memory_ranges( 241 | &self, 242 | ranges: &[MappedMemoryRange<'_, vk1_0::DeviceMemory>], 243 | ) -> Result<(), OutOfMemory> { 244 | self.device 245 | .flush_mapped_memory_ranges( 246 | &ranges 247 | .iter() 248 | .map(|range| { 249 | vk1_0::MappedMemoryRangeBuilder::new() 250 | .memory(*range.memory) 251 | .offset(range.offset) 252 | .size(range.size) 253 | }) 254 | .collect::>(), 255 | ) 256 | .result() 257 | .map_err(|err| match err { 258 | vk1_0::Result::ERROR_OUT_OF_DEVICE_MEMORY => OutOfMemory::OutOfDeviceMemory, 259 | vk1_0::Result::ERROR_OUT_OF_HOST_MEMORY => OutOfMemory::OutOfHostMemory, 260 | err => panic!("Unexpected Vulkan error: `{}`", err), 261 | }) 262 | } 263 | } 264 | 265 | /// Returns `DeviceProperties` from erupt's `InstanceLoader` for specified `PhysicalDevice`, required to create `GpuAllocator`. 266 | /// 267 | /// # Safety 268 | /// 269 | /// `physical_device` must be queried from `Instance` associated with this `instance`. 270 | /// Even if returned properties' field `buffer_device_address` is set to true, 271 | /// feature `PhysicalDeviceBufferDeviceAddressFeatures::buffer_derive_address` must be enabled explicitly on device creation 272 | /// and extension "VK_KHR_buffer_device_address" for Vulkan prior 1.2. 273 | /// Otherwise the field must be set to false before passing to `GpuAllocator::new`. 274 | pub unsafe fn device_properties( 275 | instance: &InstanceLoader, 276 | physical_device: vk1_0::PhysicalDevice, 277 | ) -> Result, vk1_0::Result> { 278 | use { 279 | erupt::{ 280 | extensions::khr_buffer_device_address::KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME, 281 | vk1_1::PhysicalDeviceFeatures2, vk1_2::PhysicalDeviceBufferDeviceAddressFeatures, 282 | }, 283 | std::ffi::CStr, 284 | }; 285 | 286 | let limits = instance 287 | .get_physical_device_properties(physical_device) 288 | .limits; 289 | 290 | let memory_properties = instance.get_physical_device_memory_properties(physical_device); 291 | 292 | let buffer_device_address = 293 | if instance.enabled().vk1_1 || instance.enabled().khr_get_physical_device_properties2 { 294 | let mut bda_features_available = instance.enabled().vk1_2; 295 | 296 | if !bda_features_available { 297 | let extensions = instance 298 | .enumerate_device_extension_properties(physical_device, None, None) 299 | .result()?; 300 | 301 | bda_features_available = extensions.iter().any(|ext| { 302 | let name = CStr::from_bytes_with_nul({ 303 | std::slice::from_raw_parts( 304 | ext.extension_name.as_ptr() as *const _, 305 | ext.extension_name.len(), 306 | ) 307 | }); 308 | if let Ok(name) = name { 309 | name == { CStr::from_ptr(KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME) } 310 | } else { 311 | false 312 | } 313 | }); 314 | } 315 | 316 | if bda_features_available { 317 | let features = PhysicalDeviceFeatures2::default().into_builder(); 318 | let mut bda_features = PhysicalDeviceBufferDeviceAddressFeatures::default(); 319 | let mut features = features.extend_from(&mut bda_features); 320 | instance.get_physical_device_features2(physical_device, &mut features); 321 | bda_features.buffer_device_address != 0 322 | } else { 323 | false 324 | } 325 | } else { 326 | false 327 | }; 328 | 329 | Ok(DeviceProperties { 330 | max_memory_allocation_count: limits.max_memory_allocation_count, 331 | max_memory_allocation_size: u64::max_value(), // FIXME: Can query this information if instance is v1.1 332 | non_coherent_atom_size: limits.non_coherent_atom_size, 333 | memory_types: memory_properties.memory_types 334 | [..memory_properties.memory_type_count as usize] 335 | .iter() 336 | .map(|memory_type| MemoryType { 337 | props: memory_properties_from_erupt(memory_type.property_flags), 338 | heap: memory_type.heap_index, 339 | }) 340 | .collect(), 341 | memory_heaps: memory_properties.memory_heaps 342 | [..memory_properties.memory_heap_count as usize] 343 | .iter() 344 | .map(|&memory_heap| MemoryHeap { 345 | size: memory_heap.size, 346 | }) 347 | .collect(), 348 | buffer_device_address, 349 | }) 350 | } 351 | 352 | pub fn memory_properties_from_erupt(props: vk1_0::MemoryPropertyFlags) -> MemoryPropertyFlags { 353 | let mut result = MemoryPropertyFlags::empty(); 354 | if props.contains(vk1_0::MemoryPropertyFlags::DEVICE_LOCAL) { 355 | result |= MemoryPropertyFlags::DEVICE_LOCAL; 356 | } 357 | if props.contains(vk1_0::MemoryPropertyFlags::HOST_VISIBLE) { 358 | result |= MemoryPropertyFlags::HOST_VISIBLE; 359 | } 360 | if props.contains(vk1_0::MemoryPropertyFlags::HOST_COHERENT) { 361 | result |= MemoryPropertyFlags::HOST_COHERENT; 362 | } 363 | if props.contains(vk1_0::MemoryPropertyFlags::HOST_CACHED) { 364 | result |= MemoryPropertyFlags::HOST_CACHED; 365 | } 366 | if props.contains(vk1_0::MemoryPropertyFlags::LAZILY_ALLOCATED) { 367 | result |= MemoryPropertyFlags::LAZILY_ALLOCATED; 368 | } 369 | result 370 | } 371 | 372 | pub fn memory_properties_to_erupt(props: MemoryPropertyFlags) -> vk1_0::MemoryPropertyFlags { 373 | let mut result = vk1_0::MemoryPropertyFlags::empty(); 374 | if props.contains(MemoryPropertyFlags::DEVICE_LOCAL) { 375 | result |= vk1_0::MemoryPropertyFlags::DEVICE_LOCAL; 376 | } 377 | if props.contains(MemoryPropertyFlags::HOST_VISIBLE) { 378 | result |= vk1_0::MemoryPropertyFlags::HOST_VISIBLE; 379 | } 380 | if props.contains(MemoryPropertyFlags::HOST_COHERENT) { 381 | result |= vk1_0::MemoryPropertyFlags::HOST_COHERENT; 382 | } 383 | if props.contains(MemoryPropertyFlags::HOST_CACHED) { 384 | result |= vk1_0::MemoryPropertyFlags::HOST_CACHED; 385 | } 386 | if props.contains(MemoryPropertyFlags::LAZILY_ALLOCATED) { 387 | result |= vk1_0::MemoryPropertyFlags::LAZILY_ALLOCATED; 388 | } 389 | result 390 | } 391 | -------------------------------------------------------------------------------- /examples/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "gpu-alloc-basic-example" 3 | version = "0.1.0" 4 | authors = ["Zakarum "] 5 | edition = "2018" 6 | publish = false 7 | 8 | [features] 9 | mock = ["gpu-alloc-mock"] 10 | 11 | [dependencies] 12 | gpu-alloc = { path = "../gpu-alloc", version = "=0.6.0", features = [ 13 | "tracing", 14 | ] } 15 | eyre = "0.6" 16 | color-eyre = "0.6" 17 | gpu-alloc-mock = { path = "../mock", version = "=0.3", optional = true } 18 | gpu-alloc-erupt = { path = "../erupt", version = "=0.9", optional = true } 19 | erupt = { version = "0.23.0", optional = true, features = ["loading"] } 20 | gpu-alloc-ash = { path = "../ash", version = "=0.7", optional = true } 21 | ash = { version = "0.38", default-features = false, features = [ 22 | "loaded", 23 | ], optional = true } 24 | 25 | tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } 26 | tracing-error = { version = "0.2" } 27 | tracing = { version = "0.1" } 28 | 29 | [[bin]] 30 | name = "mock" 31 | path = "src/mock.rs" 32 | required-features = ["mock"] 33 | 34 | [[bin]] 35 | name = "erupt" 36 | path = "src/erupt.rs" 37 | required-features = ["gpu-alloc-erupt", "erupt"] 38 | 39 | [[bin]] 40 | name = "ash" 41 | path = "src/ash.rs" 42 | required-features = ["gpu-alloc-ash", "ash"] 43 | 44 | [[bin]] 45 | name = "transient-reuse" 46 | path = "src/transient_reuse.rs" 47 | required-features = ["mock"] 48 | -------------------------------------------------------------------------------- /examples/src/ash.rs: -------------------------------------------------------------------------------- 1 | use { 2 | ash::{vk, Entry}, 3 | gpu_alloc::{Config, GpuAllocator, Request, UsageFlags}, 4 | gpu_alloc_ash::{device_properties, AshMemoryDevice}, 5 | std::ffi::CStr, 6 | }; 7 | 8 | fn main() -> eyre::Result<()> { 9 | color_eyre::install()?; 10 | 11 | let entry = unsafe { Entry::load() }?; 12 | 13 | let version = entry 14 | .try_enumerate_instance_version()? 15 | .unwrap_or(vk::make_api_version(0, 1, 0, 0)); 16 | 17 | let instance = unsafe { 18 | entry.create_instance( 19 | &vk::InstanceCreateInfo::builder().application_info( 20 | &vk::ApplicationInfo::builder() 21 | .engine_name(CStr::from_bytes_with_nul(b"GpuAlloc\0").unwrap()) 22 | .engine_version(1) 23 | .application_name(CStr::from_bytes_with_nul(b"GpuAllocApp\0").unwrap()) 24 | .application_version(1) 25 | .api_version(version), 26 | ), 27 | None, 28 | ) 29 | }?; 30 | 31 | let physical_devices = unsafe { instance.enumerate_physical_devices() }?; 32 | let physical_device = physical_devices[0]; 33 | 34 | let props = unsafe { device_properties(&instance, version, physical_device) }?; 35 | 36 | let device = unsafe { 37 | instance.create_device( 38 | physical_device, 39 | &vk::DeviceCreateInfo::builder().queue_create_infos(&[ 40 | vk::DeviceQueueCreateInfo::builder() 41 | .queue_family_index(0) 42 | .queue_priorities(&[0f32]) 43 | .build(), 44 | ]), 45 | None, 46 | ) 47 | }?; 48 | 49 | let config = Config::i_am_potato(); 50 | 51 | let mut allocator = GpuAllocator::new(config, props); 52 | 53 | let mut block = unsafe { 54 | allocator.alloc( 55 | AshMemoryDevice::wrap(&device), 56 | Request { 57 | size: 10, 58 | align_mask: 1, 59 | usage: UsageFlags::HOST_ACCESS, 60 | memory_types: !0, 61 | }, 62 | ) 63 | }?; 64 | 65 | unsafe { 66 | block.write_bytes( 67 | AshMemoryDevice::wrap(&device), 68 | 0, 69 | &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 70 | ) 71 | }?; 72 | 73 | unsafe { allocator.dealloc(AshMemoryDevice::wrap(&device), block) } 74 | 75 | // the `ash::Device` also implements `AsRef` 76 | // you can pass a reference of `ash::Device` directly as argument 77 | let mut block = unsafe { 78 | allocator.alloc( 79 | &device, 80 | Request { 81 | size: 10, 82 | align_mask: 1, 83 | usage: UsageFlags::HOST_ACCESS, 84 | memory_types: !0, 85 | }, 86 | ) 87 | }?; 88 | 89 | unsafe { 90 | block.write_bytes( 91 | &device, 92 | 0, 93 | &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 94 | ) 95 | }?; 96 | 97 | unsafe { allocator.dealloc(&device, block) } 98 | 99 | Ok(()) 100 | } 101 | -------------------------------------------------------------------------------- /examples/src/erupt.rs: -------------------------------------------------------------------------------- 1 | use { 2 | erupt::{vk1_0, DeviceLoader, EntryLoader, InstanceLoader}, 3 | gpu_alloc::{Config, GpuAllocator, Request, UsageFlags}, 4 | gpu_alloc_erupt::{device_properties, EruptMemoryDevice}, 5 | std::ffi::CStr, 6 | }; 7 | 8 | fn main() -> eyre::Result<()> { 9 | color_eyre::install()?; 10 | 11 | let entry = EntryLoader::new()?; 12 | 13 | let instance = unsafe { 14 | InstanceLoader::new( 15 | &entry, 16 | &vk1_0::InstanceCreateInfo::default() 17 | .into_builder() 18 | .application_info( 19 | &vk1_0::ApplicationInfo::default() 20 | .into_builder() 21 | .engine_name(CStr::from_bytes_with_nul(b"GpuAlloc\0").unwrap()) 22 | .engine_version(1) 23 | .application_name(CStr::from_bytes_with_nul(b"GpuAllocApp\0").unwrap()) 24 | .application_version(1) 25 | .api_version(entry.instance_version()), 26 | ), 27 | ) 28 | }?; 29 | 30 | let physical_devices = unsafe { instance.enumerate_physical_devices(None) }.result()?; 31 | let physical_device = physical_devices[0]; 32 | 33 | let props = unsafe { device_properties(&instance, physical_device) }?; 34 | 35 | let device = unsafe { 36 | DeviceLoader::new( 37 | &instance, 38 | physical_device, 39 | &vk1_0::DeviceCreateInfoBuilder::new().queue_create_infos(&[ 40 | vk1_0::DeviceQueueCreateInfoBuilder::new() 41 | .queue_family_index(0) 42 | .queue_priorities(&[0f32]), 43 | ]), 44 | ) 45 | }?; 46 | 47 | let config = Config::i_am_potato(); 48 | 49 | let mut allocator = GpuAllocator::new(config, props); 50 | 51 | let mut block = unsafe { 52 | allocator.alloc( 53 | EruptMemoryDevice::wrap(&device), 54 | Request { 55 | size: 10, 56 | align_mask: 1, 57 | usage: UsageFlags::HOST_ACCESS, 58 | memory_types: !0, 59 | }, 60 | ) 61 | }?; 62 | 63 | unsafe { 64 | block.write_bytes( 65 | EruptMemoryDevice::wrap(&device), 66 | 0, 67 | &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 68 | ) 69 | }?; 70 | 71 | unsafe { allocator.dealloc(EruptMemoryDevice::wrap(&device), block) } 72 | 73 | // the `erupt::DeviceLoader` also implements `AsRef` 74 | // you can pass a reference of `erupt::DeviceLoader` directly as argument 75 | let mut block = unsafe { 76 | allocator.alloc( 77 | &device, 78 | Request { 79 | size: 10, 80 | align_mask: 1, 81 | usage: UsageFlags::HOST_ACCESS, 82 | memory_types: !0, 83 | }, 84 | ) 85 | }?; 86 | 87 | unsafe { 88 | block.write_bytes( 89 | &device, 90 | 0, 91 | &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 92 | ) 93 | }?; 94 | 95 | unsafe { allocator.dealloc(&device, block) } 96 | 97 | Ok(()) 98 | } 99 | -------------------------------------------------------------------------------- /examples/src/mock.rs: -------------------------------------------------------------------------------- 1 | use { 2 | gpu_alloc::{ 3 | Config, DeviceProperties, GpuAllocator, MemoryHeap, MemoryPropertyFlags, MemoryType, 4 | Request, UsageFlags, 5 | }, 6 | gpu_alloc_mock::MockMemoryDevice, 7 | std::borrow::Cow, 8 | }; 9 | 10 | fn main() -> eyre::Result<()> { 11 | color_eyre::install()?; 12 | 13 | let device = MockMemoryDevice::new(DeviceProperties { 14 | memory_types: Cow::Borrowed(&[MemoryType { 15 | heap: 0, 16 | props: MemoryPropertyFlags::HOST_VISIBLE, 17 | }]), 18 | memory_heaps: Cow::Borrowed(&[MemoryHeap { size: 1024 * 1024 }]), 19 | max_memory_allocation_count: 32, 20 | max_memory_allocation_size: 1024 * 1024, 21 | non_coherent_atom_size: 8, 22 | buffer_device_address: false, 23 | }); 24 | 25 | let config = Config::i_am_potato(); 26 | 27 | let mut allocator = GpuAllocator::new(config, device.props()); 28 | 29 | let mut block = unsafe { 30 | allocator.alloc( 31 | &device, 32 | Request { 33 | size: 10, 34 | align_mask: 1, 35 | usage: UsageFlags::HOST_ACCESS, 36 | memory_types: !0, 37 | }, 38 | ) 39 | }?; 40 | 41 | let another_block = unsafe { 42 | allocator.alloc( 43 | &device, 44 | Request { 45 | size: 10, 46 | align_mask: 1, 47 | usage: UsageFlags::HOST_ACCESS, 48 | memory_types: !0, 49 | }, 50 | ) 51 | }?; 52 | 53 | unsafe { block.write_bytes(&device, 0, &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) }?; 54 | 55 | unsafe { allocator.dealloc(&device, block) } 56 | unsafe { allocator.dealloc(&device, another_block) } 57 | 58 | Ok(()) 59 | } 60 | -------------------------------------------------------------------------------- /examples/src/transient_reuse.rs: -------------------------------------------------------------------------------- 1 | use { 2 | gpu_alloc::{ 3 | Config, DeviceProperties, GpuAllocator, MemoryHeap, MemoryPropertyFlags, MemoryType, 4 | Request, UsageFlags, 5 | }, 6 | gpu_alloc_mock::MockMemoryDevice, 7 | std::{borrow::Cow, collections::VecDeque}, 8 | tracing_subscriber::layer::SubscriberExt as _, 9 | }; 10 | 11 | fn main() -> eyre::Result<()> { 12 | color_eyre::install()?; 13 | 14 | tracing::subscriber::set_global_default( 15 | tracing_subscriber::fmt() 16 | .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) 17 | .pretty() 18 | .finish() 19 | .with(tracing_error::ErrorLayer::default()), 20 | )?; 21 | 22 | let device = MockMemoryDevice::new(DeviceProperties { 23 | memory_types: Cow::Borrowed(&[MemoryType { 24 | heap: 0, 25 | props: MemoryPropertyFlags::HOST_VISIBLE, 26 | }]), 27 | memory_heaps: Cow::Borrowed(&[MemoryHeap { 28 | size: 32 * 1024 * 1024, 29 | }]), 30 | max_memory_allocation_count: 5, 31 | max_memory_allocation_size: 1024 * 1024, 32 | non_coherent_atom_size: 8, 33 | buffer_device_address: false, 34 | }); 35 | 36 | let config = Config::i_am_potato(); 37 | 38 | let mut allocator = GpuAllocator::new(config, device.props()); 39 | 40 | let mut blocks = VecDeque::new(); 41 | 42 | for _ in 0..1_000_000 { 43 | if blocks.len() >= 1024 { 44 | while blocks.len() > 700 { 45 | let block = blocks.pop_front().unwrap(); 46 | 47 | unsafe { 48 | allocator.dealloc(&device, block); 49 | } 50 | } 51 | } 52 | 53 | let block = unsafe { 54 | allocator.alloc( 55 | &device, 56 | Request { 57 | size: 128, 58 | align_mask: 0, 59 | usage: UsageFlags::HOST_ACCESS | UsageFlags::TRANSIENT, 60 | memory_types: !0, 61 | }, 62 | ) 63 | }?; 64 | 65 | blocks.push_back(block); 66 | } 67 | 68 | while let Some(block) = blocks.pop_front() { 69 | unsafe { 70 | allocator.dealloc(&device, block); 71 | } 72 | } 73 | 74 | // assert_eq!(device.total_allocations(), 2); 75 | 76 | tracing::warn!( 77 | "Total memory object allocations: {}", 78 | device.total_allocations() 79 | ); 80 | 81 | unsafe { 82 | allocator.cleanup(&device); 83 | } 84 | 85 | Ok(()) 86 | } 87 | -------------------------------------------------------------------------------- /gpu-alloc/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "gpu-alloc" 3 | version = "0.6.0" 4 | authors = ["Zakarum "] 5 | edition = "2018" 6 | description = "Implementation agnostic memory allocator for Vulkan like APIs" 7 | documentation = "https://docs.rs/gpu-alloc-types" 8 | readme = "../README.md" 9 | homepage = "https://github.com/zakarumych/gpu-alloc" 10 | repository = "https://github.com/zakarumych/gpu-alloc" 11 | license = "MIT OR Apache-2.0" 12 | keywords = ["gpu", "vulkan", "allocation", "no-std"] 13 | categories = ["graphics", "memory-management", "no-std", "game-development"] 14 | 15 | [features] 16 | std = [] 17 | default = ["std"] 18 | serde = ["dep:serde", "bitflags/serde"] 19 | 20 | [dependencies] 21 | gpu-alloc-types = { path = "../types", version = "=0.3.0" } 22 | tracing = { version = "0.1.27", optional = true, features = ["attributes"], default-features = false } 23 | bitflags = { version = "2.0", default-features = false } 24 | serde = { version = "1.0", optional = true, default-features = false, features = ["derive"] } 25 | -------------------------------------------------------------------------------- /gpu-alloc/src/allocator.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::{ 3 | align_down, 4 | block::{MemoryBlock, MemoryBlockFlavor}, 5 | buddy::{BuddyAllocator, BuddyBlock}, 6 | config::Config, 7 | error::AllocationError, 8 | freelist::{FreeListAllocator, FreeListBlock}, 9 | heap::Heap, 10 | usage::{MemoryForUsage, UsageFlags}, 11 | MemoryBounds, Request, 12 | }, 13 | alloc::boxed::Box, 14 | core::convert::TryFrom as _, 15 | gpu_alloc_types::{ 16 | AllocationFlags, DeviceProperties, MemoryDevice, MemoryPropertyFlags, MemoryType, 17 | OutOfMemory, 18 | }, 19 | }; 20 | 21 | /// Memory allocator for Vulkan-like APIs. 22 | #[derive(Debug)] 23 | pub struct GpuAllocator { 24 | dedicated_threshold: u64, 25 | preferred_dedicated_threshold: u64, 26 | transient_dedicated_threshold: u64, 27 | max_memory_allocation_size: u64, 28 | memory_for_usage: MemoryForUsage, 29 | memory_types: Box<[MemoryType]>, 30 | memory_heaps: Box<[Heap]>, 31 | allocations_remains: u32, 32 | non_coherent_atom_mask: u64, 33 | starting_free_list_chunk: u64, 34 | final_free_list_chunk: u64, 35 | minimal_buddy_size: u64, 36 | initial_buddy_dedicated_size: u64, 37 | buffer_device_address: bool, 38 | 39 | buddy_allocators: Box<[Option>]>, 40 | freelist_allocators: Box<[Option>]>, 41 | } 42 | 43 | /// Hints for allocator to decide on allocation strategy. 44 | #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] 45 | #[non_exhaustive] 46 | pub enum Dedicated { 47 | /// Allocation directly from device.\ 48 | /// Very slow. 49 | /// Count of allocations is limited.\ 50 | /// Use with caution.\ 51 | /// Must be used if resource has to be bound to dedicated memory object. 52 | Required, 53 | 54 | /// Hint for allocator that dedicated memory object is preferred.\ 55 | /// Should be used if it is known that resource placed in dedicated memory object 56 | /// would allow for better performance.\ 57 | /// Implementation is allowed to return block to shared memory object. 58 | Preferred, 59 | } 60 | 61 | impl GpuAllocator 62 | where 63 | M: MemoryBounds + 'static, 64 | { 65 | /// Creates new instance of `GpuAllocator`. 66 | /// Provided `DeviceProperties` should match properties of `MemoryDevice` that will be used 67 | /// with created `GpuAllocator` instance. 68 | #[cfg_attr(feature = "tracing", tracing::instrument)] 69 | pub fn new(config: Config, props: DeviceProperties<'_>) -> Self { 70 | assert!( 71 | props.non_coherent_atom_size.is_power_of_two(), 72 | "`non_coherent_atom_size` must be power of two" 73 | ); 74 | 75 | assert!( 76 | isize::try_from(props.non_coherent_atom_size).is_ok(), 77 | "`non_coherent_atom_size` must fit host address space" 78 | ); 79 | 80 | GpuAllocator { 81 | dedicated_threshold: config.dedicated_threshold, 82 | preferred_dedicated_threshold: config 83 | .preferred_dedicated_threshold 84 | .min(config.dedicated_threshold), 85 | 86 | transient_dedicated_threshold: config 87 | .transient_dedicated_threshold 88 | .max(config.dedicated_threshold), 89 | 90 | max_memory_allocation_size: props.max_memory_allocation_size, 91 | 92 | memory_for_usage: MemoryForUsage::new(props.memory_types.as_ref()), 93 | 94 | memory_types: props.memory_types.as_ref().iter().copied().collect(), 95 | memory_heaps: props 96 | .memory_heaps 97 | .as_ref() 98 | .iter() 99 | .map(|heap| Heap::new(heap.size)) 100 | .collect(), 101 | 102 | buffer_device_address: props.buffer_device_address, 103 | 104 | allocations_remains: props.max_memory_allocation_count, 105 | non_coherent_atom_mask: props.non_coherent_atom_size - 1, 106 | 107 | starting_free_list_chunk: config.starting_free_list_chunk, 108 | final_free_list_chunk: config.final_free_list_chunk, 109 | minimal_buddy_size: config.minimal_buddy_size, 110 | initial_buddy_dedicated_size: config.initial_buddy_dedicated_size, 111 | 112 | buddy_allocators: props.memory_types.as_ref().iter().map(|_| None).collect(), 113 | freelist_allocators: props.memory_types.as_ref().iter().map(|_| None).collect(), 114 | } 115 | } 116 | 117 | /// Allocates memory block from specified `device` according to the `request`. 118 | /// 119 | /// # Safety 120 | /// 121 | /// * `device` must be one with `DeviceProperties` that were provided to create this `GpuAllocator` instance. 122 | /// * Same `device` instance must be used for all interactions with one `GpuAllocator` instance 123 | /// and memory blocks allocated from it. 124 | #[cfg_attr(feature = "tracing", tracing::instrument(skip(self, device)))] 125 | pub unsafe fn alloc( 126 | &mut self, 127 | device: &impl AsRef, 128 | request: Request, 129 | ) -> Result, AllocationError> 130 | where 131 | MD: MemoryDevice, 132 | { 133 | self.alloc_internal(device.as_ref(), request, None) 134 | } 135 | 136 | /// Allocates memory block from specified `device` according to the `request`. 137 | /// This function allows user to force specific allocation strategy. 138 | /// Improper use can lead to suboptimal performance or too large overhead. 139 | /// Prefer `GpuAllocator::alloc` if doubt. 140 | /// 141 | /// # Safety 142 | /// 143 | /// * `device` must be one with `DeviceProperties` that were provided to create this `GpuAllocator` instance. 144 | /// * Same `device` instance must be used for all interactions with one `GpuAllocator` instance 145 | /// and memory blocks allocated from it. 146 | #[cfg_attr(feature = "tracing", tracing::instrument(skip(self, device)))] 147 | pub unsafe fn alloc_with_dedicated( 148 | &mut self, 149 | device: &impl AsRef, 150 | request: Request, 151 | dedicated: Dedicated, 152 | ) -> Result, AllocationError> 153 | where 154 | MD: MemoryDevice, 155 | { 156 | self.alloc_internal(device.as_ref(), request, Some(dedicated)) 157 | } 158 | 159 | unsafe fn alloc_internal( 160 | &mut self, 161 | device: &impl MemoryDevice, 162 | mut request: Request, 163 | dedicated: Option, 164 | ) -> Result, AllocationError> { 165 | enum Strategy { 166 | Buddy, 167 | Dedicated, 168 | FreeList, 169 | } 170 | 171 | request.usage = with_implicit_usage_flags(request.usage); 172 | 173 | if request.usage.contains(UsageFlags::DEVICE_ADDRESS) { 174 | assert!(self.buffer_device_address, "`DEVICE_ADDRESS` cannot be requested when `DeviceProperties::buffer_device_address` is false"); 175 | } 176 | 177 | if request.size > self.max_memory_allocation_size { 178 | return Err(AllocationError::OutOfDeviceMemory); 179 | } 180 | 181 | if let Some(Dedicated::Required) = dedicated { 182 | if self.allocations_remains == 0 { 183 | return Err(AllocationError::TooManyObjects); 184 | } 185 | } 186 | 187 | if 0 == self.memory_for_usage.mask(request.usage) & request.memory_types { 188 | #[cfg(feature = "tracing")] 189 | tracing::error!( 190 | "Cannot serve request {:?}, no memory among bitset `{}` support usage {:?}", 191 | request, 192 | request.memory_types, 193 | request.usage 194 | ); 195 | 196 | return Err(AllocationError::NoCompatibleMemoryTypes); 197 | } 198 | 199 | let transient = request.usage.contains(UsageFlags::TRANSIENT); 200 | 201 | for &index in self.memory_for_usage.types(request.usage) { 202 | if 0 == request.memory_types & (1 << index) { 203 | // Skip memory type incompatible with the request. 204 | continue; 205 | } 206 | 207 | let memory_type = &self.memory_types[index as usize]; 208 | let heap = memory_type.heap; 209 | let heap = &mut self.memory_heaps[heap as usize]; 210 | 211 | if request.size > heap.size() { 212 | // Impossible to use memory type from this heap. 213 | continue; 214 | } 215 | 216 | let atom_mask = if host_visible_non_coherent(memory_type.props) { 217 | self.non_coherent_atom_mask 218 | } else { 219 | 0 220 | }; 221 | 222 | let flags = if self.buffer_device_address { 223 | AllocationFlags::DEVICE_ADDRESS 224 | } else { 225 | AllocationFlags::empty() 226 | }; 227 | 228 | let strategy = match (dedicated, transient) { 229 | (Some(Dedicated::Required), _) => Strategy::Dedicated, 230 | (Some(Dedicated::Preferred), _) 231 | if request.size >= self.preferred_dedicated_threshold => 232 | { 233 | Strategy::Dedicated 234 | } 235 | (_, true) => { 236 | let threshold = self.transient_dedicated_threshold.min(heap.size() / 32); 237 | 238 | if request.size < threshold { 239 | Strategy::FreeList 240 | } else { 241 | Strategy::Dedicated 242 | } 243 | } 244 | (_, false) => { 245 | let threshold = self.dedicated_threshold.min(heap.size() / 32); 246 | 247 | if request.size < threshold { 248 | Strategy::Buddy 249 | } else { 250 | Strategy::Dedicated 251 | } 252 | } 253 | }; 254 | 255 | match strategy { 256 | Strategy::Dedicated => { 257 | #[cfg(feature = "tracing")] 258 | tracing::debug!( 259 | "Allocating memory object `{}@{:?}`", 260 | request.size, 261 | memory_type 262 | ); 263 | 264 | match device.allocate_memory(request.size, index, flags) { 265 | Ok(memory) => { 266 | self.allocations_remains -= 1; 267 | heap.alloc(request.size); 268 | 269 | return Ok(MemoryBlock::new( 270 | index, 271 | memory_type.props, 272 | 0, 273 | request.size, 274 | atom_mask, 275 | MemoryBlockFlavor::Dedicated { memory }, 276 | )); 277 | } 278 | Err(OutOfMemory::OutOfDeviceMemory) => continue, 279 | Err(OutOfMemory::OutOfHostMemory) => { 280 | return Err(AllocationError::OutOfHostMemory) 281 | } 282 | } 283 | } 284 | Strategy::FreeList => { 285 | let allocator = match &mut self.freelist_allocators[index as usize] { 286 | Some(allocator) => allocator, 287 | slot => { 288 | let starting_free_list_chunk = match align_down( 289 | self.starting_free_list_chunk.min(heap.size() / 32), 290 | atom_mask, 291 | ) { 292 | 0 => atom_mask, 293 | other => other, 294 | }; 295 | 296 | let final_free_list_chunk = match align_down( 297 | self.final_free_list_chunk 298 | .max(self.starting_free_list_chunk) 299 | .max(self.transient_dedicated_threshold) 300 | .min(heap.size() / 32), 301 | atom_mask, 302 | ) { 303 | 0 => atom_mask, 304 | other => other, 305 | }; 306 | 307 | slot.get_or_insert(FreeListAllocator::new( 308 | starting_free_list_chunk, 309 | final_free_list_chunk, 310 | index, 311 | memory_type.props, 312 | if host_visible_non_coherent(memory_type.props) { 313 | self.non_coherent_atom_mask 314 | } else { 315 | 0 316 | }, 317 | )) 318 | } 319 | }; 320 | let result = allocator.alloc( 321 | device, 322 | request.size, 323 | request.align_mask, 324 | flags, 325 | heap, 326 | &mut self.allocations_remains, 327 | ); 328 | 329 | match result { 330 | Ok(block) => { 331 | return Ok(MemoryBlock::new( 332 | index, 333 | memory_type.props, 334 | block.offset, 335 | block.size, 336 | atom_mask, 337 | MemoryBlockFlavor::FreeList { 338 | chunk: block.chunk, 339 | ptr: block.ptr, 340 | memory: block.memory, 341 | }, 342 | )) 343 | } 344 | Err(AllocationError::OutOfDeviceMemory) => continue, 345 | Err(err) => return Err(err), 346 | } 347 | } 348 | 349 | Strategy::Buddy => { 350 | let allocator = match &mut self.buddy_allocators[index as usize] { 351 | Some(allocator) => allocator, 352 | slot => { 353 | let minimal_buddy_size = self 354 | .minimal_buddy_size 355 | .min(heap.size() / 1024) 356 | .next_power_of_two(); 357 | 358 | let initial_buddy_dedicated_size = self 359 | .initial_buddy_dedicated_size 360 | .min(heap.size() / 32) 361 | .next_power_of_two(); 362 | 363 | slot.get_or_insert(BuddyAllocator::new( 364 | minimal_buddy_size, 365 | initial_buddy_dedicated_size, 366 | index, 367 | memory_type.props, 368 | if host_visible_non_coherent(memory_type.props) { 369 | self.non_coherent_atom_mask 370 | } else { 371 | 0 372 | }, 373 | )) 374 | } 375 | }; 376 | let result = allocator.alloc( 377 | device, 378 | request.size, 379 | request.align_mask, 380 | flags, 381 | heap, 382 | &mut self.allocations_remains, 383 | ); 384 | 385 | match result { 386 | Ok(block) => { 387 | return Ok(MemoryBlock::new( 388 | index, 389 | memory_type.props, 390 | block.offset, 391 | block.size, 392 | atom_mask, 393 | MemoryBlockFlavor::Buddy { 394 | chunk: block.chunk, 395 | ptr: block.ptr, 396 | index: block.index, 397 | memory: block.memory, 398 | }, 399 | )) 400 | } 401 | Err(AllocationError::OutOfDeviceMemory) => continue, 402 | Err(err) => return Err(err), 403 | } 404 | } 405 | } 406 | } 407 | 408 | Err(AllocationError::OutOfDeviceMemory) 409 | } 410 | 411 | /// Creates a memory block from an existing memory allocation, transferring ownership to the allocator. 412 | /// 413 | /// This function allows the [`GpuAllocator`] to manage memory allocated outside of the typical 414 | /// [`GpuAllocator::alloc`] family of functions. 415 | /// 416 | /// # Usage 417 | /// 418 | /// If you need to import external memory, such as a Win32 `HANDLE` or a Linux `dmabuf`, import the device 419 | /// memory using the graphics api and platform dependent functions. Once that is done, call this function 420 | /// to make the [`GpuAllocator`] take ownership of the imported memory. 421 | /// 422 | /// When calling this function, you **must** ensure there are [enough remaining allocations](GpuAllocator::remaining_allocations). 423 | /// 424 | /// # Safety 425 | /// 426 | /// - The `memory` must be allocated with the same device that was provided to create this [`GpuAllocator`] 427 | /// instance. 428 | /// - The `memory` must be valid. 429 | /// - The `props`, `offset` and `size` must match the properties, offset and size of the memory allocation. 430 | /// - The memory must have been allocated with the specified `memory_type`. 431 | /// - There must be enough remaining allocations. 432 | /// - The memory allocation must not come from an existing memory block created by this allocator. 433 | /// - The underlying memory object must be deallocated using the returned [`MemoryBlock`] with 434 | /// [`GpuAllocator::dealloc`]. 435 | pub unsafe fn import_memory( 436 | &mut self, 437 | memory: M, 438 | memory_type: u32, 439 | props: MemoryPropertyFlags, 440 | offset: u64, 441 | size: u64, 442 | ) -> MemoryBlock { 443 | // Get the heap which the imported memory is from. 444 | let heap = self 445 | .memory_types 446 | .get(memory_type as usize) 447 | .expect("Invalid memory type specified when importing memory") 448 | .heap; 449 | let heap = &mut self.memory_heaps[heap as usize]; 450 | 451 | #[cfg(feature = "tracing")] 452 | tracing::debug!( 453 | "Importing memory object {:?} `{}@{:?}`", 454 | memory, 455 | size, 456 | memory_type 457 | ); 458 | 459 | assert_ne!( 460 | self.allocations_remains, 0, 461 | "Out of allocations when importing a memory block. Ensure you check GpuAllocator::remaining_allocations before import." 462 | ); 463 | self.allocations_remains -= 1; 464 | 465 | let atom_mask = if host_visible_non_coherent(props) { 466 | self.non_coherent_atom_mask 467 | } else { 468 | 0 469 | }; 470 | 471 | heap.alloc(size); 472 | 473 | MemoryBlock::new( 474 | memory_type, 475 | props, 476 | offset, 477 | size, 478 | atom_mask, 479 | MemoryBlockFlavor::Dedicated { memory }, 480 | ) 481 | } 482 | 483 | /// Deallocates memory block previously allocated from this `GpuAllocator` instance. 484 | /// 485 | /// # Safety 486 | /// 487 | /// * Memory block must have been allocated by this `GpuAllocator` instance 488 | /// * `device` must be one with `DeviceProperties` that were provided to create this `GpuAllocator` instance 489 | /// * Same `device` instance must be used for all interactions with one `GpuAllocator` instance 490 | /// and memory blocks allocated from it 491 | #[cfg_attr(feature = "tracing", tracing::instrument(skip(self, device)))] 492 | pub unsafe fn dealloc(&mut self, device: &impl AsRef, block: MemoryBlock) 493 | where 494 | MD: MemoryDevice, 495 | { 496 | let device = device.as_ref(); 497 | let memory_type = block.memory_type(); 498 | let offset = block.offset(); 499 | let size = block.size(); 500 | let flavor = block.deallocate(); 501 | match flavor { 502 | MemoryBlockFlavor::Dedicated { memory } => { 503 | let heap = self.memory_types[memory_type as usize].heap; 504 | device.deallocate_memory(memory); 505 | self.allocations_remains += 1; 506 | self.memory_heaps[heap as usize].dealloc(size); 507 | } 508 | MemoryBlockFlavor::Buddy { 509 | chunk, 510 | ptr, 511 | index, 512 | memory, 513 | } => { 514 | let heap = self.memory_types[memory_type as usize].heap; 515 | let heap = &mut self.memory_heaps[heap as usize]; 516 | 517 | let allocator = self.buddy_allocators[memory_type as usize] 518 | .as_mut() 519 | .expect("Allocator should exist"); 520 | 521 | allocator.dealloc( 522 | device, 523 | BuddyBlock { 524 | memory, 525 | ptr, 526 | offset, 527 | size, 528 | chunk, 529 | index, 530 | }, 531 | heap, 532 | &mut self.allocations_remains, 533 | ); 534 | } 535 | MemoryBlockFlavor::FreeList { chunk, ptr, memory } => { 536 | let heap = self.memory_types[memory_type as usize].heap; 537 | let heap = &mut self.memory_heaps[heap as usize]; 538 | 539 | let allocator = self.freelist_allocators[memory_type as usize] 540 | .as_mut() 541 | .expect("Allocator should exist"); 542 | 543 | allocator.dealloc( 544 | device, 545 | FreeListBlock { 546 | memory, 547 | ptr, 548 | chunk, 549 | offset, 550 | size, 551 | }, 552 | heap, 553 | &mut self.allocations_remains, 554 | ); 555 | } 556 | } 557 | } 558 | 559 | /// Returns the maximum allocation size supported. 560 | pub fn max_allocation_size(&self) -> u64 { 561 | self.max_memory_allocation_size 562 | } 563 | 564 | /// Returns the number of remaining available allocations. 565 | /// 566 | /// This may be useful if you need know if the allocator can allocate a number of allocations ahead of 567 | /// time. This function is also useful for ensuring you do not allocate too much memory outside allocator 568 | /// (such as external memory). 569 | pub fn remaining_allocations(&self) -> u32 { 570 | self.allocations_remains 571 | } 572 | 573 | /// Sets the number of remaining available allocations. 574 | /// 575 | /// # Safety 576 | /// 577 | /// The caller is responsible for ensuring the number of remaining allocations does not exceed how many 578 | /// remaining allocations there actually are on the memory device. 579 | pub unsafe fn set_remaining_allocations(&mut self, remaining: u32) { 580 | self.allocations_remains = remaining; 581 | } 582 | 583 | /// Deallocates leftover memory objects. 584 | /// Should be used before dropping. 585 | /// 586 | /// # Safety 587 | /// 588 | /// * `device` must be one with `DeviceProperties` that were provided to create this `GpuAllocator` instance 589 | /// * Same `device` instance must be used for all interactions with one `GpuAllocator` instance 590 | /// and memory blocks allocated from it 591 | #[cfg_attr(feature = "tracing", tracing::instrument(skip(self, device)))] 592 | pub unsafe fn cleanup(&mut self, device: &impl AsRef) 593 | where 594 | MD: MemoryDevice, 595 | { 596 | for (index, allocator) in self 597 | .freelist_allocators 598 | .iter_mut() 599 | .enumerate() 600 | .filter_map(|(index, allocator)| Some((index, allocator.as_mut()?))) 601 | { 602 | let device = device.as_ref(); 603 | let memory_type = &self.memory_types[index]; 604 | let heap = memory_type.heap; 605 | let heap = &mut self.memory_heaps[heap as usize]; 606 | 607 | allocator.cleanup(device, heap, &mut self.allocations_remains); 608 | } 609 | } 610 | } 611 | 612 | fn host_visible_non_coherent(props: MemoryPropertyFlags) -> bool { 613 | (props & (MemoryPropertyFlags::HOST_COHERENT | MemoryPropertyFlags::HOST_VISIBLE)) 614 | == MemoryPropertyFlags::HOST_VISIBLE 615 | } 616 | 617 | fn with_implicit_usage_flags(usage: UsageFlags) -> UsageFlags { 618 | if usage.is_empty() { 619 | UsageFlags::FAST_DEVICE_ACCESS 620 | } else if usage.intersects(UsageFlags::DOWNLOAD | UsageFlags::UPLOAD) { 621 | usage | UsageFlags::HOST_ACCESS 622 | } else { 623 | usage 624 | } 625 | } 626 | -------------------------------------------------------------------------------- /gpu-alloc/src/block.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::{align_down, align_up, error::MapError}, 3 | alloc::sync::Arc, 4 | core::{ 5 | convert::TryFrom as _, 6 | ptr::{copy_nonoverlapping, NonNull}, 7 | // sync::atomic::{AtomicU8, Ordering::*}, 8 | }, 9 | gpu_alloc_types::{MappedMemoryRange, MemoryDevice, MemoryPropertyFlags}, 10 | }; 11 | 12 | #[derive(Debug)] 13 | struct Relevant; 14 | 15 | impl Drop for Relevant { 16 | fn drop(&mut self) { 17 | report_error_on_drop!("Memory block wasn't deallocated"); 18 | } 19 | } 20 | 21 | /// Memory block allocated by `GpuAllocator`. 22 | #[derive(Debug)] 23 | pub struct MemoryBlock { 24 | memory_type: u32, 25 | props: MemoryPropertyFlags, 26 | offset: u64, 27 | size: u64, 28 | atom_mask: u64, 29 | mapped: bool, 30 | flavor: MemoryBlockFlavor, 31 | relevant: Relevant, 32 | } 33 | 34 | impl MemoryBlock { 35 | pub(crate) fn new( 36 | memory_type: u32, 37 | props: MemoryPropertyFlags, 38 | offset: u64, 39 | size: u64, 40 | atom_mask: u64, 41 | flavor: MemoryBlockFlavor, 42 | ) -> Self { 43 | isize::try_from(atom_mask).expect("`atom_mask` is too large"); 44 | MemoryBlock { 45 | memory_type, 46 | props, 47 | offset, 48 | size, 49 | atom_mask, 50 | flavor, 51 | mapped: false, 52 | relevant: Relevant, 53 | } 54 | } 55 | 56 | pub(crate) fn deallocate(self) -> MemoryBlockFlavor { 57 | core::mem::forget(self.relevant); 58 | self.flavor 59 | } 60 | } 61 | 62 | unsafe impl Sync for MemoryBlock where M: Sync {} 63 | unsafe impl Send for MemoryBlock where M: Send {} 64 | 65 | #[derive(Debug)] 66 | pub(crate) enum MemoryBlockFlavor { 67 | Dedicated { 68 | memory: M, 69 | }, 70 | Buddy { 71 | chunk: usize, 72 | index: usize, 73 | ptr: Option>, 74 | memory: Arc, 75 | }, 76 | FreeList { 77 | chunk: u64, 78 | ptr: Option>, 79 | memory: Arc, 80 | }, 81 | } 82 | 83 | impl MemoryBlock { 84 | /// Returns reference to parent memory object. 85 | #[inline(always)] 86 | pub fn memory(&self) -> &M { 87 | match &self.flavor { 88 | MemoryBlockFlavor::Dedicated { memory } => memory, 89 | MemoryBlockFlavor::Buddy { memory, .. } => memory, 90 | MemoryBlockFlavor::FreeList { memory, .. } => memory, 91 | } 92 | } 93 | 94 | /// Returns offset in bytes from start of memory object to start of this block. 95 | #[inline(always)] 96 | pub fn offset(&self) -> u64 { 97 | self.offset 98 | } 99 | 100 | /// Returns size of this memory block. 101 | #[inline(always)] 102 | pub fn size(&self) -> u64 { 103 | self.size 104 | } 105 | 106 | /// Returns memory property flags for parent memory object. 107 | #[inline(always)] 108 | pub fn props(&self) -> MemoryPropertyFlags { 109 | self.props 110 | } 111 | 112 | /// Returns index of type of parent memory object. 113 | #[inline(always)] 114 | pub fn memory_type(&self) -> u32 { 115 | self.memory_type 116 | } 117 | 118 | /// Returns pointer to mapped memory range of this block. 119 | /// This blocks becomes mapped. 120 | /// 121 | /// The user of returned pointer must guarantee that any previously submitted command that writes to this range has completed 122 | /// before the host reads from or writes to that range, 123 | /// and that any previously submitted command that reads from that range has completed 124 | /// before the host writes to that region. 125 | /// If the device memory was allocated without the `HOST_COHERENT` property flag set, 126 | /// these guarantees must be made for an extended range: 127 | /// the user must round down the start of the range to the nearest multiple of `non_coherent_atom_size`, 128 | /// and round the end of the range up to the nearest multiple of `non_coherent_atom_size`. 129 | /// 130 | /// # Panics 131 | /// 132 | /// This function panics if block is currently mapped. 133 | /// 134 | /// # Safety 135 | /// 136 | /// `block` must have been allocated from specified `device`. 137 | #[inline(always)] 138 | pub unsafe fn map( 139 | &mut self, 140 | device: &impl AsRef, 141 | offset: u64, 142 | size: usize, 143 | ) -> Result, MapError> 144 | where 145 | MD: MemoryDevice, 146 | { 147 | let size_u64 = u64::try_from(size).expect("`size` doesn't fit device address space"); 148 | assert!(offset < self.size, "`offset` is out of memory block bounds"); 149 | assert!( 150 | size_u64 <= self.size - offset, 151 | "`offset + size` is out of memory block bounds" 152 | ); 153 | 154 | let ptr = match &mut self.flavor { 155 | MemoryBlockFlavor::Dedicated { memory } => { 156 | let end = align_up(offset + size_u64, self.atom_mask) 157 | .expect("mapping end doesn't fit device address space"); 158 | let aligned_offset = align_down(offset, self.atom_mask); 159 | 160 | if !acquire_mapping(&mut self.mapped) { 161 | return Err(MapError::AlreadyMapped); 162 | } 163 | let result = 164 | device.as_ref().map_memory(memory, self.offset + aligned_offset, end - aligned_offset); 165 | 166 | match result { 167 | // the overflow is checked in `Self::new()` 168 | Ok(ptr) => { 169 | let ptr_offset = (offset - aligned_offset) as isize; 170 | ptr.as_ptr().offset(ptr_offset) 171 | } 172 | Err(err) => { 173 | release_mapping(&mut self.mapped); 174 | return Err(err.into()); 175 | } 176 | } 177 | } 178 | MemoryBlockFlavor::FreeList { ptr: Some(ptr), .. } 179 | | MemoryBlockFlavor::Buddy { ptr: Some(ptr), .. } => { 180 | if !acquire_mapping(&mut self.mapped) { 181 | return Err(MapError::AlreadyMapped); 182 | } 183 | let offset_isize = isize::try_from(offset) 184 | .expect("Buddy and linear block should fit host address space"); 185 | ptr.as_ptr().offset(offset_isize) 186 | } 187 | _ => return Err(MapError::NonHostVisible), 188 | }; 189 | 190 | Ok(NonNull::new_unchecked(ptr)) 191 | } 192 | 193 | /// Unmaps memory range of this block that was previously mapped with `Block::map`. 194 | /// This block becomes unmapped. 195 | /// 196 | /// # Panics 197 | /// 198 | /// This function panics if this block is not currently mapped. 199 | /// 200 | /// # Safety 201 | /// 202 | /// `block` must have been allocated from specified `device`. 203 | #[inline(always)] 204 | pub unsafe fn unmap(&mut self, device: &impl AsRef) -> bool 205 | where 206 | MD: MemoryDevice, 207 | { 208 | if !release_mapping(&mut self.mapped) { 209 | return false; 210 | } 211 | match &mut self.flavor { 212 | MemoryBlockFlavor::Dedicated { memory } => { 213 | device.as_ref().unmap_memory(memory); 214 | } 215 | MemoryBlockFlavor::Buddy { .. } => {} 216 | MemoryBlockFlavor::FreeList { .. } => {} 217 | } 218 | true 219 | } 220 | 221 | /// Transiently maps block memory range and copies specified data 222 | /// to the mapped memory range. 223 | /// 224 | /// # Panics 225 | /// 226 | /// This function panics if block is currently mapped. 227 | /// 228 | /// # Safety 229 | /// 230 | /// `block` must have been allocated from specified `device`. 231 | /// The caller must guarantee that any previously submitted command that reads or writes to this range has completed. 232 | #[inline(always)] 233 | pub unsafe fn write_bytes( 234 | &mut self, 235 | device: &impl AsRef, 236 | offset: u64, 237 | data: &[u8], 238 | ) -> Result<(), MapError> 239 | where 240 | MD: MemoryDevice, 241 | { 242 | let size = data.len(); 243 | let ptr = self.map(device, offset, size)?; 244 | 245 | copy_nonoverlapping(data.as_ptr(), ptr.as_ptr(), size); 246 | let result = if !self.coherent() { 247 | let aligned_offset = align_down(offset, self.atom_mask); 248 | let end = align_up(offset + data.len() as u64, self.atom_mask).unwrap(); 249 | 250 | device.as_ref().flush_memory_ranges(&[MappedMemoryRange { 251 | memory: self.memory(), 252 | offset: self.offset + aligned_offset, 253 | size: end - aligned_offset, 254 | }]) 255 | } else { 256 | Ok(()) 257 | }; 258 | 259 | self.unmap(device); 260 | result.map_err(Into::into) 261 | } 262 | 263 | /// Transiently maps block memory range and copies specified data 264 | /// from the mapped memory range. 265 | /// 266 | /// # Panics 267 | /// 268 | /// This function panics if block is currently mapped. 269 | /// 270 | /// # Safety 271 | /// 272 | /// `block` must have been allocated from specified `device`. 273 | /// The caller must guarantee that any previously submitted command that reads to this range has completed. 274 | #[inline(always)] 275 | pub unsafe fn read_bytes( 276 | &mut self, 277 | device: &impl AsRef, 278 | offset: u64, 279 | data: &mut [u8], 280 | ) -> Result<(), MapError> 281 | where 282 | MD: MemoryDevice, 283 | { 284 | #[cfg(feature = "tracing")] 285 | { 286 | if !self.cached() { 287 | tracing::warn!("Reading from non-cached memory may be slow. Consider allocating HOST_CACHED memory block for host reads.") 288 | } 289 | } 290 | 291 | let size = data.len(); 292 | let ptr = self.map(device, offset, size)?; 293 | let result = if !self.coherent() { 294 | let aligned_offset = align_down(offset, self.atom_mask); 295 | let end = align_up(offset + data.len() as u64, self.atom_mask).unwrap(); 296 | 297 | device.as_ref().invalidate_memory_ranges(&[MappedMemoryRange { 298 | memory: self.memory(), 299 | offset: self.offset + aligned_offset, 300 | size: end - aligned_offset, 301 | }]) 302 | } else { 303 | Ok(()) 304 | }; 305 | if result.is_ok() { 306 | copy_nonoverlapping(ptr.as_ptr(), data.as_mut_ptr(), size); 307 | } 308 | 309 | self.unmap(device); 310 | result.map_err(Into::into) 311 | } 312 | 313 | fn coherent(&self) -> bool { 314 | self.props.contains(MemoryPropertyFlags::HOST_COHERENT) 315 | } 316 | 317 | #[cfg(feature = "tracing")] 318 | fn cached(&self) -> bool { 319 | self.props.contains(MemoryPropertyFlags::HOST_CACHED) 320 | } 321 | } 322 | 323 | fn acquire_mapping(mapped: &mut bool) -> bool { 324 | if *mapped { 325 | false 326 | } else { 327 | *mapped = true; 328 | true 329 | } 330 | } 331 | 332 | fn release_mapping(mapped: &mut bool) -> bool { 333 | if *mapped { 334 | *mapped = false; 335 | true 336 | } else { 337 | false 338 | } 339 | } 340 | -------------------------------------------------------------------------------- /gpu-alloc/src/buddy.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::{ 3 | align_up, error::AllocationError, heap::Heap, slab::Slab, unreachable_unchecked, 4 | util::try_arc_unwrap, MemoryBounds, 5 | }, 6 | alloc::{sync::Arc, vec::Vec}, 7 | core::{convert::TryFrom as _, mem::replace, ptr::NonNull}, 8 | gpu_alloc_types::{AllocationFlags, DeviceMapError, MemoryDevice, MemoryPropertyFlags}, 9 | }; 10 | 11 | #[derive(Debug)] 12 | pub(crate) struct BuddyBlock { 13 | pub memory: Arc, 14 | pub ptr: Option>, 15 | pub offset: u64, 16 | pub size: u64, 17 | pub chunk: usize, 18 | pub index: usize, 19 | } 20 | 21 | unsafe impl Sync for BuddyBlock where M: Sync {} 22 | unsafe impl Send for BuddyBlock where M: Send {} 23 | 24 | #[derive(Clone, Copy, Debug)] 25 | enum PairState { 26 | Exhausted, 27 | Ready { 28 | ready: Side, 29 | next: usize, 30 | prev: usize, 31 | }, 32 | } 33 | 34 | impl PairState { 35 | unsafe fn replace_next(&mut self, value: usize) -> usize { 36 | match self { 37 | PairState::Exhausted => unreachable_unchecked(), 38 | PairState::Ready { next, .. } => replace(next, value), 39 | } 40 | } 41 | 42 | unsafe fn replace_prev(&mut self, value: usize) -> usize { 43 | match self { 44 | PairState::Exhausted => unreachable_unchecked(), 45 | PairState::Ready { prev, .. } => replace(prev, value), 46 | } 47 | } 48 | } 49 | 50 | #[derive(Clone, Copy, Debug, PartialEq, Eq)] 51 | enum Side { 52 | Left, 53 | Right, 54 | } 55 | use Side::*; 56 | 57 | #[derive(Debug)] 58 | struct PairEntry { 59 | state: PairState, 60 | chunk: usize, 61 | offset: u64, 62 | parent: Option, 63 | } 64 | 65 | struct SizeBlockEntry { 66 | chunk: usize, 67 | offset: u64, 68 | index: usize, 69 | } 70 | 71 | #[derive(Debug)] 72 | struct Size { 73 | next_ready: usize, 74 | pairs: Slab, 75 | } 76 | #[derive(Debug)] 77 | enum Release { 78 | None, 79 | Parent(usize), 80 | Chunk(usize), 81 | } 82 | 83 | impl Size { 84 | fn new() -> Self { 85 | Size { 86 | pairs: Slab::new(), 87 | next_ready: 0, 88 | } 89 | } 90 | 91 | unsafe fn add_pair_and_acquire_left( 92 | &mut self, 93 | chunk: usize, 94 | offset: u64, 95 | parent: Option, 96 | ) -> SizeBlockEntry { 97 | if self.next_ready < self.pairs.len() { 98 | unreachable_unchecked() 99 | } 100 | 101 | let index = self.pairs.insert(PairEntry { 102 | state: PairState::Exhausted, 103 | chunk, 104 | offset, 105 | parent, 106 | }); 107 | 108 | let entry = self.pairs.get_unchecked_mut(index); 109 | entry.state = PairState::Ready { 110 | next: index, 111 | prev: index, 112 | ready: Right, // Left is allocated. 113 | }; 114 | self.next_ready = index; 115 | 116 | SizeBlockEntry { 117 | chunk, 118 | offset, 119 | index: index << 1, 120 | } 121 | } 122 | 123 | fn acquire(&mut self, size: u64) -> Option { 124 | if self.next_ready >= self.pairs.len() { 125 | return None; 126 | } 127 | 128 | let ready = self.next_ready; 129 | 130 | let entry = unsafe { self.pairs.get_unchecked_mut(ready) }; 131 | let chunk = entry.chunk; 132 | let offset = entry.offset; 133 | 134 | let bit = match entry.state { 135 | PairState::Exhausted => unsafe { unreachable_unchecked() }, 136 | PairState::Ready { ready, next, prev } => { 137 | entry.state = PairState::Exhausted; 138 | 139 | if prev == self.next_ready { 140 | // The only ready entry. 141 | debug_assert_eq!(next, self.next_ready); 142 | self.next_ready = self.pairs.len(); 143 | } else { 144 | let prev_entry = unsafe { self.pairs.get_unchecked_mut(prev) }; 145 | let prev_next = unsafe { prev_entry.state.replace_next(next) }; 146 | debug_assert_eq!(prev_next, self.next_ready); 147 | 148 | let next_entry = unsafe { self.pairs.get_unchecked_mut(next) }; 149 | let next_prev = unsafe { next_entry.state.replace_prev(prev) }; 150 | debug_assert_eq!(next_prev, self.next_ready); 151 | 152 | self.next_ready = next; 153 | } 154 | 155 | match ready { 156 | Left => 0, 157 | Right => 1, 158 | } 159 | } 160 | }; 161 | 162 | Some(SizeBlockEntry { 163 | chunk, 164 | offset: offset + bit as u64 * size, 165 | index: (ready << 1) | bit as usize, 166 | }) 167 | } 168 | 169 | fn release(&mut self, index: usize) -> Release { 170 | let side = match index & 1 { 171 | 0 => Side::Left, 172 | 1 => Side::Right, 173 | _ => unsafe { unreachable_unchecked() }, 174 | }; 175 | let entry_index = index >> 1; 176 | 177 | let len = self.pairs.len(); 178 | 179 | let entry = self.pairs.get_mut(entry_index); 180 | 181 | let chunk = entry.chunk; 182 | let offset = entry.offset; 183 | let parent = entry.parent; 184 | 185 | match entry.state { 186 | PairState::Exhausted => { 187 | if self.next_ready == len { 188 | entry.state = PairState::Ready { 189 | ready: side, 190 | next: entry_index, 191 | prev: entry_index, 192 | }; 193 | self.next_ready = entry_index; 194 | } else { 195 | debug_assert!(self.next_ready < len); 196 | 197 | let next = self.next_ready; 198 | let next_entry = unsafe { self.pairs.get_unchecked_mut(next) }; 199 | let prev = unsafe { next_entry.state.replace_prev(entry_index) }; 200 | 201 | let prev_entry = unsafe { self.pairs.get_unchecked_mut(prev) }; 202 | let prev_next = unsafe { prev_entry.state.replace_next(entry_index) }; 203 | debug_assert_eq!(prev_next, next); 204 | 205 | let entry = unsafe { self.pairs.get_unchecked_mut(entry_index) }; 206 | entry.state = PairState::Ready { 207 | ready: side, 208 | next, 209 | prev, 210 | }; 211 | } 212 | Release::None 213 | } 214 | 215 | PairState::Ready { ready, .. } if ready == side => { 216 | panic!("Attempt to dealloate already free block") 217 | } 218 | 219 | PairState::Ready { next, prev, .. } => { 220 | unsafe { 221 | self.pairs.remove_unchecked(entry_index); 222 | } 223 | 224 | if prev == entry_index { 225 | debug_assert_eq!(next, entry_index); 226 | self.next_ready = self.pairs.len(); 227 | } else { 228 | let prev_entry = unsafe { self.pairs.get_unchecked_mut(prev) }; 229 | let prev_next = unsafe { prev_entry.state.replace_next(next) }; 230 | debug_assert_eq!(prev_next, entry_index); 231 | 232 | let next_entry = unsafe { self.pairs.get_unchecked_mut(next) }; 233 | let next_prev = unsafe { next_entry.state.replace_prev(prev) }; 234 | debug_assert_eq!(next_prev, entry_index); 235 | 236 | self.next_ready = next; 237 | } 238 | 239 | match parent { 240 | Some(parent) => Release::Parent(parent), 241 | None => { 242 | debug_assert_eq!(offset, 0); 243 | Release::Chunk(chunk) 244 | } 245 | } 246 | } 247 | } 248 | } 249 | } 250 | 251 | #[derive(Debug)] 252 | struct Chunk { 253 | memory: Arc, 254 | ptr: Option>, 255 | size: u64, 256 | } 257 | 258 | #[derive(Debug)] 259 | pub(crate) struct BuddyAllocator { 260 | minimal_size: u64, 261 | chunks: Slab>, 262 | sizes: Vec, 263 | memory_type: u32, 264 | props: MemoryPropertyFlags, 265 | atom_mask: u64, 266 | } 267 | 268 | unsafe impl Sync for BuddyAllocator where M: Sync {} 269 | unsafe impl Send for BuddyAllocator where M: Send {} 270 | 271 | impl BuddyAllocator 272 | where 273 | M: MemoryBounds + 'static, 274 | { 275 | pub fn new( 276 | minimal_size: u64, 277 | initial_dedicated_size: u64, 278 | memory_type: u32, 279 | props: MemoryPropertyFlags, 280 | atom_mask: u64, 281 | ) -> Self { 282 | assert!( 283 | minimal_size.is_power_of_two(), 284 | "Minimal allocation size of buddy allocator must be power of two" 285 | ); 286 | assert!( 287 | initial_dedicated_size.is_power_of_two(), 288 | "Dedicated allocation size of buddy allocator must be power of two" 289 | ); 290 | 291 | let initial_sizes = (initial_dedicated_size 292 | .trailing_zeros() 293 | .saturating_sub(minimal_size.trailing_zeros())) as usize; 294 | 295 | BuddyAllocator { 296 | minimal_size, 297 | chunks: Slab::new(), 298 | sizes: (0..initial_sizes).map(|_| Size::new()).collect(), 299 | memory_type, 300 | props, 301 | atom_mask: atom_mask | (minimal_size - 1), 302 | } 303 | } 304 | 305 | #[cfg_attr(feature = "tracing", tracing::instrument(skip(self, device)))] 306 | pub unsafe fn alloc( 307 | &mut self, 308 | device: &impl MemoryDevice, 309 | size: u64, 310 | align_mask: u64, 311 | flags: AllocationFlags, 312 | heap: &mut Heap, 313 | allocations_remains: &mut u32, 314 | ) -> Result, AllocationError> { 315 | let align_mask = align_mask | self.atom_mask; 316 | 317 | let size = align_up(size, align_mask) 318 | .and_then(|size| size.checked_next_power_of_two()) 319 | .ok_or(AllocationError::OutOfDeviceMemory)?; 320 | 321 | let size = size.max(self.minimal_size); 322 | 323 | let size_index = size.trailing_zeros() - self.minimal_size.trailing_zeros(); 324 | let size_index = 325 | usize::try_from(size_index).map_err(|_| AllocationError::OutOfDeviceMemory)?; 326 | 327 | while self.sizes.len() <= size_index { 328 | self.sizes.push(Size::new()); 329 | } 330 | 331 | let host_visible = self.host_visible(); 332 | 333 | let mut candidate_size_index = size_index; 334 | 335 | let (mut entry, entry_size_index) = loop { 336 | let sizes_len = self.sizes.len(); 337 | 338 | let candidate_size_entry = &mut self.sizes[candidate_size_index]; 339 | let candidate_size = self.minimal_size << candidate_size_index; 340 | 341 | if let Some(entry) = candidate_size_entry.acquire(candidate_size) { 342 | break (entry, candidate_size_index); 343 | } 344 | 345 | if sizes_len == candidate_size_index + 1 { 346 | // That's size of device allocation. 347 | if *allocations_remains == 0 { 348 | return Err(AllocationError::TooManyObjects); 349 | } 350 | 351 | let chunk_size = self.minimal_size << (candidate_size_index + 1); 352 | let mut memory = device.allocate_memory(chunk_size, self.memory_type, flags)?; 353 | *allocations_remains -= 1; 354 | heap.alloc(chunk_size); 355 | 356 | let ptr = if host_visible { 357 | match device.map_memory(&mut memory, 0, chunk_size) { 358 | Ok(ptr) => Some(ptr), 359 | Err(DeviceMapError::OutOfDeviceMemory) => { 360 | return Err(AllocationError::OutOfDeviceMemory) 361 | } 362 | Err(DeviceMapError::MapFailed) | Err(DeviceMapError::OutOfHostMemory) => { 363 | return Err(AllocationError::OutOfHostMemory) 364 | } 365 | } 366 | } else { 367 | None 368 | }; 369 | 370 | let chunk = self.chunks.insert(Chunk { 371 | memory: Arc::new(memory), 372 | ptr, 373 | size: chunk_size, 374 | }); 375 | 376 | let entry = candidate_size_entry.add_pair_and_acquire_left(chunk, 0, None); 377 | 378 | break (entry, candidate_size_index); 379 | } 380 | 381 | candidate_size_index += 1; 382 | }; 383 | 384 | for size_index in (size_index..entry_size_index).rev() { 385 | let size_entry = &mut self.sizes[size_index]; 386 | entry = 387 | size_entry.add_pair_and_acquire_left(entry.chunk, entry.offset, Some(entry.index)); 388 | } 389 | 390 | let chunk_entry = self.chunks.get_unchecked(entry.chunk); 391 | 392 | debug_assert!( 393 | entry 394 | .offset 395 | .checked_add(size) 396 | .map_or(false, |end| end <= chunk_entry.size), 397 | "Offset + size is not in chunk bounds" 398 | ); 399 | 400 | Ok(BuddyBlock { 401 | memory: chunk_entry.memory.clone(), 402 | ptr: chunk_entry 403 | .ptr 404 | .map(|ptr| NonNull::new_unchecked(ptr.as_ptr().add(entry.offset as usize))), 405 | offset: entry.offset, 406 | size, 407 | chunk: entry.chunk, 408 | index: entry.index, 409 | }) 410 | } 411 | 412 | #[cfg_attr(feature = "tracing", tracing::instrument(skip(self, device)))] 413 | pub unsafe fn dealloc( 414 | &mut self, 415 | device: &impl MemoryDevice, 416 | block: BuddyBlock, 417 | heap: &mut Heap, 418 | allocations_remains: &mut u32, 419 | ) { 420 | debug_assert!(block.size.is_power_of_two()); 421 | 422 | let size_index = 423 | (block.size.trailing_zeros() - self.minimal_size.trailing_zeros()) as usize; 424 | 425 | let mut release_index = block.index; 426 | let mut release_size_index = size_index; 427 | 428 | loop { 429 | match self.sizes[release_size_index].release(release_index) { 430 | Release::Parent(parent) => { 431 | release_size_index += 1; 432 | release_index = parent; 433 | } 434 | Release::Chunk(chunk) => { 435 | debug_assert_eq!(chunk, block.chunk); 436 | debug_assert_eq!( 437 | self.chunks.get(chunk).size, 438 | self.minimal_size << (release_size_index + 1) 439 | ); 440 | let chunk = self.chunks.remove(chunk); 441 | drop(block); 442 | 443 | let memory = try_arc_unwrap(chunk.memory) 444 | .expect("Memory shared after last block deallocated"); 445 | 446 | device.deallocate_memory(memory); 447 | *allocations_remains += 1; 448 | heap.dealloc(chunk.size); 449 | 450 | return; 451 | } 452 | Release::None => return, 453 | } 454 | } 455 | } 456 | 457 | fn host_visible(&self) -> bool { 458 | self.props.contains(MemoryPropertyFlags::HOST_VISIBLE) 459 | } 460 | } 461 | -------------------------------------------------------------------------------- /gpu-alloc/src/config.rs: -------------------------------------------------------------------------------- 1 | /// Configuration for [`GpuAllocator`] 2 | /// 3 | /// [`GpuAllocator`]: type.GpuAllocator 4 | #[derive(Clone, Copy, Debug, PartialEq, Eq)] 5 | #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] 6 | pub struct Config { 7 | /// Size in bytes of request that will be served by dedicated memory object. 8 | /// This value should be large enough to not exhaust memory object limit 9 | /// and not use slow memory object allocation when it is not necessary. 10 | pub dedicated_threshold: u64, 11 | 12 | /// Size in bytes of request that will be served by dedicated memory object if preferred. 13 | /// This value should be large enough to not exhaust memory object limit 14 | /// and not use slow memory object allocation when it is not necessary. 15 | /// 16 | /// This won't make much sense if this value is larger than `dedicated_threshold`. 17 | pub preferred_dedicated_threshold: u64, 18 | 19 | /// Size in bytes of transient memory request that will be served by dedicated memory object. 20 | /// This value should be large enough to not exhaust memory object limit 21 | /// and not use slow memory object allocation when it is not necessary. 22 | /// 23 | /// This won't make much sense if this value is lesser than `dedicated_threshold`. 24 | pub transient_dedicated_threshold: u64, 25 | 26 | /// Size in bytes of first chunk in free-list allocator. 27 | pub starting_free_list_chunk: u64, 28 | 29 | /// Upper limit for size in bytes of chunks in free-list allocator. 30 | pub final_free_list_chunk: u64, 31 | 32 | /// Minimal size for buddy allocator. 33 | pub minimal_buddy_size: u64, 34 | 35 | /// Initial memory object size for buddy allocator. 36 | /// If less than `minimal_buddy_size` then `minimal_buddy_size` is used instead. 37 | pub initial_buddy_dedicated_size: u64, 38 | } 39 | 40 | impl Config { 41 | /// Returns default configuration. 42 | /// 43 | /// This is not `Default` implementation to discourage usage outside of 44 | /// prototyping. 45 | /// 46 | /// Proper configuration should depend on hardware and intended usage.\ 47 | /// But those values can be used as starting point.\ 48 | /// Note that they can simply not work for some platforms with lesser 49 | /// memory capacity than today's "modern" GPU (year 2020). 50 | pub fn i_am_prototyping() -> Self { 51 | // Assume that today's modern GPU is made of 1024 potatoes. 52 | let potato = Config::i_am_potato(); 53 | 54 | Config { 55 | dedicated_threshold: potato.dedicated_threshold * 1024, 56 | preferred_dedicated_threshold: potato.preferred_dedicated_threshold * 1024, 57 | transient_dedicated_threshold: potato.transient_dedicated_threshold * 1024, 58 | starting_free_list_chunk: potato.starting_free_list_chunk * 1024, 59 | final_free_list_chunk: potato.final_free_list_chunk * 1024, 60 | minimal_buddy_size: potato.minimal_buddy_size * 1024, 61 | initial_buddy_dedicated_size: potato.initial_buddy_dedicated_size * 1024, 62 | } 63 | } 64 | 65 | /// Returns default configuration for average sized potato. 66 | pub fn i_am_potato() -> Self { 67 | Config { 68 | dedicated_threshold: 32 * 1024, 69 | preferred_dedicated_threshold: 1024, 70 | transient_dedicated_threshold: 128 * 1024, 71 | starting_free_list_chunk: 8 * 1024, 72 | final_free_list_chunk: 128 * 1024, 73 | minimal_buddy_size: 1, 74 | initial_buddy_dedicated_size: 8 * 1024, 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /gpu-alloc/src/error.rs: -------------------------------------------------------------------------------- 1 | use { 2 | core::fmt::{self, Display}, 3 | gpu_alloc_types::{DeviceMapError, OutOfMemory}, 4 | }; 5 | 6 | /// Enumeration of possible errors that may occur during memory allocation. 7 | #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] 8 | pub enum AllocationError { 9 | /// Backend reported that device memory has been exhausted.\ 10 | /// Deallocating device memory from the same heap may increase chance 11 | /// that another allocation would succeed. 12 | OutOfDeviceMemory, 13 | 14 | /// Backend reported that host memory has been exhausted.\ 15 | /// Deallocating host memory may increase chance that another allocation would succeed. 16 | OutOfHostMemory, 17 | 18 | /// Allocation request cannot be fulfilled as no available memory types allowed 19 | /// by `Request.memory_types` mask is compatible with `request.usage`. 20 | NoCompatibleMemoryTypes, 21 | 22 | /// Reached limit on allocated memory objects count.\ 23 | /// Deallocating device memory may increase chance that another allocation would succeed. 24 | /// Especially dedicated memory blocks. 25 | /// 26 | /// If this error is returned when memory heaps are far from exhausted 27 | /// `Config` should be tweaked to allocate larger memory objects. 28 | TooManyObjects, 29 | } 30 | 31 | impl From for AllocationError { 32 | fn from(err: OutOfMemory) -> Self { 33 | match err { 34 | OutOfMemory::OutOfDeviceMemory => AllocationError::OutOfDeviceMemory, 35 | OutOfMemory::OutOfHostMemory => AllocationError::OutOfHostMemory, 36 | } 37 | } 38 | } 39 | 40 | impl Display for AllocationError { 41 | fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { 42 | match self { 43 | AllocationError::OutOfDeviceMemory => fmt.write_str("Device memory exhausted"), 44 | AllocationError::OutOfHostMemory => fmt.write_str("Host memory exhausted"), 45 | AllocationError::NoCompatibleMemoryTypes => fmt.write_str( 46 | "No compatible memory types from requested types support requested usage", 47 | ), 48 | AllocationError::TooManyObjects => { 49 | fmt.write_str("Reached limit on allocated memory objects count") 50 | } 51 | } 52 | } 53 | } 54 | 55 | #[cfg(feature = "std")] 56 | impl std::error::Error for AllocationError {} 57 | 58 | /// Enumeration of possible errors that may occur during memory mapping. 59 | #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] 60 | pub enum MapError { 61 | /// Backend reported that device memory has been exhausted.\ 62 | /// Deallocating device memory from the same heap may increase chance 63 | /// that another mapping would succeed. 64 | OutOfDeviceMemory, 65 | 66 | /// Backend reported that host memory has been exhausted.\ 67 | /// Deallocating host memory may increase chance that another mapping would succeed. 68 | OutOfHostMemory, 69 | 70 | /// Attempt to map memory block with non-host-visible memory type.\ 71 | /// Ensure to include `UsageFlags::HOST_ACCESS` into allocation request 72 | /// when memory mapping is intended. 73 | NonHostVisible, 74 | 75 | /// Map failed for implementation specific reason.\ 76 | /// For Vulkan backend this includes failed attempt 77 | /// to allocate large enough virtual address space. 78 | MapFailed, 79 | 80 | /// Mapping failed due to block being already mapped. 81 | AlreadyMapped, 82 | } 83 | 84 | impl From for MapError { 85 | fn from(err: DeviceMapError) -> Self { 86 | match err { 87 | DeviceMapError::OutOfDeviceMemory => MapError::OutOfDeviceMemory, 88 | DeviceMapError::OutOfHostMemory => MapError::OutOfHostMemory, 89 | DeviceMapError::MapFailed => MapError::MapFailed, 90 | } 91 | } 92 | } 93 | 94 | impl From for MapError { 95 | fn from(err: OutOfMemory) -> Self { 96 | match err { 97 | OutOfMemory::OutOfDeviceMemory => MapError::OutOfDeviceMemory, 98 | OutOfMemory::OutOfHostMemory => MapError::OutOfHostMemory, 99 | } 100 | } 101 | } 102 | 103 | impl Display for MapError { 104 | fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { 105 | match self { 106 | MapError::OutOfDeviceMemory => fmt.write_str("Device memory exhausted"), 107 | MapError::OutOfHostMemory => fmt.write_str("Host memory exhausted"), 108 | MapError::MapFailed => fmt.write_str("Failed to map memory object"), 109 | MapError::NonHostVisible => fmt.write_str("Impossible to map non-host-visible memory"), 110 | MapError::AlreadyMapped => fmt.write_str("Block is already mapped"), 111 | } 112 | } 113 | } 114 | 115 | #[cfg(feature = "std")] 116 | impl std::error::Error for MapError {} 117 | -------------------------------------------------------------------------------- /gpu-alloc/src/freelist.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::{ 3 | align_down, align_up, 4 | error::AllocationError, 5 | heap::Heap, 6 | util::{arc_unwrap, is_arc_unique}, 7 | MemoryBounds, 8 | }, 9 | alloc::{sync::Arc, vec::Vec}, 10 | core::{cmp::Ordering, ptr::NonNull}, 11 | gpu_alloc_types::{AllocationFlags, DeviceMapError, MemoryDevice, MemoryPropertyFlags}, 12 | }; 13 | 14 | unsafe fn opt_ptr_add(ptr: Option>, size: u64) -> Option> { 15 | ptr.map(|ptr| { 16 | // Size is within memory region started at `ptr`. 17 | // size is within `chunk_size` that fits `isize`. 18 | NonNull::new_unchecked(ptr.as_ptr().offset(size as isize)) 19 | }) 20 | } 21 | 22 | #[derive(Debug)] 23 | pub(super) struct FreeList { 24 | array: Vec>, 25 | counter: u64, 26 | } 27 | 28 | impl FreeList { 29 | pub fn new() -> Self { 30 | FreeList { 31 | array: Vec::new(), 32 | counter: 0, 33 | } 34 | } 35 | 36 | pub fn get_block_from_new_memory( 37 | &mut self, 38 | memory: Arc, 39 | memory_size: u64, 40 | ptr: Option>, 41 | align_mask: u64, 42 | size: u64, 43 | ) -> FreeListBlock { 44 | debug_assert!(size <= memory_size); 45 | 46 | self.counter += 1; 47 | self.array.push(FreeListRegion { 48 | memory, 49 | ptr, 50 | chunk: self.counter, 51 | start: 0, 52 | end: memory_size, 53 | }); 54 | self.get_block_at(self.array.len() - 1, align_mask, size) 55 | } 56 | 57 | pub fn get_block(&mut self, align_mask: u64, size: u64) -> Option> { 58 | let (index, _) = self.array.iter().enumerate().rev().find(|(_, region)| { 59 | match region.end.checked_sub(size) { 60 | Some(start) => { 61 | let aligned_start = align_down(start, align_mask); 62 | aligned_start >= region.start 63 | } 64 | None => false, 65 | } 66 | })?; 67 | 68 | Some(self.get_block_at(index, align_mask, size)) 69 | } 70 | 71 | fn get_block_at(&mut self, index: usize, align_mask: u64, size: u64) -> FreeListBlock { 72 | let region = &mut self.array[index]; 73 | 74 | let start = region.end - size; 75 | let aligned_start = align_down(start, align_mask); 76 | 77 | if aligned_start > region.start { 78 | let block = FreeListBlock { 79 | offset: aligned_start, 80 | size: region.end - aligned_start, 81 | chunk: region.chunk, 82 | ptr: unsafe { opt_ptr_add(region.ptr, aligned_start - region.start) }, 83 | memory: region.memory.clone(), 84 | }; 85 | 86 | region.end = aligned_start; 87 | 88 | block 89 | } else { 90 | debug_assert_eq!(aligned_start, region.start); 91 | let region = self.array.remove(index); 92 | region.into_block() 93 | } 94 | } 95 | 96 | pub fn insert_block(&mut self, block: FreeListBlock) { 97 | match self.array.binary_search_by(|b| b.cmp(&block)) { 98 | Ok(_) => { 99 | panic!("Overlapping block found in free list"); 100 | } 101 | Err(index) if self.array.len() > index => match &mut self.array[..=index] { 102 | [] => unreachable!(), 103 | [next] => { 104 | debug_assert!(!next.is_suffix_block(&block)); 105 | 106 | if next.is_prefix_block(&block) { 107 | next.merge_prefix_block(block); 108 | } else { 109 | self.array.insert(0, FreeListRegion::from_block(block)); 110 | } 111 | } 112 | [.., prev, next] => { 113 | debug_assert!(!prev.is_prefix_block(&block)); 114 | debug_assert!(!next.is_suffix_block(&block)); 115 | 116 | if next.is_prefix_block(&block) { 117 | next.merge_prefix_block(block); 118 | 119 | if prev.consecutive(&*next) { 120 | let next = self.array.remove(index); 121 | let prev = &mut self.array[index - 1]; 122 | prev.merge(next); 123 | } 124 | } else if prev.is_suffix_block(&block) { 125 | prev.merge_suffix_block(block); 126 | } else { 127 | self.array.insert(index, FreeListRegion::from_block(block)); 128 | } 129 | } 130 | }, 131 | Err(_) => match &mut self.array[..] { 132 | [] => self.array.push(FreeListRegion::from_block(block)), 133 | [.., prev] => { 134 | debug_assert!(!prev.is_prefix_block(&block)); 135 | if prev.is_suffix_block(&block) { 136 | prev.merge_suffix_block(block); 137 | } else { 138 | self.array.push(FreeListRegion::from_block(block)); 139 | } 140 | } 141 | }, 142 | } 143 | } 144 | 145 | pub fn drain(&mut self, keep_last: bool) -> Option + '_> { 146 | // Time to deallocate 147 | 148 | let len = self.array.len(); 149 | 150 | let mut del = 0; 151 | { 152 | let regions = &mut self.array[..]; 153 | 154 | for i in 0..len { 155 | if (i < len - 1 || !keep_last) && is_arc_unique(&mut regions[i].memory) { 156 | del += 1; 157 | } else if del > 0 { 158 | regions.swap(i - del, i); 159 | } 160 | } 161 | } 162 | 163 | if del > 0 { 164 | Some(self.array.drain(len - del..).map(move |region| { 165 | debug_assert_eq!(region.start, 0); 166 | (unsafe { arc_unwrap(region.memory) }, region.end) 167 | })) 168 | } else { 169 | None 170 | } 171 | } 172 | } 173 | 174 | #[derive(Debug)] 175 | struct FreeListRegion { 176 | memory: Arc, 177 | ptr: Option>, 178 | chunk: u64, 179 | start: u64, 180 | end: u64, 181 | } 182 | 183 | unsafe impl Sync for FreeListRegion where M: Sync {} 184 | unsafe impl Send for FreeListRegion where M: Send {} 185 | 186 | impl FreeListRegion { 187 | pub fn cmp(&self, block: &FreeListBlock) -> Ordering { 188 | debug_assert_eq!( 189 | Arc::ptr_eq(&self.memory, &block.memory), 190 | self.chunk == block.chunk 191 | ); 192 | 193 | if self.chunk == block.chunk { 194 | debug_assert_eq!( 195 | Ord::cmp(&self.start, &block.offset), 196 | Ord::cmp(&self.end, &(block.offset + block.size)), 197 | "Free region {{ start: {}, end: {} }} overlaps with block {{ offset: {}, size: {} }}", 198 | self.start, 199 | self.end, 200 | block.offset, 201 | block.size, 202 | ); 203 | } 204 | 205 | Ord::cmp(&self.chunk, &block.chunk).then(Ord::cmp(&self.start, &block.offset)) 206 | } 207 | 208 | fn from_block(block: FreeListBlock) -> Self { 209 | FreeListRegion { 210 | memory: block.memory, 211 | chunk: block.chunk, 212 | ptr: block.ptr, 213 | start: block.offset, 214 | end: block.offset + block.size, 215 | } 216 | } 217 | 218 | fn into_block(self) -> FreeListBlock { 219 | FreeListBlock { 220 | memory: self.memory, 221 | chunk: self.chunk, 222 | ptr: self.ptr, 223 | offset: self.start, 224 | size: self.end - self.start, 225 | } 226 | } 227 | 228 | fn consecutive(&self, other: &Self) -> bool { 229 | if self.chunk != other.chunk { 230 | return false; 231 | } 232 | 233 | debug_assert!(Arc::ptr_eq(&self.memory, &other.memory)); 234 | 235 | debug_assert_eq!( 236 | Ord::cmp(&self.start, &other.start), 237 | Ord::cmp(&self.end, &other.end) 238 | ); 239 | 240 | self.end == other.start 241 | } 242 | 243 | fn merge(&mut self, next: FreeListRegion) { 244 | debug_assert!(self.consecutive(&next)); 245 | self.end = next.end; 246 | } 247 | 248 | fn is_prefix_block(&self, block: &FreeListBlock) -> bool { 249 | if self.chunk != block.chunk { 250 | return false; 251 | } 252 | 253 | debug_assert!(Arc::ptr_eq(&self.memory, &block.memory)); 254 | 255 | debug_assert_eq!( 256 | Ord::cmp(&self.start, &block.offset), 257 | Ord::cmp(&self.end, &(block.offset + block.size)) 258 | ); 259 | 260 | self.start == (block.offset + block.size) 261 | } 262 | 263 | fn merge_prefix_block(&mut self, block: FreeListBlock) { 264 | debug_assert!(self.is_prefix_block(&block)); 265 | self.start = block.offset; 266 | self.ptr = block.ptr; 267 | } 268 | 269 | fn is_suffix_block(&self, block: &FreeListBlock) -> bool { 270 | if self.chunk != block.chunk { 271 | return false; 272 | } 273 | 274 | debug_assert!(Arc::ptr_eq(&self.memory, &block.memory)); 275 | 276 | debug_assert_eq!( 277 | Ord::cmp(&self.start, &block.offset), 278 | Ord::cmp(&self.end, &(block.offset + block.size)) 279 | ); 280 | 281 | self.end == block.offset 282 | } 283 | 284 | fn merge_suffix_block(&mut self, block: FreeListBlock) { 285 | debug_assert!(self.is_suffix_block(&block)); 286 | self.end += block.size; 287 | } 288 | } 289 | 290 | #[derive(Debug)] 291 | pub struct FreeListBlock { 292 | pub memory: Arc, 293 | pub ptr: Option>, 294 | pub chunk: u64, 295 | pub offset: u64, 296 | pub size: u64, 297 | } 298 | 299 | unsafe impl Sync for FreeListBlock where M: Sync {} 300 | unsafe impl Send for FreeListBlock where M: Send {} 301 | 302 | #[derive(Debug)] 303 | pub(crate) struct FreeListAllocator { 304 | freelist: FreeList, 305 | chunk_size: u64, 306 | final_chunk_size: u64, 307 | memory_type: u32, 308 | props: MemoryPropertyFlags, 309 | atom_mask: u64, 310 | 311 | total_allocations: u64, 312 | total_deallocations: u64, 313 | } 314 | 315 | impl Drop for FreeListAllocator { 316 | fn drop(&mut self) { 317 | match Ord::cmp(&self.total_allocations, &self.total_deallocations) { 318 | Ordering::Equal => {} 319 | Ordering::Greater => { 320 | report_error_on_drop!("Not all blocks were deallocated") 321 | } 322 | Ordering::Less => { 323 | report_error_on_drop!("More blocks deallocated than allocated") 324 | } 325 | } 326 | 327 | if !self.freelist.array.is_empty() { 328 | report_error_on_drop!( 329 | "FreeListAllocator has free blocks on drop. Allocator should be cleaned" 330 | ); 331 | } 332 | } 333 | } 334 | 335 | impl FreeListAllocator 336 | where 337 | M: MemoryBounds + 'static, 338 | { 339 | pub fn new( 340 | starting_chunk_size: u64, 341 | final_chunk_size: u64, 342 | memory_type: u32, 343 | props: MemoryPropertyFlags, 344 | atom_mask: u64, 345 | ) -> Self { 346 | debug_assert_eq!( 347 | align_down(starting_chunk_size, atom_mask), 348 | starting_chunk_size 349 | ); 350 | 351 | let starting_chunk_size = min(starting_chunk_size, isize::max_value()); 352 | 353 | debug_assert_eq!(align_down(final_chunk_size, atom_mask), final_chunk_size); 354 | let final_chunk_size = min(final_chunk_size, isize::max_value()); 355 | 356 | FreeListAllocator { 357 | freelist: FreeList::new(), 358 | chunk_size: starting_chunk_size, 359 | final_chunk_size, 360 | memory_type, 361 | props, 362 | atom_mask, 363 | 364 | total_allocations: 0, 365 | total_deallocations: 0, 366 | } 367 | } 368 | 369 | #[cfg_attr(feature = "tracing", tracing::instrument(skip(self, device)))] 370 | pub unsafe fn alloc( 371 | &mut self, 372 | device: &impl MemoryDevice, 373 | size: u64, 374 | align_mask: u64, 375 | flags: AllocationFlags, 376 | heap: &mut Heap, 377 | allocations_remains: &mut u32, 378 | ) -> Result, AllocationError> { 379 | debug_assert!( 380 | self.final_chunk_size >= size, 381 | "GpuAllocator must not request allocations equal or greater to chunks size" 382 | ); 383 | 384 | let size = align_up(size, self.atom_mask).expect( 385 | "Any value not greater than final chunk size (which is aligned) has to fit for alignment", 386 | ); 387 | 388 | let align_mask = align_mask | self.atom_mask; 389 | let host_visible = self.host_visible(); 390 | 391 | if size <= self.chunk_size { 392 | // Otherwise there can't be any sufficiently large free blocks 393 | if let Some(block) = self.freelist.get_block(align_mask, size) { 394 | self.total_allocations += 1; 395 | return Ok(block); 396 | } 397 | } 398 | 399 | // New allocation is required. 400 | if *allocations_remains == 0 { 401 | return Err(AllocationError::TooManyObjects); 402 | } 403 | 404 | if size > self.chunk_size { 405 | let multiple = (size - 1) / self.chunk_size + 1; 406 | let multiple = multiple.next_power_of_two(); 407 | 408 | self.chunk_size = (self.chunk_size * multiple).min(self.final_chunk_size); 409 | } 410 | 411 | let mut memory = device.allocate_memory(self.chunk_size, self.memory_type, flags)?; 412 | *allocations_remains -= 1; 413 | heap.alloc(self.chunk_size); 414 | 415 | // Map host visible allocations 416 | let ptr = if host_visible { 417 | match device.map_memory(&mut memory, 0, self.chunk_size) { 418 | Ok(ptr) => Some(ptr), 419 | Err(DeviceMapError::MapFailed) => { 420 | #[cfg(feature = "tracing")] 421 | tracing::error!("Failed to map host-visible memory in linear allocator"); 422 | device.deallocate_memory(memory); 423 | *allocations_remains += 1; 424 | heap.dealloc(self.chunk_size); 425 | 426 | return Err(AllocationError::OutOfHostMemory); 427 | } 428 | Err(DeviceMapError::OutOfDeviceMemory) => { 429 | return Err(AllocationError::OutOfDeviceMemory); 430 | } 431 | Err(DeviceMapError::OutOfHostMemory) => { 432 | return Err(AllocationError::OutOfHostMemory); 433 | } 434 | } 435 | } else { 436 | None 437 | }; 438 | 439 | let memory = Arc::new(memory); 440 | let block = 441 | self.freelist 442 | .get_block_from_new_memory(memory, self.chunk_size, ptr, align_mask, size); 443 | 444 | if self.chunk_size < self.final_chunk_size { 445 | // Double next chunk size 446 | // Limit to final value. 447 | self.chunk_size = (self.chunk_size * 2).min(self.final_chunk_size); 448 | } 449 | 450 | self.total_allocations += 1; 451 | Ok(block) 452 | } 453 | 454 | #[cfg_attr(feature = "tracing", tracing::instrument(skip(self, device)))] 455 | pub unsafe fn dealloc( 456 | &mut self, 457 | device: &impl MemoryDevice, 458 | block: FreeListBlock, 459 | heap: &mut Heap, 460 | allocations_remains: &mut u32, 461 | ) { 462 | debug_assert!(block.size < self.chunk_size); 463 | debug_assert_ne!(block.size, 0); 464 | self.freelist.insert_block(block); 465 | self.total_deallocations += 1; 466 | 467 | if let Some(memory) = self.freelist.drain(true) { 468 | memory.for_each(|(memory, size)| { 469 | device.deallocate_memory(memory); 470 | *allocations_remains += 1; 471 | heap.dealloc(size); 472 | }); 473 | } 474 | } 475 | 476 | /// Deallocates leftover memory objects. 477 | /// Should be used before dropping. 478 | /// 479 | /// # Safety 480 | /// 481 | /// * `device` must be one with `DeviceProperties` that were provided to create this `GpuAllocator` instance 482 | /// * Same `device` instance must be used for all interactions with one `GpuAllocator` instance 483 | /// and memory blocks allocated from it 484 | #[cfg_attr(feature = "tracing", tracing::instrument(skip(self, device)))] 485 | pub unsafe fn cleanup( 486 | &mut self, 487 | device: &impl MemoryDevice, 488 | heap: &mut Heap, 489 | allocations_remains: &mut u32, 490 | ) { 491 | if let Some(memory) = self.freelist.drain(false) { 492 | memory.for_each(|(memory, size)| { 493 | device.deallocate_memory(memory); 494 | *allocations_remains += 1; 495 | heap.dealloc(size); 496 | }); 497 | } 498 | 499 | #[cfg(feature = "tracing")] 500 | { 501 | if self.total_allocations == self.total_deallocations && !self.freelist.array.is_empty() 502 | { 503 | tracing::error!( 504 | "Some regions were not deallocated on cleanup, although all blocks are free. 505 | This is a bug in `FreeBlockAllocator`. 506 | See array of free blocks left: 507 | {:#?}", 508 | self.freelist.array, 509 | ); 510 | } 511 | } 512 | } 513 | 514 | fn host_visible(&self) -> bool { 515 | self.props.contains(MemoryPropertyFlags::HOST_VISIBLE) 516 | } 517 | } 518 | 519 | fn min(l: L, r: R) -> L 520 | where 521 | R: core::convert::TryInto, 522 | L: Ord, 523 | { 524 | match r.try_into() { 525 | Ok(r) => core::cmp::min(l, r), 526 | Err(_) => l, 527 | } 528 | } 529 | -------------------------------------------------------------------------------- /gpu-alloc/src/heap.rs: -------------------------------------------------------------------------------- 1 | #[derive(Debug)] 2 | pub(crate) struct Heap { 3 | size: u64, 4 | used: u64, 5 | allocated: u128, 6 | deallocated: u128, 7 | } 8 | 9 | impl Heap { 10 | pub(crate) fn new(size: u64) -> Self { 11 | Heap { 12 | size, 13 | used: 0, 14 | allocated: 0, 15 | deallocated: 0, 16 | } 17 | } 18 | 19 | pub(crate) fn size(&mut self) -> u64 { 20 | self.size 21 | } 22 | 23 | pub(crate) fn alloc(&mut self, size: u64) { 24 | self.used += size; 25 | self.allocated += u128::from(size); 26 | } 27 | 28 | pub(crate) fn dealloc(&mut self, size: u64) { 29 | self.used -= size; 30 | self.deallocated += u128::from(size); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /gpu-alloc/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! 2 | //! Implementation agnostic memory allocator for Vulkan like APIs. 3 | //! 4 | //! This crate is intended to be used as part of safe API implementations.\ 5 | //! Use with caution. There are unsafe functions all over the place. 6 | //! 7 | //! # Usage 8 | //! 9 | //! Start with fetching `DeviceProperties` from `gpu-alloc-` crate for the backend of choice.\ 10 | //! Then create `GpuAllocator` instance and use it for all device memory allocations.\ 11 | //! `GpuAllocator` will take care for all necessary bookkeeping like memory object count limit, 12 | //! heap budget and memory mapping. 13 | //! 14 | //! ### Backends implementations 15 | //! 16 | //! Backend supporting crates should not depend on this crate.\ 17 | //! Instead they should depend on `gpu-alloc-types` which is much more stable, 18 | //! allowing to upgrade `gpu-alloc` version without `gpu-alloc-` upgrade. 19 | //! 20 | 21 | #![cfg_attr(not(feature = "std"), no_std)] 22 | 23 | extern crate alloc; 24 | 25 | #[cfg(feature = "tracing")] 26 | macro_rules! report_error_on_drop { 27 | ($($tokens:tt)*) => {{ 28 | #[cfg(feature = "std")] 29 | { 30 | if std::thread::panicking() { 31 | return; 32 | } 33 | } 34 | 35 | tracing::error!($($tokens)*) 36 | }}; 37 | } 38 | 39 | #[cfg(all(not(feature = "tracing"), feature = "std"))] 40 | macro_rules! report_error_on_drop { 41 | ($($tokens:tt)*) => {{ 42 | if std::thread::panicking() { 43 | return; 44 | } 45 | eprintln!($($tokens)*) 46 | }}; 47 | } 48 | 49 | #[cfg(all(not(feature = "tracing"), not(feature = "std")))] 50 | macro_rules! report_error_on_drop { 51 | ($($tokens:tt)*) => {{ 52 | panic!($($tokens)*) 53 | }}; 54 | } 55 | 56 | mod allocator; 57 | mod block; 58 | mod buddy; 59 | mod config; 60 | mod error; 61 | mod freelist; 62 | mod heap; 63 | mod slab; 64 | mod usage; 65 | mod util; 66 | 67 | pub use { 68 | self::{allocator::*, block::MemoryBlock, config::*, error::*, usage::*}, 69 | gpu_alloc_types::*, 70 | }; 71 | 72 | /// Memory request for allocator. 73 | #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] 74 | pub struct Request { 75 | /// Minimal size of memory block required. 76 | /// Returned block may have larger size, 77 | /// use `MemoryBlock::size` to learn actual size of returned block. 78 | pub size: u64, 79 | 80 | /// Minimal alignment mask required. 81 | /// Returned block may have larger alignment, 82 | /// use `MemoryBlock::align` to learn actual alignment of returned block. 83 | pub align_mask: u64, 84 | 85 | /// Intended memory usage. 86 | /// Returned block may support additional usages, 87 | /// use `MemoryBlock::props` to learn memory properties of returned block. 88 | pub usage: UsageFlags, 89 | 90 | /// Bitset for memory types. 91 | /// Returned block will be from memory type corresponding to one of set bits, 92 | /// use `MemoryBlock::memory_type` to learn memory type index of returned block. 93 | pub memory_types: u32, 94 | } 95 | 96 | /// Aligns `value` up to `align_mask` 97 | /// Returns smallest integer not lesser than `value` aligned by `align_mask`. 98 | /// Returns `None` on overflow. 99 | pub(crate) fn align_up(value: u64, align_mask: u64) -> Option { 100 | Some(value.checked_add(align_mask)? & !align_mask) 101 | } 102 | 103 | /// Align `value` down to `align_mask` 104 | /// Returns largest integer not bigger than `value` aligned by `align_mask`. 105 | pub(crate) fn align_down(value: u64, align_mask: u64) -> u64 { 106 | value & !align_mask 107 | } 108 | 109 | #[cfg(debug_assertions)] 110 | #[allow(unused_unsafe)] 111 | unsafe fn unreachable_unchecked() -> ! { 112 | unreachable!() 113 | } 114 | 115 | #[cfg(not(debug_assertions))] 116 | unsafe fn unreachable_unchecked() -> ! { 117 | core::hint::unreachable_unchecked() 118 | } 119 | 120 | // #[cfg(feature = "tracing")] 121 | use core::fmt::Debug as MemoryBounds; 122 | 123 | // #[cfg(not(feature = "tracing"))] 124 | // use core::any::Any as MemoryBounds; 125 | -------------------------------------------------------------------------------- /gpu-alloc/src/slab.rs: -------------------------------------------------------------------------------- 1 | use {crate::unreachable_unchecked, alloc::vec::Vec, core::mem::replace}; 2 | 3 | #[derive(Debug)] 4 | enum Entry { 5 | Vacant(usize), 6 | Occupied(T), 7 | } 8 | #[derive(Debug)] 9 | pub(crate) struct Slab { 10 | next_vacant: usize, 11 | entries: Vec>, 12 | } 13 | 14 | impl Slab { 15 | pub fn new() -> Self { 16 | Slab { 17 | next_vacant: !0, 18 | entries: Vec::new(), 19 | } 20 | } 21 | 22 | /// Inserts value into this linked vec and returns index 23 | /// at which value can be accessed in constant time. 24 | pub fn insert(&mut self, value: T) -> usize { 25 | if self.next_vacant >= self.entries.len() { 26 | self.entries.push(Entry::Occupied(value)); 27 | self.entries.len() - 1 28 | } else { 29 | match *unsafe { self.entries.get_unchecked(self.next_vacant) } { 30 | Entry::Vacant(next_vacant) => { 31 | unsafe { 32 | *self.entries.get_unchecked_mut(self.next_vacant) = Entry::Occupied(value); 33 | } 34 | replace(&mut self.next_vacant, next_vacant) 35 | } 36 | _ => unsafe { unreachable_unchecked() }, 37 | } 38 | } 39 | } 40 | 41 | pub fn len(&self) -> usize { 42 | self.entries.len() 43 | } 44 | 45 | pub unsafe fn get_unchecked(&self, index: usize) -> &T { 46 | debug_assert!(index < self.len()); 47 | 48 | match self.entries.get_unchecked(index) { 49 | Entry::Occupied(value) => value, 50 | _ => unreachable_unchecked(), 51 | } 52 | } 53 | 54 | pub unsafe fn get_unchecked_mut(&mut self, index: usize) -> &mut T { 55 | debug_assert!(index < self.len()); 56 | 57 | match self.entries.get_unchecked_mut(index) { 58 | Entry::Occupied(value) => value, 59 | _ => unreachable_unchecked(), 60 | } 61 | } 62 | 63 | pub fn get(&self, index: usize) -> &T { 64 | match self.entries.get(index) { 65 | Some(Entry::Occupied(value)) => value, 66 | _ => panic!("Invalid index"), 67 | } 68 | } 69 | 70 | pub fn get_mut(&mut self, index: usize) -> &mut T { 71 | match self.entries.get_mut(index) { 72 | Some(Entry::Occupied(value)) => value, 73 | _ => panic!("Invalid index"), 74 | } 75 | } 76 | 77 | pub unsafe fn remove_unchecked(&mut self, index: usize) -> T { 78 | let entry = replace( 79 | self.entries.get_unchecked_mut(index), 80 | Entry::Vacant(self.next_vacant), 81 | ); 82 | 83 | self.next_vacant = index; 84 | 85 | match entry { 86 | Entry::Occupied(value) => value, 87 | _ => unreachable_unchecked(), 88 | } 89 | } 90 | 91 | pub fn remove(&mut self, index: usize) -> T { 92 | match self.entries.get_mut(index) { 93 | Some(Entry::Occupied(_)) => unsafe { self.remove_unchecked(index) }, 94 | _ => panic!("Invalid index"), 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /gpu-alloc/src/usage.rs: -------------------------------------------------------------------------------- 1 | use { 2 | core::fmt::{self, Debug}, 3 | gpu_alloc_types::{MemoryPropertyFlags, MemoryType}, 4 | }; 5 | 6 | bitflags::bitflags! { 7 | /// Memory usage type. 8 | /// Bits set define intended usage for requested memory. 9 | #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] 10 | #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] 11 | pub struct UsageFlags: u8 { 12 | /// Hints for allocator to find memory with faster device access. 13 | /// If no flags is specified than `FAST_DEVICE_ACCESS` is implied. 14 | const FAST_DEVICE_ACCESS = 0x01; 15 | 16 | /// Memory will be accessed from host. 17 | /// This flags guarantees that host memory operations will be available. 18 | /// Otherwise implementation is encouraged to use non-host-accessible memory. 19 | const HOST_ACCESS = 0x02; 20 | 21 | /// Hints allocator that memory will be used for data downloading. 22 | /// Allocator will strongly prefer host-cached memory. 23 | /// Implies `HOST_ACCESS` flag. 24 | const DOWNLOAD = 0x04; 25 | 26 | /// Hints allocator that memory will be used for data uploading. 27 | /// If `DOWNLOAD` flag is not set then allocator will assume that 28 | /// host will access memory in write-only manner and may 29 | /// pick not host-cached. 30 | /// Implies `HOST_ACCESS` flag. 31 | const UPLOAD = 0x08; 32 | 33 | /// Hints allocator that memory will be used for short duration 34 | /// allowing to use faster algorithm with less memory overhead. 35 | /// If use holds returned memory block for too long then 36 | /// effective memory overhead increases instead. 37 | /// Best use case is for staging buffer for single batch of operations. 38 | const TRANSIENT = 0x10; 39 | 40 | /// Requests memory that can be addressed with `u64`. 41 | /// Allows fetching device address for resources bound to that memory. 42 | const DEVICE_ADDRESS = 0x20; 43 | } 44 | } 45 | 46 | #[derive(Clone, Copy, Debug)] 47 | struct MemoryForOneUsage { 48 | mask: u32, 49 | types: [u32; 32], 50 | types_count: u32, 51 | } 52 | 53 | pub(crate) struct MemoryForUsage { 54 | usages: [MemoryForOneUsage; 64], 55 | } 56 | 57 | impl Debug for MemoryForUsage { 58 | fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { 59 | fmt.debug_struct("MemoryForUsage") 60 | .field("usages", &&self.usages[..]) 61 | .finish() 62 | } 63 | } 64 | 65 | impl MemoryForUsage { 66 | pub fn new(memory_types: &[MemoryType]) -> Self { 67 | assert!( 68 | memory_types.len() <= 32, 69 | "Only up to 32 memory types supported" 70 | ); 71 | 72 | let mut mfu = MemoryForUsage { 73 | usages: [MemoryForOneUsage { 74 | mask: 0, 75 | types: [0; 32], 76 | types_count: 0, 77 | }; 64], 78 | }; 79 | 80 | for usage in 0..64 { 81 | mfu.usages[usage as usize] = 82 | one_usage(UsageFlags::from_bits_truncate(usage), memory_types); 83 | } 84 | 85 | mfu 86 | } 87 | 88 | /// Returns mask with bits set for memory type indices that support the 89 | /// usage. 90 | pub fn mask(&self, usage: UsageFlags) -> u32 { 91 | self.usages[usage.bits() as usize].mask 92 | } 93 | 94 | /// Returns slice of memory type indices that support the usage. 95 | /// Earlier memory type has priority over later. 96 | pub fn types(&self, usage: UsageFlags) -> &[u32] { 97 | let usage = &self.usages[usage.bits() as usize]; 98 | &usage.types[..usage.types_count as usize] 99 | } 100 | } 101 | 102 | fn one_usage(usage: UsageFlags, memory_types: &[MemoryType]) -> MemoryForOneUsage { 103 | let mut types = [0; 32]; 104 | let mut types_count = 0; 105 | 106 | for (index, mt) in memory_types.iter().enumerate() { 107 | if compatible(usage, mt.props) { 108 | types[types_count as usize] = index as u32; 109 | types_count += 1; 110 | } 111 | } 112 | 113 | types[..types_count as usize] 114 | .sort_unstable_by_key(|&index| reverse_priority(usage, memory_types[index as usize].props)); 115 | 116 | let mask = types[..types_count as usize] 117 | .iter() 118 | .fold(0u32, |mask, index| mask | 1u32 << index); 119 | 120 | MemoryForOneUsage { 121 | mask, 122 | types, 123 | types_count, 124 | } 125 | } 126 | 127 | fn compatible(usage: UsageFlags, flags: MemoryPropertyFlags) -> bool { 128 | type Flags = MemoryPropertyFlags; 129 | if flags.contains(Flags::LAZILY_ALLOCATED) || flags.contains(Flags::PROTECTED) { 130 | // Unsupported 131 | false 132 | } else if usage.intersects(UsageFlags::HOST_ACCESS | UsageFlags::UPLOAD | UsageFlags::DOWNLOAD) 133 | { 134 | // Requires HOST_VISIBLE 135 | flags.contains(Flags::HOST_VISIBLE) 136 | } else { 137 | true 138 | } 139 | } 140 | 141 | /// Returns reversed priority of memory with specified flags for specified usage. 142 | /// Lesser value returned = more prioritized. 143 | fn reverse_priority(usage: UsageFlags, flags: MemoryPropertyFlags) -> u32 { 144 | type Flags = MemoryPropertyFlags; 145 | 146 | // Highly prefer device local memory when `FAST_DEVICE_ACCESS` usage is specified 147 | // or usage is empty. 148 | let device_local: bool = flags.contains(Flags::DEVICE_LOCAL) 149 | ^ (usage.is_empty() || usage.contains(UsageFlags::FAST_DEVICE_ACCESS)); 150 | 151 | assert!( 152 | flags.contains(Flags::HOST_VISIBLE) 153 | || !usage 154 | .intersects(UsageFlags::HOST_ACCESS | UsageFlags::UPLOAD | UsageFlags::DOWNLOAD) 155 | ); 156 | 157 | // Prefer non-host-visible memory when host access is not required. 158 | let host_visible: bool = flags.contains(Flags::HOST_VISIBLE) 159 | ^ usage.intersects(UsageFlags::HOST_ACCESS | UsageFlags::UPLOAD | UsageFlags::DOWNLOAD); 160 | 161 | // Prefer cached memory for downloads. 162 | // Or non-cached if downloads are not expected. 163 | let host_cached: bool = 164 | flags.contains(Flags::HOST_CACHED) ^ usage.contains(UsageFlags::DOWNLOAD); 165 | 166 | // Prefer coherent for both uploads and downloads. 167 | // Prefer non-coherent if neither flags is set. 168 | let host_coherent: bool = flags.contains(Flags::HOST_COHERENT) 169 | ^ (usage.intersects(UsageFlags::UPLOAD | UsageFlags::DOWNLOAD)); 170 | 171 | // Each boolean is false if flags are preferred. 172 | device_local as u32 * 8 173 | + host_visible as u32 * 4 174 | + host_cached as u32 * 2 175 | + host_coherent as u32 176 | } 177 | -------------------------------------------------------------------------------- /gpu-alloc/src/util.rs: -------------------------------------------------------------------------------- 1 | use alloc::sync::Arc; 2 | 3 | /// Guarantees uniqueness only if `Weak` pointers are never created 4 | /// from this `Arc` or clones. 5 | pub(crate) fn is_arc_unique(arc: &mut Arc) -> bool { 6 | let strong_count = Arc::strong_count(&*arc); 7 | debug_assert_ne!(strong_count, 0, "This Arc should exist"); 8 | 9 | debug_assert!( 10 | strong_count > 1 || Arc::get_mut(arc).is_some(), 11 | "`Weak` pointer exists" 12 | ); 13 | 14 | strong_count == 1 15 | } 16 | 17 | /// Can be used instead of `Arc::try_unwrap(arc).unwrap()` 18 | /// when it is guaranteed to succeed. 19 | pub(crate) unsafe fn arc_unwrap(mut arc: Arc) -> M { 20 | use core::{mem::ManuallyDrop, ptr::read}; 21 | debug_assert!(is_arc_unique(&mut arc)); 22 | 23 | // Get raw pointer to inner value. 24 | let raw = Arc::into_raw(arc); 25 | 26 | // As `Arc` is unique and no Weak pointers exist 27 | // it won't be dereferenced elsewhere. 28 | let inner = read(raw); 29 | 30 | // Cast to `ManuallyDrop` which guarantees to have same layout 31 | // and will skip dropping. 32 | drop(Arc::from_raw(raw as *const ManuallyDrop)); 33 | inner 34 | } 35 | 36 | /// Can be used instead of `Arc::try_unwrap` 37 | /// only if `Weak` pointers are never created from this `Arc` or clones. 38 | pub(crate) unsafe fn try_arc_unwrap(mut arc: Arc) -> Option { 39 | if is_arc_unique(&mut arc) { 40 | Some(arc_unwrap(arc)) 41 | } else { 42 | None 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /license/APACHE: -------------------------------------------------------------------------------- 1 | Copyright 2020 The gpu-alloc project developers 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /license/MIT: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020 The gpu-alloc project developers 2 | 3 | Permission is hereby granted, free of charge, to any 4 | person obtaining a copy of this software and associated 5 | documentation files (the "Software"), to deal in the 6 | Software without restriction, including without 7 | limitation the rights to use, copy, modify, merge, 8 | publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software 10 | is furnished to do so, subject to the following 11 | conditions: 12 | 13 | The above copyright notice and this permission notice 14 | shall be included in all copies or substantial portions 15 | of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 18 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 19 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 20 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 21 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 22 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 23 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 24 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /mock/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "gpu-alloc-mock" 3 | version = "0.3.0" 4 | authors = ["Zakarum "] 5 | edition = "2018" 6 | publish = false 7 | 8 | [dependencies] 9 | gpu-alloc-types = { path = "../types", version = "=0.3.0" } 10 | tracing = { version = "0.1", features = ["attributes"] } 11 | slab = "0.4" 12 | -------------------------------------------------------------------------------- /mock/src/lib.rs: -------------------------------------------------------------------------------- 1 | use { 2 | gpu_alloc_types::{ 3 | AllocationFlags, DeviceMapError, DeviceProperties, MappedMemoryRange, MemoryDevice, 4 | MemoryHeap, MemoryPropertyFlags, MemoryType, OutOfMemory, 5 | }, 6 | slab::Slab, 7 | std::{ 8 | borrow::Cow, 9 | cell::{Cell, RefCell, UnsafeCell}, 10 | convert::TryFrom as _, 11 | mem::transmute, 12 | ptr::NonNull, 13 | }, 14 | }; 15 | 16 | struct MemoryMapping { 17 | content: Box>, 18 | offset: u64, 19 | } 20 | 21 | struct MockMemory { 22 | memory_type: u32, 23 | size: u64, 24 | mapped: Option, 25 | } 26 | 27 | pub struct MockMemoryDevice { 28 | memory_types: Box<[MemoryType]>, 29 | memory_heaps: Box<[MemoryHeap]>, 30 | max_memory_allocation_count: u32, 31 | max_memory_allocation_size: u64, 32 | non_coherent_atom_size: u64, 33 | buffer_device_address: bool, 34 | 35 | allocations_remains: Cell, 36 | memory_heaps_remaining_capacity: Box<[Cell]>, 37 | allocations: RefCell>, 38 | 39 | total_allocations_counter: Cell, 40 | total_deallocations_counter: Cell, 41 | } 42 | 43 | impl MockMemoryDevice { 44 | pub fn new(props: DeviceProperties<'_>) -> Self { 45 | MockMemoryDevice { 46 | memory_heaps_remaining_capacity: props 47 | .memory_heaps 48 | .as_ref() 49 | .iter() 50 | .map(|heap| Cell::new(heap.size)) 51 | .collect(), 52 | 53 | memory_types: props.memory_types.into_owned().into_boxed_slice(), 54 | memory_heaps: props.memory_heaps.into_owned().into_boxed_slice(), 55 | max_memory_allocation_count: props.max_memory_allocation_count, 56 | max_memory_allocation_size: props.max_memory_allocation_size, 57 | non_coherent_atom_size: props.non_coherent_atom_size, 58 | buffer_device_address: props.buffer_device_address, 59 | 60 | allocations_remains: Cell::new(props.max_memory_allocation_count), 61 | allocations: RefCell::new(Slab::new()), 62 | 63 | total_allocations_counter: Cell::new(0), 64 | total_deallocations_counter: Cell::new(0), 65 | } 66 | } 67 | 68 | pub fn props(&self) -> DeviceProperties<'_> { 69 | DeviceProperties { 70 | memory_types: Cow::Borrowed(&self.memory_types), 71 | memory_heaps: Cow::Borrowed(&self.memory_heaps), 72 | max_memory_allocation_count: self.max_memory_allocation_count, 73 | max_memory_allocation_size: self.max_memory_allocation_size, 74 | non_coherent_atom_size: self.non_coherent_atom_size, 75 | buffer_device_address: self.buffer_device_address, 76 | } 77 | } 78 | 79 | pub fn total_allocations(&self) -> u64 { 80 | self.total_allocations_counter.get() 81 | } 82 | 83 | pub fn total_deallocations(&self) -> u64 { 84 | self.total_deallocations_counter.get() 85 | } 86 | } 87 | 88 | impl MemoryDevice for MockMemoryDevice { 89 | #[tracing::instrument(skip(self))] 90 | unsafe fn allocate_memory( 91 | &self, 92 | size: u64, 93 | memory_type: u32, 94 | flags: AllocationFlags, 95 | ) -> Result { 96 | assert!(self.buffer_device_address || !flags.contains(AllocationFlags::DEVICE_ADDRESS), 97 | "`AllocationFlags::DEVICE_ADDRESS` cannot be specified unless DeviceProperties contain `DeviceProperties::device_address is true`"); 98 | 99 | assert!( 100 | size <= self.max_memory_allocation_size, 101 | "Allocation size exceeds limit" 102 | ); 103 | 104 | let allocations_remains = self.allocations_remains.get(); 105 | assert!( 106 | allocations_remains > 0, 107 | "Allocator should not try to allocate too many objects" 108 | ); 109 | self.allocations_remains.set(allocations_remains - 1); 110 | 111 | let heap = &self.memory_heaps_remaining_capacity 112 | [self.memory_types[memory_type as usize].heap as usize]; 113 | if heap.get() < size { 114 | return Err(OutOfMemory::OutOfDeviceMemory); 115 | } 116 | heap.set(heap.get() - size); 117 | 118 | tracing::info!("Memory object allocated"); 119 | 120 | self.total_allocations_counter 121 | .set(self.total_allocations_counter.get() + 1); 122 | 123 | Ok(self.allocations.borrow_mut().insert(MockMemory { 124 | memory_type, 125 | size, 126 | mapped: None, 127 | })) 128 | } 129 | 130 | #[tracing::instrument(skip(self))] 131 | unsafe fn deallocate_memory(&self, memory: usize) { 132 | let memory = self.allocations.borrow_mut().remove(memory); 133 | self.allocations_remains 134 | .set(self.allocations_remains.get() + 1); 135 | let heap = &self.memory_heaps_remaining_capacity 136 | [self.memory_types[memory.memory_type as usize].heap as usize]; 137 | heap.set(heap.get() + memory.size); 138 | tracing::info!("Memory object deallocated"); 139 | 140 | self.total_deallocations_counter 141 | .set(self.total_deallocations_counter.get() + 1); 142 | } 143 | 144 | #[tracing::instrument(skip(self))] 145 | unsafe fn map_memory( 146 | &self, 147 | memory: &mut usize, 148 | offset: u64, 149 | size: u64, 150 | ) -> Result, DeviceMapError> { 151 | assert_ne!(size, 0, "Mapping size must be larger than 0"); 152 | 153 | let mut allocations = self.allocations.borrow_mut(); 154 | let memory = allocations 155 | .get_mut(*memory) 156 | .expect("Non-existing memory object"); 157 | 158 | assert!( 159 | self.memory_types[memory.memory_type as usize] 160 | .props 161 | .contains(MemoryPropertyFlags::HOST_VISIBLE), 162 | "Attempt to map non-host-visible memory" 163 | ); 164 | 165 | assert!(memory.mapped.is_none(), "Already mapped"); 166 | 167 | assert!( 168 | offset < memory.size, 169 | "offset must be less than the size of memory" 170 | ); 171 | assert_ne!(size, 0, "Mapping size must be greater than 0"); 172 | assert!( 173 | size <= memory.size - offset, 174 | "size must be less than or equal to the size of the memory minus offset" 175 | ); 176 | 177 | let size_usize = usize::try_from(size).map_err(|_| DeviceMapError::OutOfHostMemory)?; 178 | let mapping = memory.mapped.get_or_insert(MemoryMapping { 179 | content: transmute(vec![0; size_usize].into_boxed_slice()), 180 | offset, 181 | }); 182 | 183 | tracing::info!("Memory object mapped"); 184 | Ok(NonNull::from(&mut (&mut *mapping.content.get())[0])) 185 | } 186 | 187 | unsafe fn unmap_memory(&self, memory: &mut usize) { 188 | let mut allocations = self.allocations.borrow_mut(); 189 | let memory = allocations 190 | .get_mut(*memory) 191 | .expect("Non-existing memory object"); 192 | assert!(memory.mapped.take().is_some(), "Was not mapped"); 193 | } 194 | 195 | unsafe fn invalidate_memory_ranges( 196 | &self, 197 | ranges: &[MappedMemoryRange<'_, usize>], 198 | ) -> Result<(), OutOfMemory> { 199 | for range in ranges { 200 | let mut allocations = self.allocations.borrow_mut(); 201 | let memory = allocations 202 | .get_mut(*range.memory) 203 | .expect("Non-existing memory object"); 204 | 205 | let mapped = memory.mapped.as_ref().expect("Not mapped"); 206 | 207 | let coherent = self.memory_types[memory.memory_type as usize] 208 | .props 209 | .contains(MemoryPropertyFlags::HOST_COHERENT); 210 | 211 | if coherent { 212 | tracing::warn!("Invalidating host-coherent memory"); 213 | } 214 | 215 | let mapped_size = (*mapped.content.get()).len() as u64; 216 | 217 | assert!( 218 | range.offset >= mapped.offset, 219 | "range `offset` specifies range before mapped region" 220 | ); 221 | assert!( 222 | range.offset - mapped.offset <= mapped_size, 223 | "range `offset` specifies range after mapped region" 224 | ); 225 | assert!( 226 | range.size < mapped_size - (range.offset - mapped.offset), 227 | "range `size` specifies range after mapped region" 228 | ); 229 | assert_eq!( 230 | range.offset % self.non_coherent_atom_size, 231 | 0, 232 | "`offset` must be a multiple of `non_coherent_atom_size`" 233 | ); 234 | assert!( 235 | range.size % self.non_coherent_atom_size == 0 236 | || range.offset + range.size == memory.size, 237 | "`size` must either be a multiple of `non_coherent_atom_size`, or `offset + size` must equal the size of memory" 238 | ); 239 | } 240 | 241 | Ok(()) 242 | } 243 | 244 | unsafe fn flush_memory_ranges( 245 | &self, 246 | ranges: &[MappedMemoryRange<'_, usize>], 247 | ) -> Result<(), OutOfMemory> { 248 | for range in ranges { 249 | let mut allocations = self.allocations.borrow_mut(); 250 | let memory = allocations 251 | .get_mut(*range.memory) 252 | .expect("Non-existing memory object"); 253 | 254 | let mapped = memory.mapped.as_ref().expect("Not mapped"); 255 | 256 | let coherent = self.memory_types[memory.memory_type as usize] 257 | .props 258 | .contains(MemoryPropertyFlags::HOST_COHERENT); 259 | 260 | if coherent { 261 | tracing::warn!("Invalidating host-coherent memory"); 262 | } 263 | 264 | let mapped_size = (*mapped.content.get()).len() as u64; 265 | 266 | assert!( 267 | range.offset >= mapped.offset, 268 | "`offset` specifies range before mapped region" 269 | ); 270 | assert!( 271 | range.offset - mapped.offset <= mapped_size, 272 | "`offset` specifies range after mapped region" 273 | ); 274 | assert!( 275 | range.size < mapped_size - (range.offset - mapped.offset), 276 | "`size` specifies range after mapped region" 277 | ); 278 | assert_eq!( 279 | range.offset % self.non_coherent_atom_size, 280 | 0, 281 | "`offset` must be a multiple of `non_coherent_atom_size`" 282 | ); 283 | assert!( 284 | range.size % self.non_coherent_atom_size == 0 285 | || range.offset + range.size == memory.size, 286 | "`size` must either be a multiple of `non_coherent_atom_size`, or `offset + size` must equal the size of memory" 287 | ); 288 | } 289 | Ok(()) 290 | } 291 | } 292 | 293 | // MockMemoryDevice is not a wrapper for external type in other crate, 294 | // this impl is needed to be compatible with the new signature of GpuAllocator. 295 | impl AsRef for MockMemoryDevice { 296 | #[inline(always)] 297 | fn as_ref(&self) -> &MockMemoryDevice { 298 | self 299 | } 300 | } 301 | -------------------------------------------------------------------------------- /types/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "gpu-alloc-types" 3 | version = "0.3.0" 4 | authors = ["Zakarum "] 5 | edition = "2018" 6 | description = "Core types of gpu-alloc crate" 7 | documentation = "https://docs.rs/gpu-alloc-types" 8 | readme = "../README.md" 9 | homepage = "https://github.com/zakarumych/gpu-alloc" 10 | repository = "https://github.com/zakarumych/gpu-alloc" 11 | license = "MIT OR Apache-2.0" 12 | keywords = ["gpu", "vulkan", "allocation", "no-std"] 13 | categories = ["graphics", "memory-management", "no-std", "game-development"] 14 | 15 | [dependencies] 16 | bitflags = { version = "2.0", default-features = false } 17 | -------------------------------------------------------------------------------- /types/src/device.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::types::{MemoryHeap, MemoryType}, 3 | alloc::borrow::Cow, 4 | core::ptr::NonNull, 5 | }; 6 | 7 | /// Memory exhausted error. 8 | #[derive(Debug)] 9 | pub enum OutOfMemory { 10 | /// Device memory exhausted. 11 | OutOfDeviceMemory, 12 | 13 | /// Host memory exhausted. 14 | OutOfHostMemory, 15 | } 16 | 17 | /// Memory mapped error. 18 | #[derive(Debug)] 19 | pub enum DeviceMapError { 20 | /// Device memory exhausted. 21 | OutOfDeviceMemory, 22 | 23 | /// Host memory exhausted. 24 | OutOfHostMemory, 25 | 26 | /// Map failed due to implementation specific error. 27 | MapFailed, 28 | } 29 | 30 | /// Specifies range of the mapped memory region. 31 | #[derive(Debug)] 32 | pub struct MappedMemoryRange<'a, M> { 33 | /// Memory object reference. 34 | pub memory: &'a M, 35 | 36 | /// Offset in bytes from start of the memory object. 37 | pub offset: u64, 38 | 39 | /// Size in bytes of the memory range. 40 | pub size: u64, 41 | } 42 | 43 | /// Properties of the device that will be used for allocating memory objects. 44 | /// 45 | /// See `gpu-alloc-` crate to learn how to obtain one for backend of choice. 46 | #[derive(Debug)] 47 | pub struct DeviceProperties<'a> { 48 | /// Array of memory types provided by the device. 49 | pub memory_types: Cow<'a, [MemoryType]>, 50 | 51 | /// Array of memory heaps provided by the device. 52 | pub memory_heaps: Cow<'a, [MemoryHeap]>, 53 | 54 | /// Maximum number of valid memory allocations that can exist simultaneously within the device. 55 | pub max_memory_allocation_count: u32, 56 | 57 | /// Maximum size for single allocation supported by the device. 58 | pub max_memory_allocation_size: u64, 59 | 60 | /// Atom size for host mappable non-coherent memory. 61 | pub non_coherent_atom_size: u64, 62 | 63 | /// Specifies if feature required to fetch device address is enabled. 64 | pub buffer_device_address: bool, 65 | } 66 | 67 | bitflags::bitflags! { 68 | /// Allocation flags 69 | #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] 70 | #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] 71 | pub struct AllocationFlags : u8 { 72 | /// Specifies that the memory can be used for buffers created 73 | /// with flag that allows fetching device address. 74 | const DEVICE_ADDRESS = 0x1; 75 | } 76 | } 77 | 78 | /// Abstract device that can be used to allocate memory objects. 79 | pub trait MemoryDevice { 80 | /// Allocates new memory object from device. 81 | /// This function may be expensive and even limit maximum number of memory 82 | /// objects allocated. 83 | /// Which is the reason for sub-allocation this crate provides. 84 | /// 85 | /// # Safety 86 | /// 87 | /// `memory_type` must be valid index for memory type associated with this device. 88 | /// Retrieving this information is implementation specific. 89 | /// 90 | /// `flags` must be supported by the device. 91 | unsafe fn allocate_memory( 92 | &self, 93 | size: u64, 94 | memory_type: u32, 95 | flags: AllocationFlags, 96 | ) -> Result; 97 | 98 | /// Deallocate memory object. 99 | /// 100 | /// # Safety 101 | /// 102 | /// Memory object must have been allocated from this device.\ 103 | /// All clones of specified memory handle must be dropped before calling this function. 104 | unsafe fn deallocate_memory(&self, memory: M); 105 | 106 | /// Map region of device memory to host memory space. 107 | /// 108 | /// # Safety 109 | /// 110 | /// * Memory object must have been allocated from this device. 111 | /// * Memory object must not be already mapped. 112 | /// * Memory must be allocated from type with `HOST_VISIBLE` property. 113 | /// * `offset + size` must not overflow. 114 | /// * `offset + size` must not be larger than memory object size specified when 115 | /// memory object was allocated from this device. 116 | unsafe fn map_memory( 117 | &self, 118 | memory: &mut M, 119 | offset: u64, 120 | size: u64, 121 | ) -> Result, DeviceMapError>; 122 | 123 | /// Unmap previously mapped memory region. 124 | /// 125 | /// # Safety 126 | /// 127 | /// * Memory object must have been allocated from this device. 128 | /// * Memory object must be mapped 129 | unsafe fn unmap_memory(&self, memory: &mut M); 130 | 131 | /// Invalidates ranges of memory mapped regions. 132 | /// 133 | /// # Safety 134 | /// 135 | /// * Memory objects must have been allocated from this device. 136 | /// * `offset` and `size` in each element of `ranges` must specify 137 | /// subregion of currently mapped memory region 138 | /// * if `memory` in some element of `ranges` does not contain `HOST_COHERENT` property 139 | /// then `offset` and `size` of that element must be multiple of `non_coherent_atom_size`. 140 | unsafe fn invalidate_memory_ranges( 141 | &self, 142 | ranges: &[MappedMemoryRange<'_, M>], 143 | ) -> Result<(), OutOfMemory>; 144 | 145 | /// Flushes ranges of memory mapped regions. 146 | /// 147 | /// # Safety 148 | /// 149 | /// * Memory objects must have been allocated from this device. 150 | /// * `offset` and `size` in each element of `ranges` must specify 151 | /// subregion of currently mapped memory region 152 | /// * if `memory` in some element of `ranges` does not contain `HOST_COHERENT` property 153 | /// then `offset` and `size` of that element must be multiple of `non_coherent_atom_size`. 154 | unsafe fn flush_memory_ranges( 155 | &self, 156 | ranges: &[MappedMemoryRange<'_, M>], 157 | ) -> Result<(), OutOfMemory>; 158 | } 159 | -------------------------------------------------------------------------------- /types/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![no_std] 2 | 3 | extern crate alloc; 4 | 5 | mod device; 6 | mod types; 7 | 8 | pub use self::{device::*, types::*}; 9 | -------------------------------------------------------------------------------- /types/src/types.rs: -------------------------------------------------------------------------------- 1 | bitflags::bitflags! { 2 | /// Memory properties type. 3 | #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] 4 | #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] 5 | pub struct MemoryPropertyFlags: u8 { 6 | /// This flag is set for device-local memory types. 7 | /// Device-local memory is situated "close" to the GPU cores 8 | /// and allows for fast access. 9 | const DEVICE_LOCAL = 0x01; 10 | 11 | /// This flag is set for host-visible memory types. 12 | /// Host-visible memory can be mapped to the host memory range. 13 | const HOST_VISIBLE = 0x02; 14 | 15 | /// This flag is set for host-coherent memory types. 16 | /// Host-coherent memory does not requires manual invalidation for 17 | /// modifications on GPU to become visible on host; 18 | /// nor flush for modification on host to become visible on GPU. 19 | /// Access synchronization is still required. 20 | const HOST_COHERENT = 0x04; 21 | 22 | /// This flag is set for host-cached memory types. 23 | /// Host-cached memory uses cache in host memory for faster reads from host. 24 | const HOST_CACHED = 0x08; 25 | 26 | /// This flag is set for lazily-allocated memory types. 27 | /// Lazily-allocated memory must be used (and only) for transient image attachments. 28 | const LAZILY_ALLOCATED = 0x10; 29 | 30 | /// This flag is set for protected memory types. 31 | /// Protected memory can be used for writing by protected operations 32 | /// and can be read only by protected operations. 33 | /// Protected memory cannot be host-visible. 34 | /// Implementation must guarantee that there is no way for data to flow 35 | /// from protected to unprotected memory. 36 | const PROTECTED = 0x20; 37 | } 38 | } 39 | 40 | /// Defines memory type. 41 | #[derive(Clone, Copy, Debug)] 42 | pub struct MemoryType { 43 | /// Heap index of the memory type. 44 | pub heap: u32, 45 | 46 | /// Property flags of the memory type. 47 | pub props: MemoryPropertyFlags, 48 | } 49 | 50 | /// Defines memory heap. 51 | #[derive(Clone, Copy, Debug)] 52 | pub struct MemoryHeap { 53 | /// Size of memory heap in bytes. 54 | pub size: u64, 55 | } 56 | --------------------------------------------------------------------------------