├── CONTRIBUTORS.txt ├── COPYING ├── Makefile ├── README.md ├── dm-vdo.rst ├── kvdo.spec └── vdo ├── Makefile ├── action-manager.c ├── action-manager.h ├── admin-completion.c ├── admin-completion.h ├── admin-state.c ├── admin-state.h ├── allocation-selector.c ├── allocation-selector.h ├── atomic-stats.h ├── bio.c ├── bio.h ├── block-allocator.c ├── block-allocator.h ├── block-map-entry.h ├── block-map-format.c ├── block-map-format.h ├── block-map-page.c ├── block-map-page.h ├── block-map-recovery.c ├── block-map-recovery.h ├── block-map-tree.c ├── block-map-tree.h ├── block-map.c ├── block-map.h ├── block-mapping-state.h ├── buffer.c ├── buffer.h ├── buffered-reader.c ├── buffered-reader.h ├── buffered-writer.c ├── buffered-writer.h ├── chapter-index.c ├── chapter-index.h ├── common.h ├── compiler.h ├── completion.c ├── completion.h ├── compressed-block.c ├── compressed-block.h ├── compression-state.c ├── compression-state.h ├── config.c ├── config.h ├── constants.c ├── constants.h ├── cpu.h ├── data-vio-pool.c ├── data-vio-pool.h ├── data-vio.c ├── data-vio.h ├── dedupe.c ├── dedupe.h ├── delta-index.c ├── delta-index.h ├── device-config.c ├── device-config.h ├── device-registry.c ├── device-registry.h ├── dirty-lists.c ├── dirty-lists.h ├── dm-vdo-target.c ├── dump.c ├── dump.h ├── errors.c ├── errors.h ├── event-count.c ├── event-count.h ├── flush.c ├── flush.h ├── forest.c ├── forest.h ├── funnel-queue.c ├── funnel-queue.h ├── geometry.c ├── geometry.h ├── hash-utils.c ├── hash-utils.h ├── header.c ├── header.h ├── heap.c ├── heap.h ├── index-layout.c ├── index-layout.h ├── index-page-map.c ├── index-page-map.h ├── index-session.c ├── index-session.h ├── index.c ├── index.h ├── instance-number.c ├── instance-number.h ├── int-map.c ├── int-map.h ├── io-factory.c ├── io-factory.h ├── io-submitter.c ├── io-submitter.h ├── journal-point.h ├── kernel-types.h ├── linux └── murmurhash3.h ├── lock-counter.c ├── lock-counter.h ├── logger.c ├── logger.h ├── logical-zone.c ├── logical-zone.h ├── memory-alloc.c ├── memory-alloc.h ├── messageStats.c ├── messageStats.h ├── murmurhash3.c ├── num-utils.h ├── numeric.h ├── open-chapter.c ├── open-chapter.h ├── packed-recovery-journal-block.h ├── packed-reference-block.h ├── packer.c ├── packer.h ├── page-cache.c ├── page-cache.h ├── pbn-lock-pool.c ├── pbn-lock-pool.h ├── pbn-lock.c ├── pbn-lock.h ├── permassert.c ├── permassert.h ├── physical-zone.c ├── physical-zone.h ├── pointer-map.c ├── pointer-map.h ├── pool-sysfs-stats.c ├── pool-sysfs.c ├── pool-sysfs.h ├── priority-table.c ├── priority-table.h ├── radix-sort.c ├── radix-sort.h ├── random.c ├── random.h ├── read-only-notifier.c ├── read-only-notifier.h ├── read-only-rebuild.c ├── read-only-rebuild.h ├── record-page.c ├── record-page.h ├── recovery-journal-block.c ├── recovery-journal-block.h ├── recovery-journal-entry.h ├── recovery-journal-format.c ├── recovery-journal-format.h ├── recovery-journal.c ├── recovery-journal.h ├── recovery-utils.c ├── recovery-utils.h ├── ref-counts.c ├── ref-counts.h ├── reference-count-rebuild.c ├── reference-count-rebuild.h ├── reference-operation.c ├── reference-operation.h ├── release-versions.h ├── request-queue.c ├── request-queue.h ├── slab-depot-format.c ├── slab-depot-format.h ├── slab-depot.c ├── slab-depot.h ├── slab-iterator.h ├── slab-journal-format.c ├── slab-journal-format.h ├── slab-journal.c ├── slab-journal.h ├── slab-scrubber.c ├── slab-scrubber.h ├── slab-summary-format.h ├── slab-summary.c ├── slab-summary.h ├── slab.c ├── slab.h ├── sparse-cache.c ├── sparse-cache.h ├── statistics.h ├── status-codes.c ├── status-codes.h ├── string-utils.c ├── string-utils.h ├── super-block-codec.c ├── super-block-codec.h ├── super-block.c ├── super-block.h ├── sync-completion.c ├── sync-completion.h ├── sysfs.c ├── thread-cond-var.c ├── thread-config.c ├── thread-config.h ├── thread-device.c ├── thread-device.h ├── thread-registry.c ├── thread-registry.h ├── time-utils.c ├── time-utils.h ├── type-defs.h ├── types.h ├── uds-sysfs.c ├── uds-sysfs.h ├── uds-threads.c ├── uds-threads.h ├── uds.h ├── vdo-component-states.c ├── vdo-component-states.h ├── vdo-component.c ├── vdo-component.h ├── vdo-layout.c ├── vdo-layout.h ├── vdo-load.c ├── vdo-load.h ├── vdo-page-cache.c ├── vdo-page-cache.h ├── vdo-recovery.c ├── vdo-recovery.h ├── vdo-resize-logical.c ├── vdo-resize-logical.h ├── vdo-resize.c ├── vdo-resize.h ├── vdo-resume.c ├── vdo-resume.h ├── vdo-suspend.c ├── vdo-suspend.h ├── vdo.c ├── vdo.h ├── vio-pool.c ├── vio-pool.h ├── vio-read.c ├── vio-read.h ├── vio-write.c ├── vio-write.h ├── vio.c ├── vio.h ├── volume-geometry.c ├── volume-geometry.h ├── volume-index-ops.c ├── volume-index-ops.h ├── volume-index005.c ├── volume-index005.h ├── volume-index006.c ├── volume-index006.h ├── volume-store.c ├── volume-store.h ├── volume.c ├── volume.h ├── wait-queue.c ├── wait-queue.h ├── workQueue.c └── workQueue.h /CONTRIBUTORS.txt: -------------------------------------------------------------------------------- 1 | The Red Hat VDO Team: 2 | Principal Engineer/Lead Architect: 3 | J. corwin Coburn 4 | 5 | Primary Authors: 6 | Joseph Chapman 7 | Sweet Tea Dorminy 8 | *Thomas Jaskiewicz 9 | Bruce Johnston 10 | Susan McGhee 11 | Ken Raeburn 12 | Michael Sclafani 13 | Matthew Sakai 14 | Joseph Shimkus 15 | John Wiele 16 | 17 | Support, Testing, Documentation, and other things too numerous to mention: 18 | Chung Chung : 19 | Bryan Gurney 20 | *Simon J. Hernandez 21 | Jakub Krysl 22 | Marek Suchanek 23 | 24 | Project Management & Technical Direction: 25 | Jered Floyd 26 | Louis Imershein 27 | Dennis Keefe 28 | Andrew Walsh 29 | 30 | *former team members 31 | 32 | Other Contributors: 33 | Ji-Hyeon Gim : 34 | Updates for FC26/Kernel 4.13 35 | Vojtech Trefny 36 | Getting correct size of partitions 37 | Achilles Gaikwad 38 | Bash completion for the vdo and vdostats commands 39 | Jin-young Kwon 40 | Adding vdo --version command, and documentation fixes 41 | Francisco Vilmar Cardoso Ruviaro 42 | Typo corrections in vdo and uds 43 | 44 | VDO was originally created at Permabit Technology Corporation, and was 45 | subsequently acquired and open-sourced by Red Hat. 46 | 47 | Former Members of the Permabit VDO Team: 48 | Engineers: 49 | Mark Amidon 50 | David Buckle 51 | Jacky Chu 52 | Joel Hoff 53 | Dimitri Kountourogianni 54 | Alexis Layton 55 | Michael Lee 56 | Rich Macchi 57 | Dave Paniriti 58 | Karl Ramm 59 | Hooman Vassef 60 | Assar Westurlund 61 | 62 | Support, Testing, Documentation, etc. 63 | Carl Alexander 64 | Mike Chu 65 | Mark Iskra 66 | Farid Jahanmir 67 | Francesca Koulikov 68 | Erik Lattimore 69 | Jennifer Levine 70 | Randy Long 71 | Steve Looby 72 | Uche Onyekwuluje 73 | Catherine Powell 74 | Jeff Pozz 75 | Sarmad Sada 76 | John Schmidt 77 | Omri Schwarz 78 | Jay Splaine 79 | John Welle 80 | Mary-Anne Wolf 81 | Devon Yablonski 82 | Robert Zupko 83 | 84 | Interns: 85 | Ari Entlich 86 | Lori Monteleone 87 | 88 | Project Management & Technical Direction: 89 | Michael Fortson 90 | 91 | Other Past Permabit Contributors (for early work on the index): 92 | James Clough 93 | Dave Golombek 94 | Albert Lin 95 | Edwin Olson 96 | Dave Pinkney 97 | Rich Brennan 98 | 99 | And Very Special Thanks To: 100 | Norman Margolis, who started the whole thing 101 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | obj-y += vdo/ 2 | -------------------------------------------------------------------------------- /kvdo.spec: -------------------------------------------------------------------------------- 1 | %define spec_release 1 2 | %define kmod_name kvdo 3 | %define kmod_driver_version 8.2.6.3 4 | %define kmod_rpm_release %{spec_release} 5 | %define kmod_kernel_version 3.10.0-693.el7 6 | 7 | # Disable the scanning for a debug package 8 | %global debug_package %{nil} 9 | 10 | Source0: %{kmod_name}-%{kmod_driver_version}.tgz 11 | 12 | Name: kmod-kvdo 13 | Version: %{kmod_driver_version} 14 | Release: %{kmod_rpm_release}%{?dist} 15 | Summary: Kernel Modules for Virtual Data Optimizer 16 | License: GPLv2+ 17 | URL: http://github.com/dm-vdo/kvdo 18 | BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) 19 | Requires: dkms 20 | Requires: kernel-devel >= %{kmod_kernel_version} 21 | Requires: make 22 | %if 0%{?fedora} 23 | # Fedora requires elfutils-libelf-devel, while rhel does not. 24 | BuildRequires: elfutils-libelf-devel 25 | %endif 26 | BuildRequires: glibc 27 | %if 0%{?rhel} && 0%{?rhel} < 9 28 | # Fedora doesn't have abi whitelists, 29 | # And RHEL9 doesn't have it yet. 30 | BuildRequires: kernel-abi-whitelists 31 | %endif 32 | BuildRequires: libuuid-devel 33 | BuildRequires: redhat-rpm-config 34 | ExclusiveArch: x86_64 35 | ExcludeArch: s390 36 | ExcludeArch: s390x 37 | ExcludeArch: ppc 38 | ExcludeArch: ppc64 39 | ExcludeArch: ppc64le 40 | ExcludeArch: aarch64 41 | ExcludeArch: i686 42 | 43 | %description 44 | Virtual Data Optimizer (VDO) is a device mapper target that delivers 45 | block-level deduplication, compression, and thin provisioning. 46 | 47 | This package provides the kernel modules for VDO. 48 | 49 | %post 50 | set -x 51 | /usr/sbin/dkms --rpm_safe_upgrade add -m %{kmod_name} -v %{version} 52 | /usr/sbin/dkms --rpm_safe_upgrade build -m %{kmod_name} -v %{version} 53 | /usr/sbin/dkms --rpm_safe_upgrade install -m %{kmod_name} -v %{version} 54 | 55 | %preun 56 | # Check whether kvdo is loaded, and if so attempt to remove it. A 57 | # failure here means there is still something using the module, which 58 | # should be cleared up before attempting to remove again. 59 | for module in kvdo uds; do 60 | if grep -q "^${module}" /proc/modules; then 61 | modprobe -r ${module} 62 | fi 63 | done 64 | /usr/sbin/dkms --rpm_safe_upgrade remove -m %{kmod_name} -v %{version} --all || : 65 | 66 | %prep 67 | %setup -n %{kmod_name}-%{kmod_driver_version} 68 | 69 | %build 70 | # Nothing doing here, as we're going to build on whatever kernel we end up 71 | # running inside. 72 | 73 | %install 74 | mkdir -p $RPM_BUILD_ROOT/%{_usr}/src/%{kmod_name}-%{version} 75 | cp -r * $RPM_BUILD_ROOT/%{_usr}/src/%{kmod_name}-%{version}/ 76 | cat > $RPM_BUILD_ROOT/%{_usr}/src/%{kmod_name}-%{version}/dkms.conf < - 8.2.6.3-1 96 | - See https://github.com/dm-vdo/kvdo.git 97 | -------------------------------------------------------------------------------- /vdo/Makefile: -------------------------------------------------------------------------------- 1 | VDO_VERSION = 8.2.6.3 2 | 3 | SOURCES = $(notdir $(wildcard $(src)/*.c)) 4 | OBJECTS = $(SOURCES:%.c=%.o) 5 | INCLUDES = -I$(src) 6 | 7 | RHEL_RELEASE_EXTRA ?= $(shell uname -r | \ 8 | sed -e 's/^[0-9]\+\.[0-9]\+\.[0-9]\+\-\([0-9]\+\).*/\1/') 9 | 10 | EXTRA_CFLAGS = -std=gnu11 \ 11 | -fno-builtin-memset \ 12 | -Werror \ 13 | $(if $(CONFIG_KASAN),,-Wframe-larger-than=400) \ 14 | -DCURRENT_VERSION=\"$(VDO_VERSION)\" \ 15 | -DRHEL_RELEASE_EXTRA=$(RHEL_RELEASE_EXTRA) \ 16 | $(INCLUDES) 17 | 18 | obj-m += kvdo.o 19 | 20 | kvdo-objs = $(OBJECTS) 21 | -------------------------------------------------------------------------------- /vdo/admin-completion.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef ADMIN_COMPLETION_H 7 | #define ADMIN_COMPLETION_H 8 | 9 | #include 10 | #include 11 | 12 | #include "uds-threads.h" 13 | 14 | #include "completion.h" 15 | #include "types.h" 16 | 17 | enum admin_operation_type { 18 | VDO_ADMIN_OPERATION_UNKNOWN, 19 | VDO_ADMIN_OPERATION_GROW_LOGICAL, 20 | VDO_ADMIN_OPERATION_GROW_PHYSICAL, 21 | VDO_ADMIN_OPERATION_PREPARE_GROW_PHYSICAL, 22 | VDO_ADMIN_OPERATION_LOAD, 23 | VDO_ADMIN_OPERATION_PRE_LOAD, 24 | VDO_ADMIN_OPERATION_RESUME, 25 | VDO_ADMIN_OPERATION_SUSPEND, 26 | }; 27 | 28 | struct admin_completion; 29 | 30 | typedef thread_id_t 31 | vdo_thread_id_getter_for_phase(struct admin_completion *admin_completion); 32 | 33 | struct admin_completion { 34 | /* 35 | * FIXME this field should be replaced by container_of() when 36 | * enqueuables go away and this becomes a field of struct vdo. 37 | */ 38 | struct vdo *vdo; 39 | struct vdo_completion completion; 40 | struct vdo_completion sub_task_completion; 41 | atomic_t busy; 42 | enum admin_operation_type type; 43 | vdo_thread_id_getter_for_phase *get_thread_id; 44 | uint32_t phase; 45 | struct completion callback_sync; 46 | }; 47 | 48 | void vdo_assert_admin_operation_type(struct admin_completion *completion, 49 | enum admin_operation_type expected); 50 | 51 | struct admin_completion * __must_check 52 | vdo_admin_completion_from_sub_task(struct vdo_completion *completion); 53 | 54 | void vdo_assert_admin_phase_thread(struct admin_completion *admin_completion, 55 | const char *what, 56 | const char *phase_names[]); 57 | 58 | struct vdo * __must_check 59 | vdo_from_admin_sub_task(struct vdo_completion *completion, 60 | enum admin_operation_type expected); 61 | 62 | void vdo_initialize_admin_completion(struct vdo *vdo, 63 | struct admin_completion *admin_completion); 64 | 65 | struct vdo_completion *vdo_reset_admin_sub_task(struct vdo_completion *completion); 66 | 67 | void vdo_prepare_admin_sub_task(struct vdo *vdo, 68 | vdo_action *callback, 69 | vdo_action *error_handler); 70 | 71 | int __must_check 72 | vdo_perform_admin_operation(struct vdo *vdo, 73 | enum admin_operation_type type, 74 | vdo_thread_id_getter_for_phase *thread_id_getter, 75 | vdo_action *action, 76 | vdo_action *error_handler); 77 | 78 | #endif /* ADMIN_COMPLETION_H */ 79 | -------------------------------------------------------------------------------- /vdo/allocation-selector.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #include "allocation-selector.h" 7 | 8 | #include "memory-alloc.h" 9 | 10 | #include "types.h" 11 | 12 | enum { 13 | ALLOCATIONS_PER_ZONE = 128, 14 | }; 15 | 16 | /** 17 | * vdo_make_allocation_selector() - Make a new allocation selector. 18 | * @physical_zone_count [in] The number of physical zones. 19 | * @thread_id [in] The ID of the thread using this selector. 20 | * @selector_ptr [out] A pointer to receive the new selector. 21 | * 22 | * Return: VDO_SUCCESS or an error. 23 | */ 24 | int vdo_make_allocation_selector(zone_count_t physical_zone_count, 25 | thread_id_t thread_id, 26 | struct allocation_selector **selector_ptr) 27 | { 28 | struct allocation_selector *selector; 29 | int result = UDS_ALLOCATE(1, 30 | struct allocation_selector, 31 | __func__, 32 | &selector); 33 | if (result != VDO_SUCCESS) { 34 | return result; 35 | } 36 | 37 | *selector = (struct allocation_selector) { 38 | .next_allocation_zone = thread_id % physical_zone_count, 39 | .last_physical_zone = physical_zone_count - 1, 40 | }; 41 | 42 | *selector_ptr = selector; 43 | return VDO_SUCCESS; 44 | } 45 | 46 | /** 47 | * vdo_get_next_allocation_zone() - Get number of the physical zone from 48 | * which to allocate next. 49 | * @selector: The selector to query. 50 | * 51 | * Return: The number of the physical zone from which to allocate. 52 | */ 53 | zone_count_t vdo_get_next_allocation_zone(struct allocation_selector *selector) 54 | { 55 | if (selector->last_physical_zone > 0) { 56 | if (selector->allocation_count < ALLOCATIONS_PER_ZONE) { 57 | selector->allocation_count++; 58 | } else { 59 | selector->allocation_count = 1; 60 | if (selector->next_allocation_zone < 61 | selector->last_physical_zone) { 62 | selector->next_allocation_zone++; 63 | } else { 64 | selector->next_allocation_zone = 0; 65 | } 66 | } 67 | } 68 | 69 | return selector->next_allocation_zone; 70 | } 71 | -------------------------------------------------------------------------------- /vdo/allocation-selector.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef ALLOCATION_SELECTOR_H 7 | #define ALLOCATION_SELECTOR_H 8 | 9 | #include "completion.h" 10 | 11 | /** 12 | * DOC: Allocation selectors 13 | * 14 | * An allocation_selector is used by any zone which does data block allocations. 15 | * The selector is used to round-robin allocation requests to different 16 | * physical zones. Currently, 128 allocations will be made to a given physical 17 | * zone before switching to the next. 18 | */ 19 | 20 | /** 21 | * struct allocation_selector: Structure used to select which physical zone to 22 | * allocate from. 23 | */ 24 | struct allocation_selector { 25 | /** 26 | * @allocation_count: The number of allocations done in the current 27 | * zone. 28 | */ 29 | block_count_t allocation_count; 30 | /** @next_allocation_zone: The physical zone to allocate from next. */ 31 | zone_count_t next_allocation_zone; 32 | /** @last_physical_cone: The number of the last physical zone. */ 33 | zone_count_t last_physical_zone; 34 | }; 35 | 36 | int __must_check 37 | vdo_make_allocation_selector(zone_count_t physical_zone_count, 38 | thread_id_t thread_id, 39 | struct allocation_selector **selector_ptr); 40 | 41 | zone_count_t __must_check 42 | vdo_get_next_allocation_zone(struct allocation_selector *selector); 43 | 44 | #endif /* ALLOCATION_SELECTOR_H */ 45 | -------------------------------------------------------------------------------- /vdo/atomic-stats.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef ATOMIC_STATS_H 7 | #define ATOMIC_STATS_H 8 | 9 | #include 10 | 11 | #include "statistics.h" 12 | 13 | /* Keep struct bio statistics atomically */ 14 | struct atomic_bio_stats { 15 | atomic64_t read; /* Number of not REQ_WRITE bios */ 16 | atomic64_t write; /* Number of REQ_WRITE bios */ 17 | atomic64_t discard; /* Number of REQ_DISCARD bios */ 18 | atomic64_t flush; /* Number of REQ_FLUSH bios */ 19 | atomic64_t empty_flush; /* Number of REQ_PREFLUSH bios without data */ 20 | atomic64_t fua; /* Number of REQ_FUA bios */ 21 | }; 22 | 23 | /* 24 | * Counters are atomic since updates can arrive concurrently from arbitrary 25 | * threads. 26 | */ 27 | struct atomic_statistics { 28 | atomic64_t bios_submitted; 29 | atomic64_t bios_completed; 30 | atomic64_t flush_out; 31 | atomic64_t invalid_advice_pbn_count; 32 | atomic64_t no_space_error_count; 33 | atomic64_t read_only_error_count; 34 | struct atomic_bio_stats bios_in; 35 | struct atomic_bio_stats bios_in_partial; 36 | struct atomic_bio_stats bios_out; 37 | struct atomic_bio_stats bios_out_completed; 38 | struct atomic_bio_stats bios_acknowledged; 39 | struct atomic_bio_stats bios_acknowledged_partial; 40 | struct atomic_bio_stats bios_meta; 41 | struct atomic_bio_stats bios_meta_completed; 42 | struct atomic_bio_stats bios_journal; 43 | struct atomic_bio_stats bios_journal_completed; 44 | struct atomic_bio_stats bios_page_cache; 45 | struct atomic_bio_stats bios_page_cache_completed; 46 | }; 47 | 48 | #endif /* ATOMIC_STATS_H */ 49 | -------------------------------------------------------------------------------- /vdo/bio.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef BIO_H 7 | #define BIO_H 8 | 9 | #include 10 | #include 11 | 12 | #include "kernel-types.h" 13 | 14 | void vdo_bio_copy_data_in(struct bio *bio, char *data_ptr); 15 | void vdo_bio_copy_data_out(struct bio *bio, char *data_ptr); 16 | 17 | static inline int vdo_get_bio_result(struct bio *bio) 18 | { 19 | return blk_status_to_errno(bio->bi_status); 20 | } 21 | 22 | static inline void vdo_complete_bio(struct bio *bio, int error) 23 | { 24 | bio->bi_status = errno_to_blk_status(error); 25 | bio_endio(bio); 26 | } 27 | 28 | int vdo_create_multi_block_bio(block_count_t size, struct bio **bio_ptr); 29 | 30 | static inline int vdo_create_bio(struct bio **bio_ptr) 31 | { 32 | return vdo_create_multi_block_bio(1, bio_ptr); 33 | } 34 | 35 | void vdo_free_bio(struct bio *bio); 36 | 37 | void vdo_count_bios(struct atomic_bio_stats *bio_stats, struct bio *bio); 38 | void vdo_count_completed_bios(struct bio *bio); 39 | 40 | void vdo_complete_async_bio(struct bio *bio); 41 | 42 | void vdo_set_bio_properties(struct bio *bio, 43 | struct vio *vio, 44 | bio_end_io_t callback, 45 | unsigned int bi_opf, 46 | physical_block_number_t pbn); 47 | 48 | int vdo_reset_bio_with_buffer(struct bio *bio, 49 | char *data, 50 | struct vio *vio, 51 | bio_end_io_t callback, 52 | unsigned int bi_opf, 53 | physical_block_number_t pbn); 54 | 55 | #endif /* BIO_H */ 56 | -------------------------------------------------------------------------------- /vdo/block-map-entry.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef BLOCK_MAP_ENTRY_H 7 | #define BLOCK_MAP_ENTRY_H 8 | 9 | #include "block-mapping-state.h" 10 | #include "constants.h" 11 | #include "numeric.h" 12 | #include "types.h" 13 | 14 | /** 15 | * DOC: Block map entries 16 | * 17 | * The entry for each logical block in the block map is encoded into five 18 | * bytes, which saves space in both the on-disk and in-memory layouts. It 19 | * consists of the 36 low-order bits of a physical_block_number_t 20 | * (addressing 256 terabytes with a 4KB block size) and a 4-bit encoding of a 21 | * block_mapping_state. 22 | * 23 | * Of the 8 high bits of the 5-byte structure: 24 | * 25 | * Bits 7..4: The four highest bits of the 36-bit physical block 26 | * number 27 | * Bits 3..0: The 4-bit block_mapping_state 28 | * 29 | * The following 4 bytes are the low order bytes of the physical block number, 30 | * in little-endian order. 31 | * 32 | * Conversion functions to and from a data location are provided. 33 | */ 34 | struct block_map_entry { 35 | #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 36 | unsigned mapping_state : 4; 37 | unsigned pbn_high_nibble : 4; 38 | #else 39 | unsigned pbn_high_nibble : 4; 40 | unsigned mapping_state : 4; 41 | #endif 42 | 43 | __le32 pbn_low_word; 44 | } __packed; 45 | 46 | static inline struct data_location 47 | vdo_unpack_block_map_entry(const struct block_map_entry *entry) 48 | { 49 | physical_block_number_t low32 = __le32_to_cpu(entry->pbn_low_word); 50 | physical_block_number_t high4 = entry->pbn_high_nibble; 51 | 52 | return (struct data_location) { 53 | .pbn = ((high4 << 32) | low32), 54 | .state = entry->mapping_state, 55 | }; 56 | } 57 | 58 | static inline bool vdo_is_mapped_location(const struct data_location *location) 59 | { 60 | return (location->state != VDO_MAPPING_STATE_UNMAPPED); 61 | } 62 | 63 | static inline bool vdo_is_valid_location(const struct data_location *location) 64 | { 65 | if (location->pbn == VDO_ZERO_BLOCK) { 66 | return !vdo_is_state_compressed(location->state); 67 | } else { 68 | return vdo_is_mapped_location(location); 69 | } 70 | } 71 | 72 | /* FIXME: maybe this should be vdo_pack_block_map_entry() */ 73 | static inline struct block_map_entry 74 | vdo_pack_pbn(physical_block_number_t pbn, enum block_mapping_state mapping_state) 75 | { 76 | return (struct block_map_entry) { 77 | .mapping_state = (mapping_state & 0x0F), 78 | .pbn_high_nibble = ((pbn >> 32) & 0x0F), 79 | .pbn_low_word = __cpu_to_le32(pbn & UINT_MAX), 80 | }; 81 | } 82 | 83 | #endif /* BLOCK_MAP_ENTRY_H */ 84 | -------------------------------------------------------------------------------- /vdo/block-map-format.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef BLOCK_MAP_FORMAT_H 7 | #define BLOCK_MAP_FORMAT_H 8 | 9 | #include "buffer.h" 10 | 11 | #include "constants.h" 12 | #include "header.h" 13 | #include "types.h" 14 | 15 | struct block_map_state_2_0 { 16 | physical_block_number_t flat_page_origin; 17 | block_count_t flat_page_count; 18 | physical_block_number_t root_origin; 19 | block_count_t root_count; 20 | } __packed; 21 | 22 | struct boundary { 23 | page_number_t levels[VDO_BLOCK_MAP_TREE_HEIGHT]; 24 | }; 25 | 26 | extern const struct header VDO_BLOCK_MAP_HEADER_2_0; 27 | 28 | int __must_check 29 | vdo_decode_block_map_state_2_0(struct buffer *buffer, 30 | struct block_map_state_2_0 *state); 31 | 32 | size_t __must_check vdo_get_block_map_encoded_size(void); 33 | 34 | int __must_check 35 | vdo_encode_block_map_state_2_0(struct block_map_state_2_0 state, 36 | struct buffer *buffer); 37 | 38 | page_count_t vdo_compute_block_map_page_count(block_count_t entries); 39 | 40 | block_count_t __must_check 41 | vdo_compute_new_forest_pages(root_count_t root_count, 42 | struct boundary *old_sizes, 43 | block_count_t entries, 44 | struct boundary *new_sizes); 45 | 46 | #endif /* BLOCK_MAP_FORMAT_H */ 47 | -------------------------------------------------------------------------------- /vdo/block-map-page.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #include "block-map-page.h" 7 | 8 | #include "permassert.h" 9 | 10 | #include "constants.h" 11 | #include "status-codes.h" 12 | #include "types.h" 13 | 14 | enum { 15 | PAGE_HEADER_4_1_SIZE = 8 + 8 + 8 + 1 + 1 + 1 + 1, 16 | }; 17 | 18 | static const struct version_number BLOCK_MAP_4_1 = { 19 | .major_version = 4, 20 | .minor_version = 1, 21 | }; 22 | 23 | struct block_map_page *vdo_format_block_map_page(void *buffer, 24 | nonce_t nonce, 25 | physical_block_number_t pbn, 26 | bool initialized) 27 | { 28 | struct block_map_page *page = (struct block_map_page *) buffer; 29 | 30 | memset(buffer, 0, VDO_BLOCK_SIZE); 31 | page->version = vdo_pack_version_number(BLOCK_MAP_4_1); 32 | page->header.nonce = __cpu_to_le64(nonce); 33 | page->header.pbn = __cpu_to_le64(pbn); 34 | page->header.initialized = initialized; 35 | return page; 36 | } 37 | 38 | enum block_map_page_validity 39 | vdo_validate_block_map_page(struct block_map_page *page, 40 | nonce_t nonce, 41 | physical_block_number_t pbn) 42 | { 43 | STATIC_ASSERT_SIZEOF(struct block_map_page_header, 44 | PAGE_HEADER_4_1_SIZE); 45 | 46 | if (!vdo_are_same_version(BLOCK_MAP_4_1, 47 | vdo_unpack_version_number(page->version)) || 48 | !vdo_is_block_map_page_initialized(page) || 49 | (nonce != __le64_to_cpu(page->header.nonce))) { 50 | return VDO_BLOCK_MAP_PAGE_INVALID; 51 | } 52 | 53 | if (pbn != vdo_get_block_map_page_pbn(page)) { 54 | return VDO_BLOCK_MAP_PAGE_BAD; 55 | } 56 | 57 | return VDO_BLOCK_MAP_PAGE_VALID; 58 | } 59 | -------------------------------------------------------------------------------- /vdo/block-map-page.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef BLOCK_MAP_PAGE_H 7 | #define BLOCK_MAP_PAGE_H 8 | 9 | #include "numeric.h" 10 | 11 | #include "block-map-entry.h" 12 | #include "header.h" 13 | #include "types.h" 14 | 15 | struct block_map_page_header { 16 | __le64 nonce; 17 | __le64 pbn; 18 | 19 | /** May be non-zero on disk */ 20 | byte unused_long_word[8]; 21 | 22 | /* Whether this page has been written twice to disk */ 23 | bool initialized; 24 | 25 | /* Always zero on disk */ 26 | byte unused_byte1; 27 | 28 | /* May be non-zero on disk */ 29 | byte unused_byte2; 30 | byte unused_byte3; 31 | } __packed; 32 | 33 | struct block_map_page { 34 | struct packed_version_number version; 35 | struct block_map_page_header header; 36 | struct block_map_entry entries[]; 37 | } __packed; 38 | 39 | enum block_map_page_validity { 40 | VDO_BLOCK_MAP_PAGE_VALID, 41 | VDO_BLOCK_MAP_PAGE_INVALID, 42 | /* Valid page found in the wrong location on disk */ 43 | VDO_BLOCK_MAP_PAGE_BAD, 44 | }; 45 | 46 | static inline bool __must_check 47 | vdo_is_block_map_page_initialized(const struct block_map_page *page) 48 | { 49 | return page->header.initialized; 50 | } 51 | 52 | static inline bool 53 | vdo_mark_block_map_page_initialized(struct block_map_page *page, 54 | bool initialized) 55 | { 56 | if (initialized == page->header.initialized) { 57 | return false; 58 | } 59 | 60 | page->header.initialized = initialized; 61 | return true; 62 | } 63 | 64 | static inline physical_block_number_t __must_check 65 | vdo_get_block_map_page_pbn(const struct block_map_page *page) 66 | { 67 | return __le64_to_cpu(page->header.pbn); 68 | } 69 | 70 | struct block_map_page *vdo_format_block_map_page(void *buffer, 71 | nonce_t nonce, 72 | physical_block_number_t pbn, 73 | bool initialized); 74 | 75 | enum block_map_page_validity __must_check 76 | vdo_validate_block_map_page(struct block_map_page *page, 77 | nonce_t nonce, 78 | physical_block_number_t pbn); 79 | 80 | #endif /* BLOCK_MAP_PAGE_H */ 81 | -------------------------------------------------------------------------------- /vdo/block-map-recovery.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef BLOCK_MAP_RECOVERY_H 7 | #define BLOCK_MAP_RECOVERY_H 8 | 9 | #include "block-map.h" 10 | #include "block-mapping-state.h" 11 | #include "types.h" 12 | 13 | /* 14 | * An explicitly numbered block mapping. Numbering the mappings allows them to 15 | * be sorted by logical block number during recovery while still preserving 16 | * the relative order of journal entries with the same logical block number. 17 | */ 18 | struct numbered_block_mapping { 19 | struct block_map_slot block_map_slot; 20 | struct block_map_entry block_map_entry; 21 | /* A serial number to use during replay */ 22 | uint32_t number; 23 | } __packed; 24 | 25 | void vdo_recover_block_map(struct vdo *vdo, 26 | block_count_t entry_count, 27 | struct numbered_block_mapping *journal_entries, 28 | struct vdo_completion *parent); 29 | 30 | #endif /* BLOCK_MAP_RECOVERY_H */ 31 | -------------------------------------------------------------------------------- /vdo/block-map-tree.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef BLOCK_MAP_TREE_H 7 | #define BLOCK_MAP_TREE_H 8 | 9 | #include 10 | 11 | #include "block-map-format.h" 12 | #include "block-map-page.h" 13 | #include "constants.h" 14 | #include "kernel-types.h" 15 | #include "types.h" 16 | #include "wait-queue.h" 17 | 18 | struct tree_page { 19 | struct waiter waiter; 20 | 21 | /* Dirty list entry */ 22 | struct list_head entry; 23 | 24 | /* 25 | * If dirty, the tree zone flush generation in which it was last 26 | * dirtied. 27 | */ 28 | uint8_t generation; 29 | 30 | /* Whether this page is an interior tree page being written out. */ 31 | bool writing; 32 | 33 | /* 34 | * If writing, the tree zone flush generation of the copy being 35 | * written. 36 | */ 37 | uint8_t writing_generation; 38 | 39 | /* 40 | * Sequence number of the earliest recovery journal block containing 41 | * uncommitted updates to this page 42 | */ 43 | sequence_number_t recovery_lock; 44 | 45 | /* 46 | * The value of recovery_lock when the this page last started writing 47 | */ 48 | sequence_number_t writing_recovery_lock; 49 | 50 | char page_buffer[VDO_BLOCK_SIZE]; 51 | }; 52 | 53 | /* 54 | * Used to indicate that the page holding the location of a tree root has been 55 | * "loaded". 56 | */ 57 | extern const physical_block_number_t VDO_INVALID_PBN; 58 | 59 | static inline struct block_map_page * __must_check 60 | vdo_as_block_map_page(struct tree_page *tree_page) 61 | { 62 | return (struct block_map_page *) tree_page->page_buffer; 63 | } 64 | 65 | bool vdo_copy_valid_page(char *buffer, nonce_t nonce, 66 | physical_block_number_t pbn, 67 | struct block_map_page *page); 68 | 69 | int __must_check vdo_initialize_tree_zone(struct block_map_zone *zone, 70 | struct vdo *vdo, 71 | block_count_t maximum_age); 72 | 73 | void vdo_uninitialize_block_map_tree_zone(struct block_map_tree_zone *tree_zone); 74 | 75 | void vdo_set_tree_zone_initial_period(struct block_map_tree_zone *tree_zone, 76 | sequence_number_t period); 77 | 78 | bool __must_check vdo_is_tree_zone_active(struct block_map_tree_zone *zone); 79 | 80 | void vdo_advance_zone_tree_period(struct block_map_tree_zone *zone, 81 | sequence_number_t period); 82 | 83 | void vdo_drain_zone_trees(struct block_map_tree_zone *zone); 84 | 85 | void vdo_lookup_block_map_pbn(struct data_vio *data_vio); 86 | 87 | physical_block_number_t vdo_find_block_map_page_pbn(struct block_map *map, 88 | page_number_t page_number); 89 | 90 | void vdo_write_tree_page(struct tree_page *page, struct block_map_tree_zone *zone); 91 | 92 | #endif /* BLOCK_MAP_TREE_H */ 93 | -------------------------------------------------------------------------------- /vdo/block-mapping-state.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef BLOCK_MAPPING_STATE_H 7 | #define BLOCK_MAPPING_STATE_H 8 | 9 | #include "type-defs.h" 10 | 11 | /* 12 | * Four bits of each five-byte block map entry contain a mapping state value 13 | * used to distinguish unmapped or trimmed logical blocks (which are treated 14 | * as mapped to the zero block) from entries that have been mapped to a 15 | * physical block, including the zero block. 16 | * 17 | * FIXME: these should maybe be defines. 18 | */ 19 | enum block_mapping_state { 20 | VDO_MAPPING_STATE_UNMAPPED = 0, /* Must be zero to be the default value */ 21 | VDO_MAPPING_STATE_UNCOMPRESSED = 1, /* A normal (uncompressed) block */ 22 | VDO_MAPPING_STATE_COMPRESSED_BASE = 2, /* Compressed in slot 0 */ 23 | VDO_MAPPING_STATE_COMPRESSED_MAX = 15, /* Compressed in slot 13 */ 24 | }; 25 | 26 | enum { 27 | VDO_MAX_COMPRESSION_SLOTS = (VDO_MAPPING_STATE_COMPRESSED_MAX 28 | - VDO_MAPPING_STATE_COMPRESSED_BASE + 1), 29 | }; 30 | 31 | static inline enum block_mapping_state vdo_get_state_for_slot(byte slot_number) 32 | { 33 | return (slot_number + VDO_MAPPING_STATE_COMPRESSED_BASE); 34 | } 35 | 36 | static inline byte 37 | vdo_get_slot_from_state(enum block_mapping_state mapping_state) 38 | { 39 | return (mapping_state - VDO_MAPPING_STATE_COMPRESSED_BASE); 40 | } 41 | 42 | static inline bool 43 | vdo_is_state_compressed(const enum block_mapping_state mapping_state) 44 | { 45 | return (mapping_state > VDO_MAPPING_STATE_UNCOMPRESSED); 46 | } 47 | 48 | #endif /* BLOCK_MAPPING_STATE_H */ 49 | -------------------------------------------------------------------------------- /vdo/buffer.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef BUFFER_H 7 | #define BUFFER_H 8 | 9 | #include "common.h" 10 | 11 | struct buffer { 12 | size_t start; 13 | size_t end; 14 | size_t length; 15 | byte *data; 16 | bool wrapped; 17 | }; 18 | 19 | int __must_check wrap_buffer(byte *bytes, 20 | size_t length, 21 | size_t content_length, 22 | struct buffer **buffer_ptr); 23 | 24 | int __must_check make_buffer(size_t length, struct buffer **buffer_ptr); 25 | void free_buffer(struct buffer *buffer); 26 | 27 | bool __must_check ensure_available_space(struct buffer *buffer, size_t bytes); 28 | 29 | void clear_buffer(struct buffer *buffer); 30 | void compact_buffer(struct buffer *buffer); 31 | int __must_check skip_forward(struct buffer *buffer, size_t bytes_to_skip); 32 | int __must_check rewind_buffer(struct buffer *buffer, size_t bytes_to_rewind); 33 | 34 | size_t buffer_length(struct buffer *buffer); 35 | size_t content_length(struct buffer *buffer); 36 | size_t available_space(struct buffer *buffer); 37 | size_t uncompacted_amount(struct buffer *buffer); 38 | size_t buffer_used(struct buffer *buffer); 39 | 40 | int __must_check reset_buffer_end(struct buffer *buffer, size_t end); 41 | 42 | bool __must_check 43 | has_same_bytes(struct buffer *buffer, const byte *data, size_t length); 44 | bool equal_buffers(struct buffer *buffer1, struct buffer *buffer2); 45 | 46 | int __must_check get_byte(struct buffer *buffer, byte *byte_ptr); 47 | int __must_check put_byte(struct buffer *buffer, byte b); 48 | 49 | int __must_check 50 | get_bytes_from_buffer(struct buffer *buffer, size_t length, void *destination); 51 | byte *get_buffer_contents(struct buffer *buffer); 52 | int __must_check 53 | copy_bytes(struct buffer *buffer, size_t length, byte **destination_ptr); 54 | int __must_check 55 | put_bytes(struct buffer *buffer, size_t length, const void *source); 56 | int __must_check 57 | put_buffer(struct buffer *target, struct buffer *source, size_t length); 58 | 59 | int __must_check zero_bytes(struct buffer *buffer, size_t length); 60 | 61 | int __must_check get_boolean(struct buffer *buffer, bool *b); 62 | int __must_check put_boolean(struct buffer *buffer, bool b); 63 | 64 | int __must_check get_uint16_le_from_buffer(struct buffer *buffer, uint16_t *ui); 65 | int __must_check put_uint16_le_into_buffer(struct buffer *buffer, uint16_t ui); 66 | 67 | int __must_check 68 | get_uint16_les_from_buffer(struct buffer *buffer, size_t count, uint16_t *ui); 69 | int __must_check 70 | put_uint16_les_into_buffer(struct buffer *buffer, 71 | size_t count, 72 | const uint16_t *ui); 73 | 74 | int __must_check get_int32_le_from_buffer(struct buffer *buffer, int32_t *i); 75 | int __must_check get_uint32_le_from_buffer(struct buffer *buffer, uint32_t *ui); 76 | int __must_check put_uint32_le_into_buffer(struct buffer *buffer, uint32_t ui); 77 | 78 | int __must_check get_uint64_le_from_buffer(struct buffer *buffer, uint64_t *ui); 79 | int __must_check put_int64_le_into_buffer(struct buffer *buffer, int64_t i); 80 | int __must_check put_uint64_le_into_buffer(struct buffer *buffer, uint64_t ui); 81 | 82 | int __must_check 83 | get_uint64_les_from_buffer(struct buffer *buffer, size_t count, uint64_t *ui); 84 | int __must_check 85 | put_uint64_les_into_buffer(struct buffer *buffer, 86 | size_t count, 87 | const uint64_t *ui); 88 | 89 | #endif /* BUFFER_H */ 90 | -------------------------------------------------------------------------------- /vdo/buffered-reader.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef BUFFERED_READER_H 7 | #define BUFFERED_READER_H 1 8 | 9 | #include "common.h" 10 | 11 | struct buffered_reader; 12 | struct dm_bufio_client; 13 | struct io_factory; 14 | 15 | int __must_check make_buffered_reader(struct io_factory *factory, 16 | struct dm_bufio_client *client, 17 | sector_t block_limit, 18 | struct buffered_reader **reader_ptr); 19 | 20 | void free_buffered_reader(struct buffered_reader *reader); 21 | 22 | int __must_check read_from_buffered_reader(struct buffered_reader *reader, 23 | void *data, 24 | size_t length); 25 | 26 | int __must_check verify_buffered_data(struct buffered_reader *reader, 27 | const void *value, 28 | size_t length); 29 | 30 | #endif /* BUFFERED_READER_H */ 31 | -------------------------------------------------------------------------------- /vdo/buffered-writer.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef BUFFERED_WRITER_H 7 | #define BUFFERED_WRITER_H 1 8 | 9 | #include "common.h" 10 | 11 | struct buffered_writer; 12 | struct dm_bufio_client; 13 | struct io_factory; 14 | 15 | int __must_check make_buffered_writer(struct io_factory *factory, 16 | struct dm_bufio_client *client, 17 | sector_t block_limit, 18 | struct buffered_writer **writer_ptr); 19 | 20 | void free_buffered_writer(struct buffered_writer *buffer); 21 | 22 | int __must_check write_to_buffered_writer(struct buffered_writer *writer, 23 | const void *data, 24 | size_t len); 25 | 26 | int __must_check write_zeros_to_buffered_writer(struct buffered_writer *writer, 27 | size_t len); 28 | 29 | int __must_check flush_buffered_writer(struct buffered_writer *writer); 30 | 31 | #endif /* BUFFERED_WRITER_H */ 32 | -------------------------------------------------------------------------------- /vdo/chapter-index.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef CHAPTER_INDEX_H 7 | #define CHAPTER_INDEX_H 1 8 | 9 | #include "delta-index.h" 10 | #include "geometry.h" 11 | 12 | enum { 13 | /* 14 | * The value returned as the record page number when an entry is not 15 | * found in the chapter index. 16 | */ 17 | NO_CHAPTER_INDEX_ENTRY = -1 18 | }; 19 | 20 | struct open_chapter_index { 21 | const struct geometry *geometry; 22 | struct delta_index delta_index; 23 | uint64_t virtual_chapter_number; 24 | uint64_t volume_nonce; 25 | size_t memory_allocated; 26 | }; 27 | 28 | int __must_check 29 | make_open_chapter_index(struct open_chapter_index **chapter_index, 30 | const struct geometry *geometry, 31 | uint64_t volume_nonce); 32 | 33 | void free_open_chapter_index(struct open_chapter_index *chapter_index); 34 | 35 | void empty_open_chapter_index(struct open_chapter_index *chapter_index, 36 | uint64_t virtual_chapter_number); 37 | 38 | int __must_check 39 | put_open_chapter_index_record(struct open_chapter_index *chapter_index, 40 | const struct uds_chunk_name *name, 41 | unsigned int page_number); 42 | 43 | int __must_check 44 | pack_open_chapter_index_page(struct open_chapter_index *chapter_index, 45 | byte *memory, 46 | unsigned int first_list, 47 | bool last_page, 48 | unsigned int *num_lists); 49 | 50 | int __must_check 51 | initialize_chapter_index_page(struct delta_index_page *index_page, 52 | const struct geometry *geometry, 53 | byte *page_buffer, 54 | uint64_t volume_nonce); 55 | 56 | int __must_check 57 | validate_chapter_index_page(const struct delta_index_page *index_page, 58 | const struct geometry *geometry); 59 | 60 | int __must_check 61 | search_chapter_index_page(struct delta_index_page *index_page, 62 | const struct geometry *geometry, 63 | const struct uds_chunk_name *name, 64 | int *record_page_ptr); 65 | 66 | #endif /* CHAPTER_INDEX_H */ 67 | -------------------------------------------------------------------------------- /vdo/common.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef COMMON_H 7 | #define COMMON_H 8 | 9 | #include "string-utils.h" 10 | #include "type-defs.h" 11 | #include "uds.h" 12 | 13 | enum { 14 | KILOBYTE = 1024, 15 | MEGABYTE = KILOBYTE * KILOBYTE, 16 | GIGABYTE = KILOBYTE * MEGABYTE 17 | }; 18 | 19 | struct uds_chunk_data; 20 | 21 | struct uds_chunk_record { 22 | struct uds_chunk_name name; 23 | struct uds_chunk_data data; 24 | }; 25 | 26 | #endif /* COMMON_H */ 27 | -------------------------------------------------------------------------------- /vdo/compiler.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef COMMON_COMPILER_H 7 | #define COMMON_COMPILER_H 8 | 9 | #include 10 | #include 11 | 12 | 13 | #define const_container_of(ptr, type, member) \ 14 | __extension__({ \ 15 | const __typeof__(((type *) 0)->member) *__mptr = (ptr); \ 16 | (const type *) ((const char *) __mptr - \ 17 | offsetof(type, member)); \ 18 | }) 19 | 20 | /* 21 | * The "inline" keyword alone takes effect only when the optimization level 22 | * is high enough. Define INLINE to force the gcc to "always inline". 23 | */ 24 | #define INLINE __attribute__((always_inline)) inline 25 | 26 | 27 | 28 | #define __STRING(x) #x 29 | 30 | 31 | #endif /* COMMON_COMPILER_H */ 32 | -------------------------------------------------------------------------------- /vdo/compressed-block.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef COMPRESSED_BLOCK_H 7 | #define COMPRESSED_BLOCK_H 8 | 9 | #include "compiler.h" 10 | 11 | #include "block-mapping-state.h" 12 | #include "constants.h" 13 | #include "header.h" 14 | #include "types.h" 15 | 16 | /* 17 | * The header of a compressed block. 18 | */ 19 | struct compressed_block_header { 20 | /* 21 | * Unsigned 32-bit major and minor versions, 22 | * in little-endian byte order 23 | */ 24 | struct packed_version_number version; 25 | 26 | /* 27 | * List of unsigned 16-bit compressed block sizes, 28 | * in little-endian order 29 | */ 30 | __le16 sizes[VDO_MAX_COMPRESSION_SLOTS]; 31 | } __packed; 32 | 33 | enum { 34 | VDO_COMPRESSED_BLOCK_DATA_SIZE = 35 | VDO_BLOCK_SIZE - sizeof(struct compressed_block_header), 36 | 37 | /* 38 | * A compressed block is only written if we can pack at least two 39 | * fragments into it, so a fragment which fills the entire data portion 40 | * of a compressed block is too big. 41 | */ 42 | VDO_MAX_COMPRESSED_FRAGMENT_SIZE = VDO_COMPRESSED_BLOCK_DATA_SIZE - 1, 43 | }; 44 | 45 | /* 46 | * The compressed block overlay. 47 | */ 48 | struct compressed_block { 49 | struct compressed_block_header header; 50 | char data[VDO_COMPRESSED_BLOCK_DATA_SIZE]; 51 | } __packed; 52 | 53 | int vdo_get_compressed_block_fragment(enum block_mapping_state mapping_state, 54 | struct compressed_block *block, 55 | uint16_t *fragment_offset, 56 | uint16_t *fragment_size); 57 | 58 | void vdo_initialize_compressed_block(struct compressed_block *block, 59 | uint16_t size); 60 | 61 | static inline void 62 | vdo_clear_unused_compression_slots(struct compressed_block *block, 63 | slot_number_t first_unused) 64 | { 65 | if (first_unused < VDO_MAX_COMPRESSION_SLOTS) { 66 | memset(&block->header.sizes[first_unused], 67 | 0, 68 | ((VDO_MAX_COMPRESSION_SLOTS - first_unused) * 69 | sizeof(__le16))); 70 | } 71 | } 72 | 73 | void vdo_put_compressed_block_fragment(struct compressed_block *block, 74 | unsigned int fragment, 75 | uint16_t offset, 76 | const char *data, 77 | uint16_t size); 78 | 79 | #endif /* COMPRESSED_BLOCK_H */ 80 | -------------------------------------------------------------------------------- /vdo/compression-state.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef COMPRESSION_STATE_H 7 | #define COMPRESSION_STATE_H 8 | 9 | #include "kernel-types.h" 10 | #include "types.h" 11 | 12 | /* 13 | * Where a data_vio is on the compression path; advance_status() depends on the 14 | * order of this enum. 15 | */ 16 | enum vio_compression_status { 17 | /* A VIO which has not yet entered the compression path */ 18 | VIO_PRE_COMPRESSOR, 19 | /* A VIO which is in the compressor */ 20 | VIO_COMPRESSING, 21 | /* A VIO which is blocked in the packer */ 22 | VIO_PACKING, 23 | /* 24 | * A VIO which is no longer on the compression path (and never will be) 25 | */ 26 | VIO_POST_PACKER, 27 | }; 28 | 29 | struct vio_compression_state { 30 | enum vio_compression_status status; 31 | bool may_not_compress; 32 | }; 33 | 34 | struct vio_compression_state __must_check 35 | get_vio_compression_state(struct data_vio *data_vio); 36 | 37 | bool __must_check may_compress_data_vio(struct data_vio *data_vio); 38 | 39 | bool __must_check may_pack_data_vio(struct data_vio *data_vio); 40 | 41 | bool __must_check may_vio_block_in_packer(struct data_vio *data_vio); 42 | 43 | bool __must_check may_write_compressed_data_vio(struct data_vio *data_vio); 44 | 45 | void set_vio_compression_done(struct data_vio *data_vio); 46 | 47 | bool cancel_vio_compression(struct data_vio *data_vio); 48 | 49 | #endif /* COMPRESSION_STATE_H */ 50 | -------------------------------------------------------------------------------- /vdo/config.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef CONFIG_H 7 | #define CONFIG_H 8 | 9 | #include "buffered-reader.h" 10 | #include "buffered-writer.h" 11 | #include "geometry.h" 12 | #include "uds.h" 13 | 14 | enum { 15 | DEFAULT_VOLUME_INDEX_MEAN_DELTA = 4096, 16 | DEFAULT_CACHE_CHAPTERS = 7, 17 | DEFAULT_SPARSE_SAMPLE_RATE = 32, 18 | MAX_ZONES = 16, 19 | }; 20 | 21 | /* A set of configuration parameters for the indexer. */ 22 | struct configuration { 23 | /* String describing the storage device */ 24 | const char *name; 25 | 26 | /* The maximum allowable size of the index */ 27 | size_t size; 28 | 29 | /* The offset where the index should start */ 30 | off_t offset; 31 | 32 | /* Parameters for the volume */ 33 | 34 | /* The volume layout */ 35 | struct geometry *geometry; 36 | 37 | /* Index owner's nonce */ 38 | uint64_t nonce; 39 | 40 | /* The number of threads used to process index requests */ 41 | unsigned int zone_count; 42 | 43 | /* The number of threads used to read volume pages */ 44 | unsigned int read_threads; 45 | 46 | /* Size of the page cache and sparse chapter index cache in chapters */ 47 | unsigned int cache_chapters; 48 | 49 | /* Parameters for the volume index */ 50 | 51 | /* The mean delta for the volume index */ 52 | unsigned int volume_index_mean_delta; 53 | 54 | /* Sampling rate for sparse indexing */ 55 | unsigned int sparse_sample_rate; 56 | }; 57 | 58 | /* On-disk structure of data for a version 8.02 index. */ 59 | struct uds_configuration_8_02 { 60 | /* Smaller (16), Small (64) or large (256) indices */ 61 | unsigned int record_pages_per_chapter; 62 | /* Total number of chapters per volume */ 63 | unsigned int chapters_per_volume; 64 | /* Number of sparse chapters per volume */ 65 | unsigned int sparse_chapters_per_volume; 66 | /* Size of the page cache, in chapters */ 67 | unsigned int cache_chapters; 68 | /* Unused field */ 69 | unsigned int unused; 70 | /* The volume index mean delta to use */ 71 | unsigned int volume_index_mean_delta; 72 | /* Size of a page, used for both record pages and index pages */ 73 | unsigned int bytes_per_page; 74 | /* Sampling rate for sparse indexing */ 75 | unsigned int sparse_sample_rate; 76 | /* Index owner's nonce */ 77 | uint64_t nonce; 78 | /* Virtual chapter remapped from physical chapter 0 */ 79 | uint64_t remapped_virtual; 80 | /* New physical chapter which remapped chapter was moved to */ 81 | uint64_t remapped_physical; 82 | }; 83 | 84 | /* On-disk structure of data for a version 6.02 index. */ 85 | struct uds_configuration_6_02 { 86 | /* Smaller (16), Small (64) or large (256) indices */ 87 | unsigned int record_pages_per_chapter; 88 | /* Total number of chapters per volume */ 89 | unsigned int chapters_per_volume; 90 | /* Number of sparse chapters per volume */ 91 | unsigned int sparse_chapters_per_volume; 92 | /* Size of the page cache, in chapters */ 93 | unsigned int cache_chapters; 94 | /* Unused field */ 95 | unsigned int unused; 96 | /* The volume index mean delta to use */ 97 | unsigned int volume_index_mean_delta; 98 | /* Size of a page, used for both record pages and index pages */ 99 | unsigned int bytes_per_page; 100 | /* Sampling rate for sparse indexing */ 101 | unsigned int sparse_sample_rate; 102 | /* Index owner's nonce */ 103 | uint64_t nonce; 104 | }; 105 | 106 | int __must_check make_configuration(const struct uds_parameters *params, 107 | struct configuration **config_ptr); 108 | 109 | void free_configuration(struct configuration *config); 110 | 111 | int __must_check validate_config_contents(struct buffered_reader *reader, 112 | struct configuration *config); 113 | 114 | int __must_check write_config_contents(struct buffered_writer *writer, 115 | struct configuration *config, 116 | uint32_t version); 117 | 118 | void log_uds_configuration(struct configuration *config); 119 | 120 | #endif /* CONFIG_H */ 121 | -------------------------------------------------------------------------------- /vdo/constants.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #include "types.h" 7 | 8 | /* The maximum logical space is 4 petabytes, which is 1 terablock. */ 9 | const block_count_t MAXIMUM_VDO_LOGICAL_BLOCKS = 1024ULL * 1024 * 1024 * 1024; 10 | 11 | /* The maximum physical space is 256 terabytes, which is 64 gigablocks. */ 12 | const block_count_t MAXIMUM_VDO_PHYSICAL_BLOCKS = 1024ULL * 1024 * 1024 * 64; 13 | 14 | /* unit test minimum */ 15 | const block_count_t MINIMUM_VDO_SLAB_JOURNAL_BLOCKS = 2; 16 | -------------------------------------------------------------------------------- /vdo/constants.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef CONSTANTS_H 7 | #define CONSTANTS_H 8 | 9 | #include 10 | 11 | #include "types.h" 12 | 13 | enum { 14 | /** 15 | * The maximum number of contiguous PBNs which will go to a single 16 | * bio submission queue, assuming there is more than one queue. 17 | **/ 18 | VDO_BIO_ROTATION_INTERVAL_LIMIT = 1024, 19 | 20 | /** The number of entries on a block map page */ 21 | VDO_BLOCK_MAP_ENTRIES_PER_PAGE = 812, 22 | 23 | /** The origin of the flat portion of the block map */ 24 | VDO_BLOCK_MAP_FLAT_PAGE_ORIGIN = 1, 25 | 26 | /** 27 | * The height of a block map tree. Assuming a root count of 60 and 812 28 | * entries per page, this is big enough to represent almost 95 PB of 29 | * logical space. 30 | **/ 31 | VDO_BLOCK_MAP_TREE_HEIGHT = 5, 32 | 33 | /** The default number of bio submission queues. */ 34 | DEFAULT_VDO_BIO_SUBMIT_QUEUE_COUNT = 4, 35 | 36 | /** 37 | * The number of contiguous PBNs to be submitted to a single bio queue. 38 | **/ 39 | DEFAULT_VDO_BIO_SUBMIT_QUEUE_ROTATE_INTERVAL = 64, 40 | 41 | /** The number of trees in the arboreal block map */ 42 | DEFAULT_VDO_BLOCK_MAP_TREE_ROOT_COUNT = 60, 43 | 44 | /** The default size of the recovery journal, in blocks */ 45 | DEFAULT_VDO_RECOVERY_JOURNAL_SIZE = 32 * 1024, 46 | 47 | /** The default size of each slab journal, in blocks */ 48 | DEFAULT_VDO_SLAB_JOURNAL_SIZE = 224, 49 | 50 | /** 51 | * The initial size of lbn_operations and pbn_operations, which is 52 | * based upon the expected maximum number of outstanding VIOs. This 53 | * value was chosen to make it highly unlikely that the maps would 54 | * need to be resized. 55 | **/ 56 | VDO_LOCK_MAP_CAPACITY = 10000, 57 | 58 | /** The maximum number of logical zones */ 59 | MAX_VDO_LOGICAL_ZONES = 60, 60 | 61 | /** The maximum number of physical zones */ 62 | MAX_VDO_PHYSICAL_ZONES = 16, 63 | 64 | /** The base-2 logarithm of the maximum blocks in one slab */ 65 | MAX_VDO_SLAB_BITS = 23, 66 | 67 | /** The maximum number of slabs the slab depot supports */ 68 | MAX_VDO_SLABS = 8192, 69 | 70 | /** 71 | * The maximum number of block map pages to load simultaneously during 72 | * recovery or rebuild. 73 | **/ 74 | MAXIMUM_SIMULTANEOUS_VDO_BLOCK_MAP_RESTORATION_READS = 1024, 75 | 76 | /** 77 | * The maximum number of total threads in a VDO thread configuration. 78 | **/ 79 | MAXIMUM_VDO_THREADS = 100, 80 | 81 | /** The maximum number of VIOs in the system at once */ 82 | MAXIMUM_VDO_USER_VIOS = 2048, 83 | 84 | /** 85 | * The number of in-memory recovery journal blocks is determined by: 86 | * -- 311 journal entries in a 4k block 87 | * -- maximum of 2048 VIOs making entries at once 88 | * so we need at least 2048 / 312 = 7 journal blocks. 89 | **/ 90 | VDO_RECOVERY_JOURNAL_TAIL_BUFFER_SIZE = 64, 91 | 92 | /** The only physical block size supported by VDO */ 93 | VDO_BLOCK_SIZE = 4096, 94 | 95 | /** The number of sectors per block */ 96 | VDO_SECTORS_PER_BLOCK = (VDO_BLOCK_SIZE >> SECTOR_SHIFT), 97 | 98 | /** The size of a sector that will not be torn */ 99 | VDO_SECTOR_SIZE = 512, 100 | 101 | /** The physical block number reserved for storing the zero block */ 102 | VDO_ZERO_BLOCK = 0, 103 | }; 104 | 105 | /** The maximum logical space is 4 petabytes, which is 1 terablock. */ 106 | extern const block_count_t MAXIMUM_VDO_LOGICAL_BLOCKS; 107 | 108 | /** The maximum physical space is 256 terabytes, which is 64 gigablocks. */ 109 | extern const block_count_t MAXIMUM_VDO_PHYSICAL_BLOCKS; 110 | 111 | /** unit test minimum */ 112 | extern const block_count_t MINIMUM_VDO_SLAB_JOURNAL_BLOCKS; 113 | 114 | #endif /* CONSTANTS_H */ 115 | -------------------------------------------------------------------------------- /vdo/cpu.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef CPU_H 7 | #define CPU_H 8 | 9 | #include "compiler.h" 10 | #include "type-defs.h" 11 | 12 | /** 13 | * The number of bytes in a CPU cache line. In the future, we'll probably need 14 | * to move this to a processor-specific file or discover it at compilation 15 | * time (or runtime, if sufficiently heterogeneous), but this will do for now. 16 | * (Must be a \#define since enums are not proper compile-time constants.) 17 | **/ 18 | #ifdef __PPC__ 19 | /* N.B.: Some PPC processors have smaller cache lines. */ 20 | #define CACHE_LINE_BYTES 128 21 | #elif defined(__s390x__) 22 | #define CACHE_LINE_BYTES 256 23 | #elif defined(__x86_64__) || defined(__aarch64__) 24 | #define CACHE_LINE_BYTES 64 25 | #else 26 | #error "unknown cache line size" 27 | #endif 28 | 29 | /** 30 | * Minimize cache-miss latency by moving data into a CPU cache before it is 31 | * accessed. 32 | * 33 | * @param address the address to fetch (may be invalid) 34 | * @param for_write must be constant at compile time--false if 35 | * for reading, true if for writing 36 | **/ 37 | static INLINE void prefetch_address(const void *address, bool for_write) 38 | { 39 | /* 40 | * for_write won't be a constant if we are compiled with optimization 41 | * turned off, in which case prefetching really doesn't matter. 42 | * clang can't figure out that if for_write is a constant, it can be 43 | * passed as the second, mandatorily constant argument to prefetch(), 44 | * at least currently on llvm 12. 45 | */ 46 | if (__builtin_constant_p(for_write)) { 47 | if (for_write) { 48 | __builtin_prefetch(address, true); 49 | } else { 50 | __builtin_prefetch(address, false); 51 | } 52 | } 53 | } 54 | 55 | /** 56 | * Minimize cache-miss latency by moving a range of addresses into a 57 | * CPU cache before they are accessed. 58 | * 59 | * @param start the starting address to fetch (may be invalid) 60 | * @param size the number of bytes in the address range 61 | * @param for_write must be constant at compile time--false if 62 | * for reading, true if for writing 63 | **/ 64 | static INLINE void 65 | prefetch_range(const void *start, unsigned int size, bool for_write) 66 | { 67 | /* 68 | * Count the number of cache lines to fetch, allowing for the address 69 | * range to span an extra cache line boundary due to address alignment. 70 | */ 71 | const char *address = (const char *) start; 72 | unsigned int offset = ((uintptr_t) address % CACHE_LINE_BYTES); 73 | unsigned int cache_lines = (1 + ((size + offset) / CACHE_LINE_BYTES)); 74 | while (cache_lines-- > 0) { 75 | prefetch_address(address, for_write); 76 | address += CACHE_LINE_BYTES; 77 | } 78 | } 79 | 80 | #endif /* CPU_H */ 81 | -------------------------------------------------------------------------------- /vdo/data-vio-pool.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | * 5 | */ 6 | 7 | #ifndef DATA_VIO_POOL_H 8 | #define DATA_VIO_POOL_H 9 | 10 | #include 11 | 12 | #include "kernel-types.h" 13 | #include "types.h" 14 | 15 | int make_data_vio_pool(struct vdo *vdo, 16 | vio_count_t pool_size, 17 | vio_count_t discard_limit, 18 | struct data_vio_pool **pool_ptr); 19 | 20 | void free_data_vio_pool(struct data_vio_pool *pool); 21 | 22 | void vdo_launch_bio(struct data_vio_pool *pool, struct bio *bio); 23 | 24 | void release_data_vio(struct data_vio *data_vio); 25 | 26 | void drain_data_vio_pool(struct data_vio_pool *pool, 27 | struct vdo_completion *completion); 28 | 29 | void resume_data_vio_pool(struct data_vio_pool *pool, 30 | struct vdo_completion *completion); 31 | 32 | void dump_data_vio_pool(struct data_vio_pool *pool, bool dump_vios); 33 | 34 | vio_count_t get_data_vio_pool_active_discards(struct data_vio_pool *pool); 35 | vio_count_t get_data_vio_pool_discard_limit(struct data_vio_pool *pool); 36 | vio_count_t get_data_vio_pool_maximum_discards(struct data_vio_pool *pool); 37 | int __must_check set_data_vio_pool_discard_limit(struct data_vio_pool *pool, 38 | vio_count_t limit); 39 | vio_count_t get_data_vio_pool_active_requests(struct data_vio_pool *pool); 40 | vio_count_t get_data_vio_pool_request_limit(struct data_vio_pool *pool); 41 | vio_count_t get_data_vio_pool_maximum_requests(struct data_vio_pool *pool); 42 | 43 | #endif // DATA_VIO_POOL_H 44 | -------------------------------------------------------------------------------- /vdo/dedupe.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef DEDUPE_H 7 | #define DEDUPE_H 8 | 9 | #include 10 | #include 11 | 12 | #include "uds.h" 13 | 14 | #include "completion.h" 15 | #include "kernel-types.h" 16 | #include "statistics.h" 17 | #include "types.h" 18 | #include "wait-queue.h" 19 | 20 | struct hash_lock; 21 | struct hash_zone; 22 | struct hash_zones; 23 | 24 | struct pbn_lock * __must_check 25 | vdo_get_duplicate_lock(struct data_vio *data_vio); 26 | 27 | int __must_check vdo_acquire_hash_lock(struct data_vio *data_vio); 28 | 29 | void vdo_enter_hash_lock(struct data_vio *data_vio); 30 | 31 | void vdo_continue_hash_lock(struct data_vio *data_vio); 32 | 33 | void vdo_continue_hash_lock_on_error(struct data_vio *data_vio); 34 | 35 | void vdo_release_hash_lock(struct data_vio *data_vio); 36 | 37 | void vdo_share_compressed_write_lock(struct data_vio *data_vio, 38 | struct pbn_lock *pbn_lock); 39 | 40 | int __must_check 41 | vdo_make_hash_zones(struct vdo *vdo, struct hash_zones **zones_ptr); 42 | 43 | void vdo_free_hash_zones(struct hash_zones *zones); 44 | 45 | thread_id_t __must_check 46 | vdo_get_hash_zone_thread_id(const struct hash_zone *zone); 47 | 48 | void vdo_drain_hash_zones(struct hash_zones *zones, 49 | struct vdo_completion *parent); 50 | 51 | void vdo_get_dedupe_statistics(struct hash_zones *zones, 52 | struct vdo_statistics *stats); 53 | 54 | struct hash_zone * __must_check 55 | vdo_select_hash_zone(struct hash_zones *zones, 56 | const struct uds_chunk_name *name); 57 | 58 | void vdo_dump_hash_zones(struct hash_zones *zones); 59 | 60 | const char *vdo_get_dedupe_index_state_name(struct hash_zones *zones); 61 | 62 | uint64_t vdo_get_dedupe_index_timeout_count(struct hash_zones *zones); 63 | 64 | int vdo_message_dedupe_index(struct hash_zones *zones, const char *name); 65 | 66 | int vdo_add_dedupe_index_sysfs(struct hash_zones *zones); 67 | 68 | void vdo_start_dedupe_index(struct hash_zones *zones, bool create_flag); 69 | 70 | void vdo_resume_hash_zones(struct hash_zones *zones, 71 | struct vdo_completion *parent); 72 | 73 | void vdo_finish_dedupe_index(struct hash_zones *zones); 74 | 75 | /* 76 | * Interval (in milliseconds) from submission until switching to fast path and 77 | * skipping UDS. 78 | */ 79 | extern unsigned int vdo_dedupe_index_timeout_interval; 80 | 81 | /* 82 | * Minimum time interval (in milliseconds) between timer invocations to 83 | * check for requests waiting for UDS that should now time out. 84 | */ 85 | extern unsigned int vdo_dedupe_index_min_timer_interval; 86 | 87 | void vdo_set_dedupe_index_timeout_interval(unsigned int value); 88 | void vdo_set_dedupe_index_min_timer_interval(unsigned int value); 89 | 90 | #endif /* DEDUPE_H */ 91 | -------------------------------------------------------------------------------- /vdo/device-config.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | #ifndef DEVICE_CONFIG_H 6 | #define DEVICE_CONFIG_H 7 | 8 | #include 9 | #include 10 | 11 | #include "types.h" 12 | 13 | #include "kernel-types.h" 14 | 15 | /* 16 | * This structure is memcmp'd for equality. Keep it packed and don't add any 17 | * fields that are not properly set in both extant and parsed configs. 18 | */ 19 | struct thread_count_config { 20 | int bio_ack_threads; 21 | int bio_threads; 22 | int bio_rotation_interval; 23 | int cpu_threads; 24 | int logical_zones; 25 | int physical_zones; 26 | int hash_zones; 27 | } __packed; 28 | 29 | struct device_config { 30 | struct dm_target *owning_target; 31 | struct dm_dev *owned_device; 32 | struct vdo *vdo; 33 | /* All configs referencing a layer are kept on a list in the layer */ 34 | struct list_head config_list; 35 | char *original_string; 36 | unsigned int version; 37 | char *parent_device_name; 38 | block_count_t physical_blocks; 39 | /* 40 | * This is the number of logical blocks from VDO's internal point of 41 | * view. It is the number of 4K blocks regardles of the value of the 42 | * logical_block_size parameter below. 43 | */ 44 | block_count_t logical_blocks; 45 | unsigned int logical_block_size; 46 | unsigned int cache_size; 47 | unsigned int block_map_maximum_age; 48 | bool deduplication; 49 | bool compression; 50 | struct thread_count_config thread_counts; 51 | block_count_t max_discard_blocks; 52 | }; 53 | 54 | /** 55 | * vdo_as_device_config() - Convert a list entry to the device_config that 56 | * contains it. 57 | * @entry: The list entry to convert. 58 | * 59 | * If non-NULL, the list must not be empty. 60 | * 61 | * Return: The device_config wrapping the list entry. 62 | */ 63 | static inline struct device_config *vdo_as_device_config(struct list_head *entry) 64 | { 65 | if (entry == NULL) { 66 | return NULL; 67 | } 68 | return list_entry(entry, struct device_config, config_list); 69 | } 70 | 71 | int __must_check vdo_parse_device_config(int argc, 72 | char **argv, 73 | struct dm_target *ti, 74 | struct device_config **config_ptr); 75 | 76 | void vdo_free_device_config(struct device_config *config); 77 | 78 | void vdo_set_device_config(struct device_config *config, struct vdo *vdo); 79 | 80 | int __must_check 81 | vdo_validate_new_device_config(struct device_config *to_validate, 82 | struct device_config *config, 83 | bool may_grow, 84 | char **error_ptr); 85 | 86 | #endif /* DEVICE_CONFIG_H */ 87 | -------------------------------------------------------------------------------- /vdo/device-registry.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #include "device-registry.h" 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #include "memory-alloc.h" 13 | #include "permassert.h" 14 | 15 | #include "kernel-types.h" 16 | #include "status-codes.h" 17 | #include "types.h" 18 | #include "vdo.h" 19 | 20 | /* 21 | * We don't expect this set to ever get really large, so a linked list 22 | * is adequate. We can use a pointer_map if we need to later. 23 | */ 24 | struct device_registry { 25 | struct list_head links; 26 | /* 27 | * XXX: (Some) Kernel docs say rwlocks are being deprecated in favor of 28 | * RCU, please don't add more. Should we switch? 29 | */ 30 | rwlock_t lock; 31 | }; 32 | 33 | static struct device_registry registry; 34 | 35 | /** 36 | * vdo_initialize_device_registry_once() - Initialize the necessary 37 | * structures for the device registry. 38 | */ 39 | void vdo_initialize_device_registry_once(void) 40 | { 41 | INIT_LIST_HEAD(®istry.links); 42 | rwlock_init(®istry.lock); 43 | } 44 | 45 | /** 46 | * vdo_is_equal() - Implements vdo_filter_t. 47 | */ 48 | static bool vdo_is_equal(struct vdo *vdo, void *context) 49 | { 50 | return ((void *) vdo == context); 51 | } 52 | 53 | /** 54 | * filter_vdos_locked() - Find a vdo in the registry if it exists there. 55 | * @filter: The filter function to apply to devices. 56 | * @context: A bit of context to provide the filter. 57 | * 58 | * Context: Must be called holding the lock. 59 | * 60 | * Return: the vdo object found, if any. 61 | */ 62 | static struct vdo * __must_check 63 | filter_vdos_locked(vdo_filter_t *filter, void *context) 64 | { 65 | struct vdo *vdo; 66 | 67 | list_for_each_entry(vdo, ®istry.links, registration) { 68 | if (filter(vdo, context)) { 69 | return vdo; 70 | } 71 | } 72 | 73 | return NULL; 74 | } 75 | 76 | /** 77 | * vdo_register() - Register a VDO; it must not already be registered. 78 | * @vdo: The vdo to register. 79 | * 80 | * Return: VDO_SUCCESS or an error. 81 | */ 82 | int vdo_register(struct vdo *vdo) 83 | { 84 | int result; 85 | 86 | write_lock(®istry.lock); 87 | result = ASSERT(filter_vdos_locked(vdo_is_equal, vdo) == NULL, 88 | "VDO not already registered"); 89 | if (result == VDO_SUCCESS) { 90 | INIT_LIST_HEAD(&vdo->registration); 91 | list_add_tail(&vdo->registration, ®istry.links); 92 | } 93 | write_unlock(®istry.lock); 94 | 95 | return result; 96 | } 97 | 98 | /** 99 | * vdo_unregister() - Remove a vdo from the device registry. 100 | * @vdo: The vdo to remove. 101 | */ 102 | void vdo_unregister(struct vdo *vdo) 103 | { 104 | write_lock(®istry.lock); 105 | if (filter_vdos_locked(vdo_is_equal, vdo) == vdo) { 106 | list_del_init(&vdo->registration); 107 | } 108 | 109 | write_unlock(®istry.lock); 110 | } 111 | 112 | /** 113 | * vdo_find_matching() - Find and return the first (if any) vdo matching a 114 | * given filter function. 115 | * @filter: The filter function to apply to vdos. 116 | * @context: A bit of context to provide the filter. 117 | */ 118 | struct vdo *vdo_find_matching(vdo_filter_t *filter, void *context) 119 | { 120 | struct vdo *vdo; 121 | 122 | read_lock(®istry.lock); 123 | vdo = filter_vdos_locked(filter, context); 124 | read_unlock(®istry.lock); 125 | return vdo; 126 | } 127 | -------------------------------------------------------------------------------- /vdo/device-registry.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef DEVICE_REGISTRY_H 7 | #define DEVICE_REGISTRY_H 8 | 9 | #include "kernel-types.h" 10 | #include "types.h" 11 | 12 | /** 13 | * typedef vdo_filter_t - Method type for vdo matching methods. 14 | * 15 | * A filter function returns false if the vdo doesn't match. 16 | */ 17 | typedef bool vdo_filter_t(struct vdo *vdo, void *context); 18 | 19 | void vdo_initialize_device_registry_once(void); 20 | 21 | int __must_check vdo_register(struct vdo *vdo); 22 | 23 | void vdo_unregister(struct vdo *vdo); 24 | 25 | struct vdo * __must_check 26 | vdo_find_matching(vdo_filter_t *filter, void *context); 27 | 28 | #endif /* DEVICE_REGISTRY_H */ 29 | -------------------------------------------------------------------------------- /vdo/dirty-lists.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef DIRTY_LISTS_H 7 | #define DIRTY_LISTS_H 8 | 9 | #include 10 | 11 | #include "types.h" 12 | 13 | /** 14 | * struct dirty_lists - Lists of dirty elements. 15 | * 16 | * A collection of lists of dirty elements ordered by age. An element is always 17 | * placed on the oldest list in which it was dirtied (moving between lists or 18 | * removing altogether is cheap). Whenever the current period is advanced, any 19 | * elements older than the maxium age are expired. If an element is to be added 20 | * with a dirty age older than the maximum age, it is expired immediately. 21 | */ 22 | struct dirty_lists; 23 | 24 | /** 25 | * typedef vdo_dirty_callback - Callback for processing dirty elements. 26 | * @expired: The list of expired elements. 27 | * @context: The context for the callback. 28 | * 29 | * A function which will be called with a ring of dirty elements which have 30 | * been expired. All of the expired elements must be removed from the ring 31 | * before this function returns. 32 | */ 33 | typedef void vdo_dirty_callback(struct list_head *expired, void *context); 34 | 35 | int __must_check vdo_make_dirty_lists(block_count_t maximum_age, 36 | vdo_dirty_callback *callback, 37 | void *context, 38 | struct dirty_lists **dirty_lists_ptr); 39 | 40 | void vdo_set_dirty_lists_current_period(struct dirty_lists *dirty_lists, 41 | sequence_number_t period); 42 | 43 | void vdo_add_to_dirty_lists(struct dirty_lists *dirty_lists, 44 | struct list_head *entry, 45 | sequence_number_t old_period, 46 | sequence_number_t new_period); 47 | 48 | void vdo_advance_dirty_lists_period(struct dirty_lists *dirty_lists, 49 | sequence_number_t period); 50 | 51 | void vdo_flush_dirty_lists(struct dirty_lists *dirty_lists); 52 | 53 | #endif /* DIRTY_LISTS_H */ 54 | -------------------------------------------------------------------------------- /vdo/dump.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef DUMP_H 7 | #define DUMP_H 8 | 9 | #include "kernel-types.h" 10 | 11 | int vdo_dump(struct vdo *vdo, 12 | unsigned int argc, 13 | char *const *argv, 14 | const char *why); 15 | 16 | void vdo_dump_all(struct vdo *vdo, const char *why); 17 | 18 | void dump_data_vio(void *data); 19 | 20 | #endif /* DUMP_H */ 21 | -------------------------------------------------------------------------------- /vdo/errors.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef ERRORS_H 7 | #define ERRORS_H 8 | 9 | #include "compiler.h" 10 | #include "type-defs.h" 11 | 12 | /* Valid status codes for internal UDS functions. */ 13 | enum uds_status_codes { 14 | /* Successful return */ 15 | UDS_SUCCESS = 0, 16 | 17 | /* Used as a base value for reporting internal errors */ 18 | UDS_ERROR_CODE_BASE = 1024, 19 | /* Index overflow */ 20 | UDS_OVERFLOW = UDS_ERROR_CODE_BASE + 0, 21 | /* Invalid argument passed to internal routine */ 22 | UDS_INVALID_ARGUMENT = UDS_ERROR_CODE_BASE + 1, 23 | /* UDS data structures are in an invalid state */ 24 | UDS_BAD_STATE = UDS_ERROR_CODE_BASE + 2, 25 | /* Attempt to enter the same name into an internal structure twice */ 26 | UDS_DUPLICATE_NAME = UDS_ERROR_CODE_BASE + 3, 27 | /* An internal protocol violation between system components */ 28 | UDS_UNEXPECTED_RESULT = UDS_ERROR_CODE_BASE + 4, 29 | /* An assertion failed */ 30 | UDS_ASSERTION_FAILED = UDS_ERROR_CODE_BASE + 5, 31 | /* A request has been queued for later processing (not an error) */ 32 | UDS_QUEUED = UDS_ERROR_CODE_BASE + 6, 33 | /* A problem has occured with a buffer */ 34 | UDS_BUFFER_ERROR = UDS_ERROR_CODE_BASE + 7, 35 | /* No directory was found where one was expected */ 36 | UDS_NO_DIRECTORY = UDS_ERROR_CODE_BASE + 8, 37 | /* This error range has already been registered */ 38 | UDS_ALREADY_REGISTERED = UDS_ERROR_CODE_BASE + 9, 39 | /* Either read-only or write-only */ 40 | UDS_BAD_IO_DIRECTION = UDS_ERROR_CODE_BASE + 10, 41 | /* Cannot do I/O at this offset */ 42 | UDS_INCORRECT_ALIGNMENT = UDS_ERROR_CODE_BASE + 11, 43 | /* Attempt to read or write data outside the valid range */ 44 | UDS_OUT_OF_RANGE = UDS_ERROR_CODE_BASE + 12, 45 | /* Could not load modules */ 46 | UDS_EMODULE_LOAD = UDS_ERROR_CODE_BASE + 13, 47 | /* The index session is disabled */ 48 | UDS_DISABLED = UDS_ERROR_CODE_BASE + 14, 49 | /* Unknown error */ 50 | UDS_UNKNOWN_ERROR = UDS_ERROR_CODE_BASE + 15, 51 | /* The index configuration or volume format is no longer supported */ 52 | UDS_UNSUPPORTED_VERSION = UDS_ERROR_CODE_BASE + 16, 53 | /* Some index structure is corrupt */ 54 | UDS_CORRUPT_DATA = UDS_ERROR_CODE_BASE + 17, 55 | /* Short read due to truncated file */ 56 | UDS_SHORT_READ = UDS_ERROR_CODE_BASE + 18, 57 | /* Internal resource limits exceeded */ 58 | UDS_RESOURCE_LIMIT_EXCEEDED = UDS_ERROR_CODE_BASE + 19, 59 | /* Memory overflow due to storage failure */ 60 | UDS_VOLUME_OVERFLOW = UDS_ERROR_CODE_BASE + 20, 61 | /* No index state found */ 62 | UDS_NO_INDEX = UDS_ERROR_CODE_BASE + 21, 63 | /* Premature end of file in scanned file */ 64 | UDS_END_OF_FILE = UDS_ERROR_CODE_BASE + 22, 65 | /* Attempt to access incomplete index save data */ 66 | UDS_INDEX_NOT_SAVED_CLEANLY = UDS_ERROR_CODE_BASE + 23, 67 | /* One more than the last UDS_INTERNAL error code */ 68 | UDS_ERROR_CODE_LAST, 69 | /* One more than the last error this block will ever use */ 70 | UDS_ERROR_CODE_BLOCK_END = UDS_ERROR_CODE_BASE + 440, 71 | }; 72 | 73 | enum { 74 | UDS_MAX_ERROR_NAME_SIZE = 80, 75 | UDS_MAX_ERROR_MESSAGE_SIZE = 128, 76 | }; 77 | 78 | struct error_info { 79 | const char *name; 80 | const char *message; 81 | }; 82 | 83 | const char * __must_check uds_string_error(int errnum, 84 | char *buf, 85 | size_t buflen); 86 | 87 | const char *uds_string_error_name(int errnum, char *buf, size_t buflen); 88 | 89 | int uds_map_to_system_error(int error); 90 | 91 | int register_error_block(const char *block_name, 92 | int first_error, 93 | int last_reserved_error, 94 | const struct error_info *infos, 95 | size_t info_size); 96 | 97 | #endif /* ERRORS_H */ 98 | -------------------------------------------------------------------------------- /vdo/event-count.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef EVENT_COUNT_H 7 | #define EVENT_COUNT_H 8 | 9 | #include "time-utils.h" 10 | #include "type-defs.h" 11 | 12 | struct event_count; 13 | 14 | typedef unsigned int event_token_t; 15 | 16 | int __must_check make_event_count(struct event_count **count_ptr); 17 | 18 | void free_event_count(struct event_count *count); 19 | 20 | void event_count_broadcast(struct event_count *count); 21 | 22 | event_token_t __must_check event_count_prepare(struct event_count *count); 23 | 24 | void event_count_cancel(struct event_count *count, event_token_t token); 25 | 26 | bool event_count_wait(struct event_count *count, 27 | event_token_t token, 28 | const ktime_t *timeout); 29 | 30 | #endif /* EVENT_COUNT_H */ 31 | -------------------------------------------------------------------------------- /vdo/flush.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef FLUSH_H 7 | #define FLUSH_H 8 | 9 | #include "bio.h" 10 | #include "completion.h" 11 | #include "kernel-types.h" 12 | #include "types.h" 13 | #include "wait-queue.h" 14 | #include "workQueue.h" 15 | 16 | /* 17 | * A marker for tracking which journal entries are affected by a flush request. 18 | */ 19 | struct vdo_flush { 20 | /* The completion for enqueueing this flush request. */ 21 | struct vdo_completion completion; 22 | /* The flush bios covered by this request */ 23 | struct bio_list bios; 24 | /* The wait queue entry for this flush */ 25 | struct waiter waiter; 26 | /* Which flush this struct represents */ 27 | sequence_number_t flush_generation; 28 | }; 29 | 30 | int __must_check vdo_make_flusher(struct vdo *vdo); 31 | 32 | void vdo_free_flusher(struct flusher *flusher); 33 | 34 | thread_id_t __must_check vdo_get_flusher_thread_id(struct flusher *flusher); 35 | 36 | void vdo_complete_flushes(struct flusher *flusher); 37 | 38 | void vdo_dump_flusher(const struct flusher *flusher); 39 | 40 | void vdo_launch_flush(struct vdo *vdo, struct bio *bio); 41 | 42 | void vdo_drain_flusher(struct flusher *flusher, 43 | struct vdo_completion *completion); 44 | 45 | void vdo_resume_flusher(struct flusher *flusher, 46 | struct vdo_completion *parent); 47 | 48 | #endif /* FLUSH_H */ 49 | -------------------------------------------------------------------------------- /vdo/forest.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef FOREST_H 7 | #define FOREST_H 8 | 9 | #include "block-map-tree.h" 10 | #include "types.h" 11 | 12 | /** 13 | * typedef vdo_entry_callback - A function to be called for each allocated PBN 14 | * when traversing the forest. 15 | * @pbn: A PBN of a tree node. 16 | * @completion: The parent completion of the traversal. 17 | * 18 | * Return: VDO_SUCCESS or an error. 19 | */ 20 | typedef int vdo_entry_callback(physical_block_number_t pbn, 21 | struct vdo_completion *completion); 22 | 23 | struct tree_page * __must_check 24 | vdo_get_tree_page_by_index(struct forest *forest, 25 | root_count_t root_index, 26 | height_t height, 27 | page_number_t page_index); 28 | 29 | int __must_check vdo_make_forest(struct block_map *map, block_count_t entries); 30 | 31 | void vdo_free_forest(struct forest *forest); 32 | 33 | void vdo_abandon_forest(struct block_map *map); 34 | 35 | void vdo_replace_forest(struct block_map *map); 36 | 37 | void vdo_traverse_forest(struct block_map *map, 38 | vdo_entry_callback *callback, 39 | struct vdo_completion *parent); 40 | 41 | #endif /* FOREST_H */ 42 | -------------------------------------------------------------------------------- /vdo/funnel-queue.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef FUNNEL_QUEUE_H 7 | #define FUNNEL_QUEUE_H 8 | 9 | #include 10 | 11 | #include "compiler.h" 12 | #include "cpu.h" 13 | #include "type-defs.h" 14 | 15 | /* This queue link structure must be embedded in client entries. */ 16 | struct funnel_queue_entry { 17 | /* The next (newer) entry in the queue. */ 18 | struct funnel_queue_entry *volatile next; 19 | }; 20 | 21 | /* 22 | * The dynamically allocated queue structure, which is allocated on a cache 23 | * line boundary so the producer and consumer fields in the structure will land 24 | * on separate cache lines. This should be consider opaque but it is exposed 25 | * here so funnel_queue_put() can be inlined. 26 | */ 27 | struct __attribute__((aligned(CACHE_LINE_BYTES))) funnel_queue { 28 | /* 29 | * The producers' end of the queue, an atomically exchanged pointer 30 | * that will never be NULL. 31 | */ 32 | struct funnel_queue_entry *volatile newest; 33 | 34 | /* 35 | * The consumer's end of the queue, which is owned by the consumer and 36 | * never NULL. 37 | */ 38 | struct funnel_queue_entry *oldest 39 | __attribute__((aligned(CACHE_LINE_BYTES))); 40 | 41 | /* A dummy entry used to provide the non-NULL invariants above. */ 42 | struct funnel_queue_entry stub; 43 | }; 44 | 45 | int __must_check make_funnel_queue(struct funnel_queue **queue_ptr); 46 | 47 | void free_funnel_queue(struct funnel_queue *queue); 48 | 49 | /* 50 | * Put an entry on the end of the queue. 51 | * 52 | * The entry pointer must be to the struct funnel_queue_entry embedded in the 53 | * caller's data structure. The caller must be able to derive the address of 54 | * the start of their data structure from the pointer that passed in here, so 55 | * every entry in the queue must have the struct funnel_queue_entry at the same 56 | * offset within the client's structure. 57 | */ 58 | static INLINE void funnel_queue_put(struct funnel_queue *queue, 59 | struct funnel_queue_entry *entry) 60 | { 61 | struct funnel_queue_entry *previous; 62 | 63 | /* 64 | * Barrier requirements: All stores relating to the entry ("next" 65 | * pointer, containing data structure fields) must happen before the 66 | * previous->next store making it visible to the consumer. Also, the 67 | * entry's "next" field initialization to NULL must happen before any 68 | * other producer threads can see the entry (the xchg) and try to 69 | * update the "next" field. 70 | * 71 | * xchg implements a full barrier. 72 | */ 73 | entry->next = NULL; 74 | /* 75 | * The xchg macro in the PPC kernel calls a function that takes a void* 76 | * argument, triggering a warning about dropping the volatile 77 | * qualifier. 78 | */ 79 | #pragma GCC diagnostic push 80 | #if __GNUC__ >= 5 81 | #pragma GCC diagnostic ignored "-Wdiscarded-qualifiers" 82 | #endif 83 | previous = xchg(&queue->newest, entry); 84 | #pragma GCC diagnostic pop 85 | /* 86 | * Preemptions between these two statements hide the rest of the queue 87 | * from the consumer, preventing consumption until the following 88 | * assignment runs. 89 | */ 90 | previous->next = entry; 91 | } 92 | 93 | struct funnel_queue_entry *__must_check 94 | funnel_queue_poll(struct funnel_queue *queue); 95 | 96 | bool __must_check is_funnel_queue_empty(struct funnel_queue *queue); 97 | 98 | bool __must_check is_funnel_queue_idle(struct funnel_queue *queue); 99 | 100 | #endif /* FUNNEL_QUEUE_H */ 101 | -------------------------------------------------------------------------------- /vdo/hash-utils.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #include "hash-utils.h" 7 | 8 | #include "permassert.h" 9 | #include "uds.h" 10 | 11 | /* Compute the number of bits required to store a given value. */ 12 | unsigned int compute_bits(unsigned int max_value) 13 | { 14 | unsigned int bits = 0; 15 | 16 | while (max_value > 0) { 17 | max_value >>= 1; 18 | bits++; 19 | } 20 | 21 | return bits; 22 | } 23 | 24 | /* Special function wrapper required for compile-time assertions. */ 25 | void hash_utils_compile_time_assertions(void) 26 | { 27 | STATIC_ASSERT(UDS_CHUNK_NAME_SIZE == 16); 28 | } 29 | -------------------------------------------------------------------------------- /vdo/hash-utils.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef HASH_UTILS_H 7 | #define HASH_UTILS_H 1 8 | 9 | #include "compiler.h" 10 | #include "common.h" 11 | #include "geometry.h" 12 | #include "numeric.h" 13 | #include "uds.h" 14 | 15 | /* How various portions of a chunk name are apportioned. */ 16 | enum { 17 | VOLUME_INDEX_BYTES_OFFSET = 0, 18 | VOLUME_INDEX_BYTES_COUNT = 8, 19 | CHAPTER_INDEX_BYTES_OFFSET = 8, 20 | CHAPTER_INDEX_BYTES_COUNT = 6, 21 | SAMPLE_BYTES_OFFSET = 14, 22 | SAMPLE_BYTES_COUNT = 2, 23 | }; 24 | 25 | static INLINE uint64_t 26 | extract_chapter_index_bytes(const struct uds_chunk_name *name) 27 | { 28 | const byte *chapter_bits = &name->name[CHAPTER_INDEX_BYTES_OFFSET]; 29 | uint64_t bytes = (uint64_t) get_unaligned_be16(chapter_bits) << 32; 30 | 31 | bytes |= get_unaligned_be32(chapter_bits + 2); 32 | return bytes; 33 | } 34 | 35 | static INLINE uint64_t 36 | extract_volume_index_bytes(const struct uds_chunk_name *name) 37 | { 38 | return get_unaligned_be64(&name->name[VOLUME_INDEX_BYTES_OFFSET]); 39 | } 40 | 41 | static INLINE uint32_t 42 | extract_sampling_bytes(const struct uds_chunk_name *name) 43 | { 44 | return get_unaligned_be16(&name->name[SAMPLE_BYTES_OFFSET]); 45 | } 46 | 47 | /* Compute the chapter delta list for a given name. */ 48 | static INLINE unsigned int 49 | hash_to_chapter_delta_list(const struct uds_chunk_name *name, 50 | const struct geometry *geometry) 51 | { 52 | return (unsigned int) ((extract_chapter_index_bytes(name) >> 53 | geometry->chapter_address_bits) & 54 | ((1 << geometry->chapter_delta_list_bits) - 1)); 55 | } 56 | 57 | /* Compute the chapter delta address for a given name. */ 58 | static INLINE unsigned int 59 | hash_to_chapter_delta_address(const struct uds_chunk_name *name, 60 | const struct geometry *geometry) 61 | { 62 | return (unsigned int) (extract_chapter_index_bytes(name) & 63 | ((1 << geometry->chapter_address_bits) - 1)); 64 | } 65 | 66 | static INLINE unsigned int name_to_hash_slot(const struct uds_chunk_name *name, 67 | unsigned int slot_count) 68 | { 69 | return (unsigned int) (extract_chapter_index_bytes(name) % slot_count); 70 | } 71 | 72 | unsigned int __must_check compute_bits(unsigned int max_value); 73 | 74 | void hash_utils_compile_time_assertions(void); 75 | 76 | #endif /* HASH_UTILS_H */ 77 | -------------------------------------------------------------------------------- /vdo/heap.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef HEAP_H 7 | #define HEAP_H 8 | 9 | #include "type-defs.h" 10 | 11 | /** 12 | * typedef heap_comparator - Prototype for functions which compare two array 13 | * elements. 14 | * @item1: The first element to compare. 15 | * @item2: The second element to compare. 16 | * 17 | * All the time complexity claims in this module assume this operation has 18 | * O(1) time complexity. 19 | * 20 | * Return: An integer which is less than, equal to, or greater than 0 21 | * depending on whether item1 is less than, equal to, or greater 22 | * than item2, respectively 23 | */ 24 | typedef int heap_comparator(const void *item1, const void *item2); 25 | 26 | /** 27 | * typedef heap_swapper - Prototype for functions which swap two array 28 | * elements. 29 | * @item1: The first element to swap. 30 | * @item2: The second element to swap. 31 | */ 32 | typedef void heap_swapper(void *item1, void *item2); 33 | 34 | /* 35 | * A heap array can be any array of fixed-length elements in which the heap 36 | * invariant can be established. In a max-heap, every child of a node must be 37 | * at least as large as its children. Once that invariant is established in an 38 | * array by calling build_heap(), all the other heap operations may be used on 39 | * that array. 40 | */ 41 | struct heap { 42 | /* the 1-based array of heap elements (nodes) */ 43 | byte *array; 44 | /* the function to use to compare two elements */ 45 | heap_comparator *comparator; 46 | /* the function to use to swap two elements */ 47 | heap_swapper *swapper; 48 | /* the maximum number of elements that can be stored */ 49 | size_t capacity; 50 | /* the size of every element (in bytes) */ 51 | size_t element_size; 52 | /* the current number of elements in the heap */ 53 | size_t count; 54 | }; 55 | 56 | void initialize_heap(struct heap *heap, heap_comparator *comparator, 57 | heap_swapper *swapper, void *array, size_t capacity, 58 | size_t element_size); 59 | 60 | void build_heap(struct heap *heap, size_t count); 61 | 62 | /** 63 | * is_heap_empty() - Check whether the heap is currently empty. 64 | * @heap: The heap to query. 65 | * 66 | * Return: true if there are no elements in the heap. 67 | */ 68 | static inline bool is_heap_empty(const struct heap *heap) 69 | { 70 | return (heap->count == 0); 71 | } 72 | 73 | bool pop_max_heap_element(struct heap *heap, void *element_ptr); 74 | 75 | size_t sort_heap(struct heap *heap); 76 | 77 | void *sort_next_heap_element(struct heap *heap); 78 | 79 | #endif /* HEAP_H */ 80 | -------------------------------------------------------------------------------- /vdo/index-layout.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef INDEX_LAYOUT_H 7 | #define INDEX_LAYOUT_H 8 | 9 | #include "buffer.h" 10 | #include "config.h" 11 | #include "io-factory.h" 12 | #include "uds.h" 13 | 14 | struct index_layout; 15 | 16 | int __must_check make_uds_index_layout(struct configuration *config, 17 | bool new_layout, 18 | struct index_layout **layout_ptr); 19 | 20 | void free_uds_index_layout(struct index_layout *layout); 21 | 22 | int __must_check replace_index_layout_storage(struct index_layout *layout, 23 | const char *name); 24 | 25 | int __must_check load_index_state(struct index_layout *layout, 26 | struct uds_index *index); 27 | 28 | int __must_check save_index_state(struct index_layout *layout, 29 | struct uds_index *index); 30 | 31 | int discard_index_state_data(struct index_layout *layout); 32 | 33 | int __must_check discard_open_chapter(struct index_layout *layout); 34 | 35 | uint64_t __must_check get_uds_volume_nonce(struct index_layout *layout); 36 | 37 | int __must_check open_uds_volume_bufio(struct index_layout *layout, 38 | size_t block_size, 39 | unsigned int reserved_buffers, 40 | struct dm_bufio_client **client_ptr); 41 | 42 | #endif /* INDEX_LAYOUT_H */ 43 | -------------------------------------------------------------------------------- /vdo/index-page-map.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef INDEX_PAGE_MAP_H 7 | #define INDEX_PAGE_MAP_H 1 8 | 9 | #include "buffered-reader.h" 10 | #include "buffered-writer.h" 11 | #include "common.h" 12 | #include "geometry.h" 13 | 14 | struct index_page_map { 15 | const struct geometry *geometry; 16 | uint64_t last_update; 17 | unsigned int entries_per_chapter; 18 | uint16_t *entries; 19 | }; 20 | 21 | int __must_check make_index_page_map(const struct geometry *geometry, 22 | struct index_page_map **map_ptr); 23 | 24 | void free_index_page_map(struct index_page_map *map); 25 | 26 | int __must_check read_index_page_map(struct index_page_map *map, 27 | struct buffered_reader *reader); 28 | 29 | int __must_check write_index_page_map(struct index_page_map *map, 30 | struct buffered_writer *writer); 31 | 32 | void update_index_page_map(struct index_page_map *map, 33 | uint64_t virtual_chapter_number, 34 | unsigned int chapter_number, 35 | unsigned int index_page_number, 36 | unsigned int delta_list_number); 37 | 38 | unsigned int __must_check 39 | find_index_page_number(const struct index_page_map *map, 40 | const struct uds_chunk_name *name, 41 | unsigned int chapter_number); 42 | 43 | void get_list_number_bounds(const struct index_page_map *map, 44 | unsigned int chapter_number, 45 | unsigned int index_page_number, 46 | unsigned int *lowest_list, 47 | unsigned int *highest_list); 48 | 49 | uint64_t compute_index_page_map_save_size(const struct geometry *geometry); 50 | 51 | #endif /* INDEX_PAGE_MAP_H */ 52 | -------------------------------------------------------------------------------- /vdo/index-session.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef INDEX_SESSION_H 7 | #define INDEX_SESSION_H 8 | 9 | #include 10 | 11 | #include "config.h" 12 | #include "cpu.h" 13 | #include "uds-threads.h" 14 | #include "uds.h" 15 | 16 | struct __attribute__((aligned(CACHE_LINE_BYTES))) session_stats { 17 | /* Post requests that found an entry */ 18 | uint64_t posts_found; 19 | /* Post requests found in the open chapter */ 20 | uint64_t posts_found_open_chapter; 21 | /* Post requests found in the dense index */ 22 | uint64_t posts_found_dense; 23 | /* Post requests found in the sparse index */ 24 | uint64_t posts_found_sparse; 25 | /* Post requests that did not find an entry */ 26 | uint64_t posts_not_found; 27 | /* Update requests that found an entry */ 28 | uint64_t updates_found; 29 | /* Update requests that did not find an entry */ 30 | uint64_t updates_not_found; 31 | /* Delete requests that found an entry */ 32 | uint64_t deletions_found; 33 | /* Delete requests that did not find an entry */ 34 | uint64_t deletions_not_found; 35 | /* Query requests that found an entry */ 36 | uint64_t queries_found; 37 | /* Query requests that did not find an entry */ 38 | uint64_t queries_not_found; 39 | /* Total number of requests */ 40 | uint64_t requests; 41 | }; 42 | 43 | enum index_suspend_status { 44 | /* An index load has started but the index is not ready for use. */ 45 | INDEX_OPENING = 0, 46 | /* The index is able to handle requests. */ 47 | INDEX_READY, 48 | /* The index is attempting to suspend a rebuild. */ 49 | INDEX_SUSPENDING, 50 | /* An index rebuild has been suspended. */ 51 | INDEX_SUSPENDED, 52 | /* An index rebuild is being stopped in order to shut down. */ 53 | INDEX_FREEING, 54 | }; 55 | 56 | struct index_load_context { 57 | struct mutex mutex; 58 | struct cond_var cond; 59 | enum index_suspend_status status; 60 | }; 61 | 62 | struct uds_index_session { 63 | unsigned int state; 64 | struct uds_index *index; 65 | struct uds_request_queue *callback_queue; 66 | struct uds_parameters parameters; 67 | struct index_load_context load_context; 68 | struct mutex request_mutex; 69 | struct cond_var request_cond; 70 | int request_count; 71 | struct session_stats stats; 72 | }; 73 | 74 | #endif /* INDEX_SESSION_H */ 75 | -------------------------------------------------------------------------------- /vdo/index.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef INDEX_H 7 | #define INDEX_H 8 | 9 | #include "index-layout.h" 10 | #include "index-session.h" 11 | #include "open-chapter.h" 12 | #include "volume.h" 13 | #include "volume-index-ops.h" 14 | 15 | 16 | typedef void (*index_callback_t)(struct uds_request *request); 17 | 18 | struct index_zone { 19 | struct uds_index *index; 20 | struct open_chapter_zone *open_chapter; 21 | struct open_chapter_zone *writing_chapter; 22 | uint64_t oldest_virtual_chapter; 23 | uint64_t newest_virtual_chapter; 24 | unsigned int id; 25 | }; 26 | 27 | struct uds_index { 28 | bool has_saved_open_chapter; 29 | bool need_to_save; 30 | struct index_load_context *load_context; 31 | struct index_layout *layout; 32 | struct volume_index *volume_index; 33 | struct volume *volume; 34 | unsigned int zone_count; 35 | struct index_zone **zones; 36 | 37 | uint64_t oldest_virtual_chapter; 38 | uint64_t newest_virtual_chapter; 39 | 40 | uint64_t last_save; 41 | uint64_t prev_save; 42 | struct chapter_writer *chapter_writer; 43 | 44 | index_callback_t callback; 45 | struct uds_request_queue *triage_queue; 46 | struct uds_request_queue *zone_queues[]; 47 | }; 48 | 49 | enum request_stage { 50 | STAGE_TRIAGE, 51 | STAGE_INDEX, 52 | STAGE_MESSAGE, 53 | }; 54 | 55 | int __must_check make_index(struct configuration *config, 56 | enum uds_open_index_type open_type, 57 | struct index_load_context *load_context, 58 | index_callback_t callback, 59 | struct uds_index **new_index); 60 | 61 | int __must_check save_index(struct uds_index *index); 62 | 63 | void free_index(struct uds_index *index); 64 | 65 | int __must_check replace_index_storage(struct uds_index *index, 66 | const char *path); 67 | 68 | void get_index_stats(struct uds_index *index, 69 | struct uds_index_stats *counters); 70 | 71 | void enqueue_request(struct uds_request *request, enum request_stage stage); 72 | 73 | void wait_for_idle_index(struct uds_index *index); 74 | 75 | #endif /* INDEX_H */ 76 | -------------------------------------------------------------------------------- /vdo/instance-number.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef INSTANCE_NUMBER_H 7 | #define INSTANCE_NUMBER_H 8 | 9 | int vdo_allocate_instance(unsigned int *instance_ptr); 10 | 11 | void vdo_release_instance(unsigned int instance); 12 | 13 | void vdo_initialize_instance_number_tracking(void); 14 | 15 | void vdo_clean_up_instance_number_tracking(void); 16 | 17 | #endif /* INSTANCE_NUMBER_H */ 18 | -------------------------------------------------------------------------------- /vdo/int-map.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef INT_MAP_H 7 | #define INT_MAP_H 8 | 9 | #include "compiler.h" 10 | #include "type-defs.h" 11 | 12 | /** 13 | * DOC: int_map 14 | * 15 | * An int_map associates pointers (void *) with integer keys (uint64_t). NULL 16 | * pointer values are not supported. 17 | * 18 | * The map is implemented as hash table, which should provide constant-time 19 | * insert, query, and remove operations, although the insert may occasionally 20 | * grow the table, which is linear in the number of entries in the map. The 21 | * table will grow as needed to hold new entries, but will not shrink as 22 | * entries are removed. 23 | */ 24 | 25 | struct int_map; 26 | 27 | int __must_check make_int_map(size_t initial_capacity, 28 | unsigned int initial_load, 29 | struct int_map **map_ptr); 30 | 31 | void free_int_map(struct int_map *map); 32 | 33 | size_t int_map_size(const struct int_map *map); 34 | 35 | void *int_map_get(struct int_map *map, uint64_t key); 36 | 37 | int __must_check int_map_put(struct int_map *map, 38 | uint64_t key, 39 | void *new_value, 40 | bool update, 41 | void **old_value_ptr); 42 | 43 | void *int_map_remove(struct int_map *map, uint64_t key); 44 | 45 | #endif /* INT_MAP_H */ 46 | -------------------------------------------------------------------------------- /vdo/io-submitter.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef IO_SUBMITTER_H 7 | #define IO_SUBMITTER_H 8 | 9 | #include 10 | 11 | #include "completion.h" 12 | #include "kernel-types.h" 13 | #include "vio.h" 14 | 15 | int vdo_make_io_submitter(unsigned int thread_count, 16 | unsigned int rotation_interval, 17 | unsigned int max_requests_active, 18 | struct vdo *vdo, 19 | struct io_submitter **io_submitter); 20 | 21 | void vdo_cleanup_io_submitter(struct io_submitter *io_submitter); 22 | 23 | void vdo_free_io_submitter(struct io_submitter *io_submitter); 24 | 25 | void process_vio_io(struct vdo_completion *completion); 26 | 27 | void submit_data_vio_io(struct data_vio *data_vio); 28 | 29 | void vdo_submit_metadata_io(struct vio *vio, 30 | physical_block_number_t physical, 31 | bio_end_io_t callback, 32 | vdo_action *error_handler, 33 | unsigned int operation, 34 | char *data); 35 | 36 | static inline void submit_metadata_vio(struct vio *vio, 37 | physical_block_number_t physical, 38 | bio_end_io_t callback, 39 | vdo_action *error_handler, 40 | unsigned int operation) 41 | { 42 | vdo_submit_metadata_io(vio, 43 | physical, 44 | callback, 45 | error_handler, 46 | operation, 47 | vio->data); 48 | } 49 | 50 | static inline void submit_flush_vio(struct vio *vio, 51 | bio_end_io_t callback, 52 | vdo_action *error_handler) 53 | { 54 | /* FIXME: Can we just use REQ_OP_FLUSH? */ 55 | vdo_submit_metadata_io(vio, 56 | 0, 57 | callback, 58 | error_handler, 59 | REQ_OP_WRITE | REQ_PREFLUSH, 60 | NULL); 61 | } 62 | 63 | #endif /* IO_SUBMITTER_H */ 64 | -------------------------------------------------------------------------------- /vdo/linux/murmurhash3.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: LGPL-2.1+ */ 2 | /* 3 | * MurmurHash3 was written by Austin Appleby, and is placed in the public 4 | * domain. The author hereby disclaims copyright to this source code. 5 | */ 6 | 7 | #ifndef _MURMURHASH3_H_ 8 | #define _MURMURHASH3_H_ 9 | 10 | #include 11 | 12 | void murmurhash3_128(const void *key, int len, uint32_t seed, void *out); 13 | 14 | #endif /* _MURMURHASH3_H_ */ 15 | -------------------------------------------------------------------------------- /vdo/lock-counter.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef LOCK_COUNTER_H 7 | #define LOCK_COUNTER_H 8 | 9 | #include "completion.h" 10 | #include "types.h" 11 | 12 | /** 13 | * DOC: Lock Counters. 14 | * 15 | * A lock_counter provides a set of shared reference count locks which is safe 16 | * across multiple zones with a minimum of cross-thread synchronization 17 | * operations. For each lock in the set, it maintains a set of per-zone lock 18 | * counts, and a single, atomic count of the number of zones holding locks. 19 | * Whenever a zone's individual counter for a lock goes from 0 to 1, the 20 | * zone count for that lock is incremented. Whenever a zone's individual 21 | * counter for a lock goes from 1 to 0, the zone count for that lock is 22 | * decremented. If the zone count goes to 0, and the lock counter's 23 | * completion is not in use, the completion is launched to inform the counter's 24 | * owner that some lock has been released. It is the owner's responsibility to 25 | * check for which locks have been released, and to inform the lock counter 26 | * that it has received the notification by calling 27 | * vdo_acknowledge_lock_unlock(). 28 | */ 29 | 30 | int __must_check vdo_make_lock_counter(struct vdo *vdo, 31 | void *parent, 32 | vdo_action callback, 33 | thread_id_t thread_id, 34 | zone_count_t logical_zones, 35 | zone_count_t physical_zones, 36 | block_count_t locks, 37 | struct lock_counter **lock_counter_ptr); 38 | 39 | void vdo_free_lock_counter(struct lock_counter *counter); 40 | 41 | bool __must_check vdo_is_lock_locked(struct lock_counter *lock_counter, 42 | block_count_t lock_number, 43 | enum vdo_zone_type zone_type); 44 | 45 | void vdo_initialize_lock_count(struct lock_counter *counter, 46 | block_count_t lock_number, 47 | uint16_t value); 48 | 49 | void vdo_acquire_lock_count_reference(struct lock_counter *counter, 50 | block_count_t lock_number, 51 | enum vdo_zone_type zone_type, 52 | zone_count_t zone_id); 53 | 54 | void vdo_release_lock_count_reference(struct lock_counter *counter, 55 | block_count_t lock_number, 56 | enum vdo_zone_type zone_type, 57 | zone_count_t zone_id); 58 | 59 | void vdo_release_journal_zone_reference(struct lock_counter *counter, 60 | block_count_t lock_number); 61 | 62 | void 63 | vdo_release_journal_zone_reference_from_other_zone(struct lock_counter *counter, 64 | block_count_t lock_number); 65 | 66 | void vdo_acknowledge_lock_unlock(struct lock_counter *counter); 67 | 68 | bool __must_check vdo_suspend_lock_counter(struct lock_counter *counter); 69 | 70 | bool __must_check vdo_resume_lock_counter(struct lock_counter *counter); 71 | 72 | #endif /* LOCK_COUNTER_H */ 73 | -------------------------------------------------------------------------------- /vdo/logical-zone.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef LOGICAL_ZONE_H 7 | #define LOGICAL_ZONE_H 8 | 9 | #include 10 | 11 | #include "admin-state.h" 12 | #include "int-map.h" 13 | #include "types.h" 14 | 15 | struct logical_zone { 16 | /* The completion for flush notifications */ 17 | struct vdo_completion completion; 18 | /* The owner of this zone */ 19 | struct logical_zones *zones; 20 | /* Which logical zone this is */ 21 | zone_count_t zone_number; 22 | /* The thread id for this zone */ 23 | thread_id_t thread_id; 24 | /* In progress operations keyed by LBN */ 25 | struct int_map *lbn_operations; 26 | /* The logical to physical map */ 27 | struct block_map_zone *block_map_zone; 28 | /* The current flush generation */ 29 | sequence_number_t flush_generation; 30 | /* 31 | * The oldest active generation in this zone. This is mutated only on 32 | * the logical zone thread but is queried from the flusher thread. 33 | */ 34 | sequence_number_t oldest_active_generation; 35 | /* The number of IOs in the current flush generation */ 36 | block_count_t ios_in_flush_generation; 37 | /* The youngest generation of the current notification */ 38 | sequence_number_t notification_generation; 39 | /* Whether a notification is in progress */ 40 | bool notifying; 41 | /* The queue of active data write VIOs */ 42 | struct list_head write_vios; 43 | /* The administrative state of the zone */ 44 | struct admin_state state; 45 | /* The selector for determining which physical zone to allocate from */ 46 | struct allocation_selector *selector; 47 | /* The next zone */ 48 | struct logical_zone *next; 49 | }; 50 | 51 | struct logical_zones { 52 | /* The vdo whose zones these are */ 53 | struct vdo *vdo; 54 | /* The manager for administrative actions */ 55 | struct action_manager *manager; 56 | /* The number of zones */ 57 | zone_count_t zone_count; 58 | /* The logical zones themselves */ 59 | struct logical_zone zones[]; 60 | }; 61 | 62 | int __must_check 63 | vdo_make_logical_zones(struct vdo *vdo, struct logical_zones **zones_ptr); 64 | 65 | void vdo_free_logical_zones(struct logical_zones *zones); 66 | 67 | void vdo_drain_logical_zones(struct logical_zones *zones, 68 | const struct admin_state_code *operation, 69 | struct vdo_completion *completion); 70 | 71 | void vdo_resume_logical_zones(struct logical_zones *zones, 72 | struct vdo_completion *parent); 73 | 74 | void 75 | vdo_increment_logical_zone_flush_generation(struct logical_zone *zone, 76 | sequence_number_t expected_generation); 77 | 78 | int __must_check vdo_acquire_flush_generation_lock(struct data_vio *data_vio); 79 | 80 | void vdo_release_flush_generation_lock(struct data_vio *data_vio); 81 | 82 | void vdo_dump_logical_zone(const struct logical_zone *zone); 83 | 84 | #endif /* LOGICAL_ZONE_H */ 85 | -------------------------------------------------------------------------------- /vdo/messageStats.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef MESSAGE_STATS_H 7 | #define MESSAGE_STATS_H 8 | 9 | #include "types.h" 10 | 11 | int vdo_write_stats(struct vdo *vdo, char *buf, unsigned int maxlen); 12 | 13 | #endif /* MESSAGE_STATS_H */ 14 | -------------------------------------------------------------------------------- /vdo/num-utils.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | * 5 | * THIS FILE IS A CANDIDATE FOR THE EVENTUAL UTILITY LIBRARY. 6 | */ 7 | 8 | #ifndef NUM_UTILS_H 9 | #define NUM_UTILS_H 10 | 11 | #include "numeric.h" 12 | 13 | #include "types.h" 14 | 15 | #include 16 | #include 17 | 18 | #endif /* NUM_UTILS_H */ 19 | -------------------------------------------------------------------------------- /vdo/open-chapter.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef OPENCHAPTER_H 7 | #define OPENCHAPTER_H 1 8 | 9 | #include "chapter-index.h" 10 | #include "common.h" 11 | #include "geometry.h" 12 | #include "index.h" 13 | #include "volume.h" 14 | 15 | enum { 16 | OPEN_CHAPTER_RECORD_NUMBER_BITS = 23, 17 | OPEN_CHAPTER_MAX_RECORD_NUMBER = 18 | (1 << OPEN_CHAPTER_RECORD_NUMBER_BITS) - 1 19 | }; 20 | 21 | struct open_chapter_zone_slot { 22 | /* If non-zero, the record number addressed by this hash slot */ 23 | unsigned int record_number : OPEN_CHAPTER_RECORD_NUMBER_BITS; 24 | /* If true, the record at the index of this hash slot was deleted */ 25 | bool record_deleted : 1; 26 | } __packed; 27 | 28 | struct open_chapter_zone { 29 | /* The maximum number of records that can be stored */ 30 | unsigned int capacity; 31 | /* The number of records stored */ 32 | unsigned int size; 33 | /* The number of deleted records */ 34 | unsigned int deleted; 35 | /* Array of chunk records, 1-based */ 36 | struct uds_chunk_record *records; 37 | /* The number of slots in the hash table */ 38 | unsigned int slot_count; 39 | /* The hash table slots, referencing virtual record numbers */ 40 | struct open_chapter_zone_slot slots[]; 41 | }; 42 | 43 | int __must_check 44 | make_open_chapter(const struct geometry *geometry, 45 | unsigned int zone_count, 46 | struct open_chapter_zone **open_chapter_ptr); 47 | 48 | size_t __must_check 49 | open_chapter_size(const struct open_chapter_zone *open_chapter); 50 | 51 | void reset_open_chapter(struct open_chapter_zone *open_chapter); 52 | 53 | void search_open_chapter(struct open_chapter_zone *open_chapter, 54 | const struct uds_chunk_name *name, 55 | struct uds_chunk_data *metadata, 56 | bool *found); 57 | 58 | int __must_check put_open_chapter(struct open_chapter_zone *open_chapter, 59 | const struct uds_chunk_name *name, 60 | const struct uds_chunk_data *metadata, 61 | unsigned int *remaining); 62 | 63 | void remove_from_open_chapter(struct open_chapter_zone *open_chapter, 64 | const struct uds_chunk_name *name); 65 | 66 | void free_open_chapter(struct open_chapter_zone *open_chapter); 67 | 68 | int __must_check close_open_chapter(struct open_chapter_zone **chapter_zones, 69 | unsigned int zone_count, 70 | struct volume *volume, 71 | struct open_chapter_index *chapter_index, 72 | struct uds_chunk_record *collated_records, 73 | uint64_t virtual_chapter_number); 74 | 75 | int __must_check save_open_chapters(struct uds_index *index, 76 | struct buffered_writer *writer); 77 | 78 | int __must_check load_open_chapters(struct uds_index *index, 79 | struct buffered_reader *reader); 80 | 81 | uint64_t compute_saved_open_chapter_size(struct geometry *geometry); 82 | 83 | #endif /* OPENCHAPTER_H */ 84 | -------------------------------------------------------------------------------- /vdo/packed-reference-block.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef PACKED_REFERENCE_BLOCK_H 7 | #define PACKED_REFERENCE_BLOCK_H 8 | 9 | #include "constants.h" 10 | #include "journal-point.h" 11 | #include "types.h" 12 | 13 | /* 14 | * A type representing a reference count of a block. 15 | */ 16 | typedef uint8_t vdo_refcount_t; 17 | 18 | /* 19 | * Special vdo_refcount_t values. 20 | */ 21 | #define EMPTY_REFERENCE_COUNT 0 22 | enum { 23 | MAXIMUM_REFERENCE_COUNT = 254, 24 | PROVISIONAL_REFERENCE_COUNT = 255, 25 | }; 26 | 27 | enum { 28 | COUNTS_PER_SECTOR = 29 | ((VDO_SECTOR_SIZE - sizeof(struct packed_journal_point)) 30 | / sizeof(vdo_refcount_t)), 31 | COUNTS_PER_BLOCK = COUNTS_PER_SECTOR * VDO_SECTORS_PER_BLOCK, 32 | }; 33 | 34 | /* 35 | * The format of each sector of a reference_block on disk. 36 | */ 37 | struct packed_reference_sector { 38 | struct packed_journal_point commit_point; 39 | vdo_refcount_t counts[COUNTS_PER_SECTOR]; 40 | } __packed; 41 | 42 | struct packed_reference_block { 43 | struct packed_reference_sector sectors[VDO_SECTORS_PER_BLOCK]; 44 | }; 45 | 46 | #endif /* PACKED_REFERENCE_BLOCK_H */ 47 | -------------------------------------------------------------------------------- /vdo/packer.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef PACKER_H 7 | #define PACKER_H 8 | 9 | #include 10 | 11 | #include "compiler.h" 12 | 13 | #include "admin-state.h" 14 | #include "block-mapping-state.h" 15 | #include "statistics.h" 16 | #include "types.h" 17 | #include "wait-queue.h" 18 | 19 | enum { 20 | DEFAULT_PACKER_BINS = 16, 21 | }; 22 | 23 | /* 24 | * Each packer_bin holds an incomplete batch of data_vios that only partially 25 | * fill a compressed block. The bins are kept in a ring sorted by the amount of 26 | * unused space so the first bin with enough space to hold a newly-compressed 27 | * data_vio can easily be found. When the bin fills up or is flushed, the first 28 | * uncanceled data_vio in the bin is selected to be the agent for that 29 | * bin. Upon entering the packer, each data_vio already has its compressed data 30 | * in the first slot of the data_vio's compressed_block (overlaid on the 31 | * data_vio's scratch_block). So the agent's fragment is already in place. The 32 | * fragments for the other uncanceled data_vios in the bin are packed into the 33 | * agent's compressed block. The agent then writes out the compressed block. 34 | * If the write is successful, the agent shares its pbn lock which each of the 35 | * other data_vios in its compressed block and sends each on its way. Finally 36 | * the agent itself continues on the write path as before. 37 | * 38 | * There is one special bin which is used to hold data_vios which have been 39 | * canceled and removed from their bin by the packer. These data_vios need to 40 | * wait for the canceller to rendezvous with them (VDO-2809) and so they sit in 41 | * this special bin. 42 | */ 43 | struct packer_bin { 44 | /* List links for packer.packer_bins */ 45 | struct list_head list; 46 | /* The number of items in the bin */ 47 | slot_number_t slots_used; 48 | /* 49 | * The number of compressed block bytes remaining in the current batch 50 | */ 51 | size_t free_space; 52 | /* The current partial batch of data_vios, waiting for more */ 53 | struct data_vio *incoming[]; 54 | }; 55 | 56 | struct packer { 57 | /* The ID of the packer's callback thread */ 58 | thread_id_t thread_id; 59 | /* The number of bins */ 60 | block_count_t size; 61 | /* The block size minus header size */ 62 | size_t bin_data_size; 63 | /* The number of compression slots */ 64 | size_t max_slots; 65 | /* A list of all packer_bins, kept sorted by free_space */ 66 | struct list_head bins; 67 | /* 68 | * A bin to hold data_vios which were canceled out of the packer and 69 | * are waiting to rendezvous with the canceling data_vio. 70 | */ 71 | struct packer_bin *canceled_bin; 72 | 73 | /* The current flush generation */ 74 | sequence_number_t flush_generation; 75 | 76 | /* The administrative state of the packer */ 77 | struct admin_state state; 78 | 79 | /* 80 | * Statistics are only updated on the packer thread, but are 81 | * accessed from other threads. 82 | */ 83 | struct packer_statistics statistics; 84 | }; 85 | 86 | int __must_check vdo_make_packer(struct vdo *vdo, 87 | block_count_t bin_count, 88 | struct packer **packer_ptr); 89 | 90 | void vdo_free_packer(struct packer *packer); 91 | 92 | bool __must_check 93 | vdo_data_is_sufficiently_compressible(struct data_vio *data_vio); 94 | 95 | struct packer_statistics __must_check 96 | vdo_get_packer_statistics(const struct packer *packer); 97 | 98 | void vdo_attempt_packing(struct data_vio *data_vio); 99 | 100 | void vdo_flush_packer(struct packer *packer); 101 | 102 | void vdo_remove_lock_holder_from_packer(struct vdo_completion *completion); 103 | 104 | void vdo_increment_packer_flush_generation(struct packer *packer); 105 | 106 | void 107 | vdo_drain_packer(struct packer *packer, struct vdo_completion *completion); 108 | 109 | void vdo_resume_packer(struct packer *packer, struct vdo_completion *parent); 110 | 111 | void vdo_dump_packer(const struct packer *packer); 112 | 113 | #endif /* PACKER_H */ 114 | -------------------------------------------------------------------------------- /vdo/pbn-lock-pool.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef PBN_LOCK_POOL_H 7 | #define PBN_LOCK_POOL_H 8 | 9 | #include "pbn-lock.h" 10 | #include "types.h" 11 | 12 | struct pbn_lock_pool; 13 | 14 | int __must_check 15 | vdo_make_pbn_lock_pool(size_t capacity, struct pbn_lock_pool **pool_ptr); 16 | 17 | void vdo_free_pbn_lock_pool(struct pbn_lock_pool *pool); 18 | 19 | int __must_check 20 | vdo_borrow_pbn_lock_from_pool(struct pbn_lock_pool *pool, 21 | enum pbn_lock_type type, 22 | struct pbn_lock **lock_ptr); 23 | 24 | void vdo_return_pbn_lock_to_pool(struct pbn_lock_pool *pool, 25 | struct pbn_lock *lock); 26 | 27 | #endif /* PBN_LOCK_POOL_H */ 28 | -------------------------------------------------------------------------------- /vdo/pbn-lock.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef PBN_LOCK_H 7 | #define PBN_LOCK_H 8 | 9 | #include 10 | 11 | #include "kernel-types.h" 12 | #include "types.h" 13 | 14 | /* 15 | * The type of a PBN lock. 16 | */ 17 | enum pbn_lock_type { 18 | VIO_READ_LOCK, 19 | VIO_WRITE_LOCK, 20 | VIO_BLOCK_MAP_WRITE_LOCK, 21 | }; 22 | 23 | struct pbn_lock_implementation; 24 | 25 | /* 26 | * A PBN lock. 27 | */ 28 | struct pbn_lock { 29 | /* The implementation of the lock */ 30 | const struct pbn_lock_implementation *implementation; 31 | 32 | /* The number of VIOs holding or sharing this lock */ 33 | vio_count_t holder_count; 34 | /* 35 | * The number of compressed block writers holding a share of this lock 36 | * while they are acquiring a reference to the PBN. 37 | */ 38 | uint8_t fragment_locks; 39 | 40 | /* 41 | * Whether the locked PBN has been provisionally referenced on behalf of 42 | * the lock holder. 43 | */ 44 | bool has_provisional_reference; 45 | 46 | /* 47 | * For read locks, the number of references that were known to be 48 | * available on the locked block at the time the lock was acquired. 49 | */ 50 | uint8_t increment_limit; 51 | 52 | /* 53 | * For read locks, the number of data_vios that have tried to claim one 54 | * of the available increments during the lifetime of the lock. Each 55 | * claim will first increment this counter, so it can exceed the 56 | * increment limit. 57 | */ 58 | atomic_t increments_claimed; 59 | }; 60 | 61 | void vdo_initialize_pbn_lock(struct pbn_lock *lock, enum pbn_lock_type type); 62 | 63 | bool __must_check vdo_is_pbn_read_lock(const struct pbn_lock *lock); 64 | 65 | void 66 | vdo_downgrade_pbn_write_lock(struct pbn_lock *lock, bool compressed_write); 67 | 68 | bool __must_check vdo_claim_pbn_lock_increment(struct pbn_lock *lock); 69 | 70 | /** 71 | * vdo_pbn_lock_has_provisional_reference() - Check whether a PBN lock 72 | * has a provisional reference. 73 | * @lock: The PBN lock. 74 | */ 75 | static inline bool 76 | vdo_pbn_lock_has_provisional_reference(struct pbn_lock *lock) 77 | { 78 | return ((lock != NULL) && lock->has_provisional_reference); 79 | } 80 | 81 | void vdo_assign_pbn_lock_provisional_reference(struct pbn_lock *lock); 82 | 83 | void vdo_unassign_pbn_lock_provisional_reference(struct pbn_lock *lock); 84 | 85 | void 86 | vdo_release_pbn_lock_provisional_reference(struct pbn_lock *lock, 87 | physical_block_number_t locked_pbn, 88 | struct block_allocator *allocator); 89 | 90 | #endif /* PBN_LOCK_H */ 91 | -------------------------------------------------------------------------------- /vdo/permassert.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #include "permassert.h" 7 | 8 | #include "errors.h" 9 | #include "logger.h" 10 | 11 | 12 | int uds_assertion_failed(const char *expression_string, 13 | int code, 14 | const char *module_name, 15 | const char *file_name, 16 | int line_number, 17 | const char *format, 18 | ...) 19 | { 20 | va_list args; 21 | 22 | va_start(args, format); 23 | 24 | uds_log_embedded_message(UDS_LOG_ERR, 25 | module_name, 26 | "assertion \"", 27 | format, 28 | args, 29 | "\" (%s) failed at %s:%d", 30 | expression_string, 31 | file_name, 32 | line_number); 33 | uds_log_backtrace(UDS_LOG_ERR); 34 | 35 | 36 | va_end(args); 37 | 38 | return code; 39 | } 40 | -------------------------------------------------------------------------------- /vdo/physical-zone.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef PHYSICAL_ZONE_H 7 | #define PHYSICAL_ZONE_H 8 | 9 | #include "kernel-types.h" 10 | #include "pbn-lock.h" 11 | #include "types.h" 12 | 13 | struct physical_zone { 14 | /* Which physical zone this is */ 15 | zone_count_t zone_number; 16 | /* The thread ID for this zone */ 17 | thread_id_t thread_id; 18 | /* In progress operations keyed by PBN */ 19 | struct int_map *pbn_operations; 20 | /* Pool of unused pbn_lock instances */ 21 | struct pbn_lock_pool *lock_pool; 22 | /* The block allocator for this zone */ 23 | struct block_allocator *allocator; 24 | /* The next zone from which to attempt an allocation */ 25 | struct physical_zone *next; 26 | }; 27 | 28 | struct physical_zones { 29 | /* The number of zones */ 30 | zone_count_t zone_count; 31 | /* The physical zones themselves */ 32 | struct physical_zone zones[]; 33 | }; 34 | 35 | int __must_check 36 | vdo_make_physical_zones(struct vdo *vdo, struct physical_zones **zones_ptr); 37 | 38 | void vdo_free_physical_zones(struct physical_zones *zones); 39 | 40 | struct pbn_lock * __must_check 41 | vdo_get_physical_zone_pbn_lock(struct physical_zone *zone, 42 | physical_block_number_t pbn); 43 | 44 | int __must_check 45 | vdo_attempt_physical_zone_pbn_lock(struct physical_zone *zone, 46 | physical_block_number_t pbn, 47 | enum pbn_lock_type type, 48 | struct pbn_lock **lock_ptr); 49 | 50 | bool __must_check vdo_allocate_block_in_zone(struct data_vio *data_vio); 51 | 52 | void vdo_release_physical_zone_pbn_lock(struct physical_zone *zone, 53 | physical_block_number_t locked_pbn, 54 | struct pbn_lock *lock); 55 | 56 | void vdo_dump_physical_zone(const struct physical_zone *zone); 57 | 58 | 59 | #endif /* PHYSICAL_ZONE_H */ 60 | -------------------------------------------------------------------------------- /vdo/pointer-map.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef POINTER_MAP_H 7 | #define POINTER_MAP_H 8 | 9 | #include "compiler.h" 10 | #include "type-defs.h" 11 | 12 | /* 13 | * A pointer_map associates pointer values (void *) with the data 14 | * referenced by pointer keys (void *). NULL pointer 15 | * values are not supported. A NULL key value is supported when 16 | * the instance's key comparator and hasher functions support it. 17 | * 18 | * The map is implemented as hash table, which should provide constant-time 19 | * insert, query, and remove operations, although the insert may occasionally 20 | * grow the table, which is linear in the number of entries in the map. The 21 | * table will grow as needed to hold new entries, but will not shrink as 22 | * entries are removed. 23 | * 24 | * The key and value pointers passed to the map are retained and used by the 25 | * map, but are not owned by the map. Freeing the map does not attempt to free 26 | * the pointers. The client is entirely responsible for the memory managment 27 | * of the keys and values. The current interface and implementation assume 28 | * that keys will be properties of the values, or that keys will not be memory 29 | * managed, or that keys will not need to be freed as a result of being 30 | * replaced when a key is re-mapped. 31 | */ 32 | 33 | struct pointer_map; 34 | 35 | /** 36 | * typedef pointer_key_comparator - The prototype of functions that compare 37 | * the referents of two pointer keys for 38 | * equality. 39 | * @this_key: The first element to compare. 40 | * @that_key: The second element to compare. 41 | * 42 | * If two keys are equal, then both keys must have the same the hash code 43 | * associated with them by the hasher function defined below. 44 | * 45 | * Return: true if and only if the referents of the two key pointers are to be 46 | * treated as the same key by the map. 47 | */ 48 | typedef bool pointer_key_comparator(const void *this_key, const void *that_key); 49 | 50 | /** 51 | * typedef pointer_key_hasher - The prototype of functions that get or 52 | * calculate a hash code associated with the 53 | * referent of pointer key. 54 | * @key: The pointer key to hash. 55 | * 56 | * The hash code must be uniformly distributed over all uint32_t values. The 57 | * hash code associated with a given key must not change while the key is in 58 | * the map. If the comparator function says two keys are equal, then this 59 | * function must return the same hash code for both keys. This function may be 60 | * called many times for a key while an entry is stored for it in the map. 61 | * 62 | * Return: The hash code for the key. 63 | */ 64 | typedef uint32_t pointer_key_hasher(const void *key); 65 | 66 | int __must_check make_pointer_map(size_t initial_capacity, 67 | unsigned int initial_load, 68 | pointer_key_comparator comparator, 69 | pointer_key_hasher hasher, 70 | struct pointer_map **map_ptr); 71 | 72 | void free_pointer_map(struct pointer_map *map); 73 | 74 | size_t pointer_map_size(const struct pointer_map *map); 75 | 76 | void *pointer_map_get(struct pointer_map *map, const void *key); 77 | 78 | int __must_check pointer_map_put(struct pointer_map *map, 79 | const void *key, 80 | void *new_value, 81 | bool update, 82 | void **old_value_ptr); 83 | 84 | void *pointer_map_remove(struct pointer_map *map, const void *key); 85 | 86 | #endif /* POINTER_MAP_H */ 87 | -------------------------------------------------------------------------------- /vdo/pool-sysfs.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef POOL_SYSFS_H 7 | #define POOL_SYSFS_H 8 | 9 | #include 10 | 11 | /* The kobj_type used for setting up the kernel layer kobject. */ 12 | extern struct kobj_type vdo_directory_type; 13 | 14 | /* The sysfs_ops used for the "statistics" subdirectory. */ 15 | extern struct sysfs_ops vdo_pool_stats_sysfs_ops; 16 | /* The attribute used for the "statistics" subdirectory. */ 17 | extern struct attribute *vdo_pool_stats_attrs[]; 18 | 19 | #endif /* POOL_SYSFS_H */ 20 | -------------------------------------------------------------------------------- /vdo/priority-table.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef PRIORITY_TABLE_H 7 | #define PRIORITY_TABLE_H 8 | 9 | #include 10 | 11 | /** 12 | * A priority_table is a simple implementation of a priority queue for entries 13 | * with priorities that are small non-negative integer values. It implements 14 | * the obvious priority queue operations of enqueuing an entry and dequeuing 15 | * an entry with the maximum priority. It also supports removing an arbitrary 16 | * entry. The priority of an entry already in the table can be changed by 17 | * removing it and re-enqueuing it with a different priority. All operations 18 | * have O(1) complexity. 19 | * 20 | * The links for the table entries must be embedded in the entries themselves. 21 | * Lists are used to link entries in the table and no wrapper type is 22 | * declared, so an existing list entry in an object can also be used to 23 | * queue it in a priority_table, assuming the field is not used for anything 24 | * else while so queued. 25 | * 26 | * The table is implemented as an array of queues (circular lists) indexed by 27 | * priority, along with a hint for which queues are non-empty. Steven Skiena 28 | * calls a very similar structure a "bounded height priority queue", but given 29 | * the resemblance to a hash table, "priority table" seems both shorter and 30 | * more apt, if somewhat novel. 31 | **/ 32 | 33 | struct priority_table; 34 | 35 | int __must_check make_priority_table(unsigned int max_priority, 36 | struct priority_table **table_ptr); 37 | 38 | void free_priority_table(struct priority_table *table); 39 | 40 | void priority_table_enqueue(struct priority_table *table, unsigned int priority, 41 | struct list_head *entry); 42 | 43 | void reset_priority_table(struct priority_table *table); 44 | 45 | struct list_head * __must_check 46 | priority_table_dequeue(struct priority_table *table); 47 | 48 | void priority_table_remove(struct priority_table *table, 49 | struct list_head *entry); 50 | 51 | bool __must_check is_priority_table_empty(struct priority_table *table); 52 | 53 | #endif /* PRIORITY_TABLE_H */ 54 | -------------------------------------------------------------------------------- /vdo/radix-sort.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef RADIX_SORT_H 7 | #define RADIX_SORT_H 8 | 9 | #include "compiler.h" 10 | 11 | /* 12 | * The implementation uses one large object allocated on the heap. This 13 | * large object can be reused as many times as desired. There is no 14 | * further heap usage by the sorting. 15 | */ 16 | struct radix_sorter; 17 | 18 | /** 19 | * Reserve the heap storage needed by the radix_sort routine. The amount of 20 | * heap space is logarithmically proportional to the number of keys. 21 | * 22 | * @param count The maximum number of keys to be sorted 23 | * @param sorter The struct radix_sorter object is returned here 24 | * 25 | * @return UDS_SUCCESS or an error code 26 | **/ 27 | int __must_check 28 | make_radix_sorter(unsigned int count, struct radix_sorter **sorter); 29 | 30 | /** 31 | * Free the heap storage needed by the radix_sort routine. 32 | * 33 | * @param sorter The struct radix_sorter object to free 34 | **/ 35 | void free_radix_sorter(struct radix_sorter *sorter); 36 | 37 | /** 38 | * Sort pointers to fixed-length keys (arrays of bytes) using a radix sort. 39 | * 40 | * The sort implementation is unstable--relative ordering of equal keys is not 41 | * preserved. The implementation does not use any heap allocation. 42 | * 43 | * @param [in] sorter the heap storage used by the sorting 44 | * @param keys the array of key pointers to sort (modified in place) 45 | * @param [in] count the number of keys 46 | * @param [in] length the length of every key, in bytes 47 | * 48 | * @return UDS_SUCCESS or an error code 49 | **/ 50 | int __must_check radix_sort(struct radix_sorter *sorter, 51 | const unsigned char *keys[], 52 | unsigned int count, 53 | unsigned short length); 54 | 55 | #endif /* RADIX_SORT_H */ 56 | -------------------------------------------------------------------------------- /vdo/random.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #include "random.h" 7 | 8 | #include "permassert.h" 9 | 10 | unsigned int random_in_range(unsigned int lo, unsigned int hi) 11 | { 12 | return lo + random() % (hi - lo + 1); 13 | } 14 | 15 | void random_compile_time_assertions(void) 16 | { 17 | STATIC_ASSERT((((uint64_t) RAND_MAX + 1) & RAND_MAX) == 0); 18 | } 19 | 20 | -------------------------------------------------------------------------------- /vdo/random.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef RANDOM_H 7 | #define RANDOM_H 8 | 9 | #include 10 | 11 | #include "compiler.h" 12 | #include "type-defs.h" 13 | 14 | /** 15 | * Get random unsigned integer in a given range 16 | * 17 | * @param lo Minimum unsigned integer value 18 | * @param hi Maximum unsigned integer value 19 | * 20 | * @return unsigned integer in the interval [lo,hi] 21 | **/ 22 | unsigned int random_in_range(unsigned int lo, unsigned int hi); 23 | 24 | /** 25 | * Special function wrapper required for compile-time assertions. This 26 | * function will fail to compile if RAND_MAX is not of the form 2^n - 1. 27 | **/ 28 | void random_compile_time_assertions(void); 29 | 30 | /** 31 | * Fill bytes with random data. 32 | * 33 | * @param ptr where to store bytes 34 | * @param len number of bytes to write 35 | **/ 36 | static INLINE void fill_randomly(void *ptr, size_t len) 37 | { 38 | get_random_bytes(ptr, len); 39 | } 40 | 41 | #define RAND_MAX 2147483647 42 | 43 | /** 44 | * Random number generator 45 | * 46 | * @return a random number in the rand 0 to RAND_MAX 47 | **/ 48 | static INLINE long random(void) 49 | { 50 | long value; 51 | fill_randomly(&value, sizeof(value)); 52 | return value & RAND_MAX; 53 | } 54 | 55 | #endif /* RANDOM_H */ 56 | -------------------------------------------------------------------------------- /vdo/read-only-notifier.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | /* 7 | * A read_only_notifier is responsible for propogating the fact that the VDO 8 | * has encountered an unrecoverable error to all base threads. It also persists 9 | * the read-only state to the super block. 10 | * 11 | * The notifier also provides the ability to wait for any notifications to be 12 | * complete in order to not cause super block write races when shutting down 13 | * the VDO. 14 | */ 15 | 16 | #ifndef READ_ONLY_NOTIFIER_H 17 | #define READ_ONLY_NOTIFIER_H 18 | 19 | #include "completion.h" 20 | 21 | /** 22 | * typedef vdo_read_only_notification - A function to notify a listener that 23 | * the VDO has gone read-only. 24 | * @listener: The object to notify. 25 | * @parent: The completion to notify in order to acknowledge the notification. 26 | */ 27 | typedef void vdo_read_only_notification(void *listener, 28 | struct vdo_completion *parent); 29 | 30 | int __must_check 31 | vdo_make_read_only_notifier(bool is_read_only, 32 | const struct thread_config *thread_config, 33 | struct vdo *vdo, 34 | struct read_only_notifier **notifier_ptr); 35 | 36 | void vdo_free_read_only_notifier(struct read_only_notifier *notifier); 37 | 38 | void 39 | vdo_wait_until_not_entering_read_only_mode(struct read_only_notifier *notifier, 40 | struct vdo_completion *parent); 41 | 42 | void vdo_allow_read_only_mode_entry(struct read_only_notifier *notifier, 43 | struct vdo_completion *parent); 44 | 45 | void vdo_enter_read_only_mode(struct read_only_notifier *notifier, 46 | int error_code); 47 | 48 | bool __must_check vdo_is_read_only(struct read_only_notifier *notifier); 49 | 50 | bool __must_check 51 | vdo_is_or_will_be_read_only(struct read_only_notifier *notifier); 52 | 53 | int vdo_register_read_only_listener(struct read_only_notifier *notifier, 54 | void *listener, 55 | vdo_read_only_notification *notification, 56 | thread_id_t thread_id); 57 | 58 | #endif /* READ_ONLY_NOTIFIER_H */ 59 | -------------------------------------------------------------------------------- /vdo/read-only-rebuild.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef READ_ONLY_REBUILD_H 7 | #define READ_ONLY_REBUILD_H 8 | 9 | #include "completion.h" 10 | #include "vdo.h" 11 | 12 | void vdo_launch_rebuild(struct vdo *vdo, struct vdo_completion *parent); 13 | 14 | #endif /* READ_ONLY_REBUILD_H */ 15 | -------------------------------------------------------------------------------- /vdo/record-page.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #include "record-page.h" 7 | 8 | #include "permassert.h" 9 | 10 | static unsigned int 11 | encode_tree(byte record_page[], 12 | const struct uds_chunk_record *sorted_pointers[], 13 | unsigned int next_record, 14 | unsigned int node, 15 | unsigned int node_count) 16 | { 17 | if (node < node_count) { 18 | unsigned int child = (2 * node) + 1; 19 | 20 | next_record = encode_tree(record_page, 21 | sorted_pointers, 22 | next_record, 23 | child, 24 | node_count); 25 | 26 | /* 27 | * In-order traversal: copy the contents of the next record 28 | * into the page at the node offset. 29 | */ 30 | memcpy(&record_page[node * BYTES_PER_RECORD], 31 | sorted_pointers[next_record], 32 | BYTES_PER_RECORD); 33 | ++next_record; 34 | 35 | next_record = encode_tree(record_page, 36 | sorted_pointers, 37 | next_record, 38 | child + 1, 39 | node_count); 40 | } 41 | return next_record; 42 | } 43 | 44 | int encode_record_page(const struct volume *volume, 45 | const struct uds_chunk_record records[], 46 | byte record_page[]) 47 | { 48 | int result; 49 | unsigned int records_per_page = volume->geometry->records_per_page; 50 | const struct uds_chunk_record **record_pointers = 51 | volume->record_pointers; 52 | 53 | /* 54 | * Build an array of record pointers. We'll sort the pointers by the 55 | * block names in the records, which is less work than sorting the 56 | * record values. 57 | */ 58 | unsigned int i; 59 | 60 | for (i = 0; i < records_per_page; i++) { 61 | record_pointers[i] = &records[i]; 62 | } 63 | 64 | STATIC_ASSERT(offsetof(struct uds_chunk_record, name) == 0); 65 | result = radix_sort(volume->radix_sorter, 66 | (const byte **) record_pointers, 67 | records_per_page, 68 | UDS_CHUNK_NAME_SIZE); 69 | if (result != UDS_SUCCESS) { 70 | return result; 71 | } 72 | 73 | /* 74 | * Use the sorted pointers to copy the records from the chapter to the 75 | * record page in tree order. 76 | */ 77 | encode_tree(record_page, record_pointers, 0, 0, records_per_page); 78 | return UDS_SUCCESS; 79 | } 80 | 81 | bool search_record_page(const byte record_page[], 82 | const struct uds_chunk_name *name, 83 | const struct geometry *geometry, 84 | struct uds_chunk_data *metadata) 85 | { 86 | /* The record page is just an array of chunk records. */ 87 | const struct uds_chunk_record *records = 88 | (const struct uds_chunk_record *) record_page; 89 | 90 | /* 91 | * The array of records is sorted by name and stored as a binary tree 92 | * in heap order, so the root of the tree is the first array element. 93 | */ 94 | unsigned int node = 0; 95 | 96 | while (node < geometry->records_per_page) { 97 | const struct uds_chunk_record *record = &records[node]; 98 | int result = memcmp(name, &record->name, UDS_CHUNK_NAME_SIZE); 99 | 100 | if (result == 0) { 101 | if (metadata != NULL) { 102 | *metadata = record->data; 103 | } 104 | return true; 105 | } 106 | /* 107 | * The children of node N are in the heap at indexes 2N+1 and 108 | * 2N+2. 109 | */ 110 | node = ((2 * node) + ((result < 0) ? 1 : 2)); 111 | } 112 | return false; 113 | } 114 | -------------------------------------------------------------------------------- /vdo/record-page.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef RECORDPAGE_H 7 | #define RECORDPAGE_H 1 8 | 9 | #include "common.h" 10 | #include "volume.h" 11 | 12 | /** 13 | * Generate the on-disk encoding of a record page from the list of records 14 | * in the open chapter representation. 15 | * 16 | * @param volume The volume 17 | * @param records The records to be encoded 18 | * @param record_page The record page 19 | * 20 | * @return UDS_SUCCESS or an error code 21 | **/ 22 | int encode_record_page(const struct volume *volume, 23 | const struct uds_chunk_record records[], 24 | byte record_page[]); 25 | 26 | /** 27 | * Find the metadata for a given block name in this page. 28 | * 29 | * @param record_page The record page 30 | * @param name The block name to look for 31 | * @param geometry The geometry of the volume 32 | * @param metadata an array in which to place the metadata of the 33 | * record, if one was found 34 | * 35 | * @return true if the record was found 36 | **/ 37 | bool search_record_page(const byte record_page[], 38 | const struct uds_chunk_name *name, 39 | const struct geometry *geometry, 40 | struct uds_chunk_data *metadata); 41 | 42 | #endif /* RECORDPAGE_H */ 43 | -------------------------------------------------------------------------------- /vdo/recovery-journal-entry.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef RECOVERY_JOURNAL_ENTRY_H 7 | #define RECOVERY_JOURNAL_ENTRY_H 8 | 9 | #include "numeric.h" 10 | 11 | #include "block-map-entry.h" 12 | #include "journal-point.h" 13 | #include "types.h" 14 | 15 | /* 16 | * A recovery journal entry stores two physical locations: a data location 17 | * that is the value of a single mapping in the block map tree, and the 18 | * location of the block map page and slot that is either acquiring or 19 | * releasing a reference to the data location. The journal entry also stores 20 | * an operation code that says whether the reference is being acquired (an 21 | * increment) or released (a decrement), and whether the mapping is for a 22 | * logical block or for the block map tree itself. 23 | */ 24 | struct recovery_journal_entry { 25 | struct block_map_slot slot; 26 | struct data_location mapping; 27 | enum journal_operation operation; 28 | }; 29 | 30 | /* The packed, on-disk representation of a recovery journal entry. */ 31 | struct packed_recovery_journal_entry { 32 | /* 33 | * In little-endian bit order: 34 | * Bits 15..12: The four highest bits of the 36-bit physical 35 | * block number of the block map tree page Bits 11..2: The 36 | * 10-bit block map page slot number Bits 1..0: The 2-bit 37 | * journal_operation of the entry 38 | */ 39 | #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 40 | unsigned operation : 2; 41 | unsigned slot_low : 6; 42 | unsigned slot_high : 4; 43 | unsigned pbn_high_nibble : 4; 44 | #else 45 | unsigned slot_low : 6; 46 | unsigned operation : 2; 47 | unsigned pbn_high_nibble : 4; 48 | unsigned slot_high : 4; 49 | #endif 50 | 51 | /* 52 | * Bits 47..16: The 32 low-order bits of the block map page 53 | * PBN, in little-endian byte order 54 | */ 55 | __le32 pbn_low_word; 56 | 57 | /* 58 | * Bits 87..48: The five-byte block map entry encoding the 59 | * location that was or will be stored in the block map page slot 60 | */ 61 | struct block_map_entry block_map_entry; 62 | } __packed; 63 | 64 | /** 65 | * vdo_pack_recovery_journal_entry() - Return the packed, on-disk 66 | * representation of a recovery journal 67 | * entry. 68 | * @entry: The journal entry to pack. 69 | * 70 | * Return: The packed representation of the journal entry. 71 | */ 72 | static inline struct packed_recovery_journal_entry 73 | vdo_pack_recovery_journal_entry(const struct recovery_journal_entry *entry) 74 | { 75 | return (struct packed_recovery_journal_entry) { 76 | .operation = entry->operation, 77 | .slot_low = entry->slot.slot & 0x3F, 78 | .slot_high = (entry->slot.slot >> 6) & 0x0F, 79 | .pbn_high_nibble = (entry->slot.pbn >> 32) & 0x0F, 80 | .pbn_low_word = __cpu_to_le32(entry->slot.pbn & UINT_MAX), 81 | .block_map_entry = vdo_pack_pbn(entry->mapping.pbn, 82 | entry->mapping.state), 83 | }; 84 | } 85 | 86 | /** 87 | * vdo_unpack_recovery_journal_entry() - Unpack the on-disk representation of 88 | * a recovery journal entry. 89 | * @entry: The recovery journal entry to unpack. 90 | * 91 | * Return: The unpacked entry. 92 | */ 93 | static inline struct recovery_journal_entry 94 | vdo_unpack_recovery_journal_entry(const struct packed_recovery_journal_entry *entry) 95 | { 96 | physical_block_number_t low32 = __le32_to_cpu(entry->pbn_low_word); 97 | physical_block_number_t high4 = entry->pbn_high_nibble; 98 | 99 | return (struct recovery_journal_entry) { 100 | .operation = entry->operation, 101 | .slot = { 102 | .pbn = ((high4 << 32) | low32), 103 | .slot = (entry->slot_low 104 | | (entry->slot_high << 6)), 105 | }, 106 | .mapping = vdo_unpack_block_map_entry(&entry->block_map_entry), 107 | }; 108 | } 109 | 110 | #endif /* RECOVERY_JOURNAL_ENTRY_H */ 111 | -------------------------------------------------------------------------------- /vdo/recovery-journal-format.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef RECOVERY_JOURNAL_FORMAT_H 7 | #define RECOVERY_JOURNAL_FORMAT_H 8 | 9 | #include "buffer.h" 10 | 11 | #include "header.h" 12 | #include "packed-recovery-journal-block.h" 13 | #include "types.h" 14 | 15 | /* 16 | * The state of the recovery journal as encoded in the VDO super block. 17 | */ 18 | struct recovery_journal_state_7_0 { 19 | /** Sequence number to start the journal */ 20 | sequence_number_t journal_start; 21 | /** Number of logical blocks used by VDO */ 22 | block_count_t logical_blocks_used; 23 | /** Number of block map pages allocated */ 24 | block_count_t block_map_data_blocks; 25 | } __packed; 26 | 27 | extern const struct header VDO_RECOVERY_JOURNAL_HEADER_7_0; 28 | 29 | size_t __must_check vdo_get_recovery_journal_encoded_size(void); 30 | 31 | int __must_check 32 | vdo_encode_recovery_journal_state_7_0(struct recovery_journal_state_7_0 state, 33 | struct buffer *buffer); 34 | 35 | int __must_check 36 | vdo_decode_recovery_journal_state_7_0(struct buffer *buffer, 37 | struct recovery_journal_state_7_0 *state); 38 | 39 | const char * __must_check 40 | vdo_get_journal_operation_name(enum journal_operation operation); 41 | 42 | /** 43 | * vdo_is_valid_recovery_journal_sector() - Determine whether the header of 44 | * the given sector could describe a 45 | * valid sector for the given journal 46 | * block header. 47 | * @header: The unpacked block header to compare against. 48 | * @sector: The packed sector to check. 49 | * 50 | * Return: true if the sector matches the block header. 51 | */ 52 | static inline bool __must_check 53 | vdo_is_valid_recovery_journal_sector(const struct recovery_block_header *header, 54 | const struct packed_journal_sector *sector) 55 | { 56 | return ((header->check_byte == sector->check_byte) 57 | && (header->recovery_count == sector->recovery_count)); 58 | } 59 | 60 | /** 61 | * vdo_compute_recovery_journal_block_number() - Compute the physical block 62 | * number of the recovery 63 | * journal block which would 64 | * have a given sequence number. 65 | * @journal_size: The size of the journal. 66 | * @sequence_number: The sequence number. 67 | * 68 | * Return: The pbn of the journal block which would the specified sequence 69 | * number. 70 | */ 71 | static inline physical_block_number_t __must_check 72 | vdo_compute_recovery_journal_block_number(block_count_t journal_size, 73 | sequence_number_t sequence_number) 74 | { 75 | /* 76 | * Since journal size is a power of two, the block number modulus can 77 | * just be extracted from the low-order bits of the sequence. 78 | */ 79 | return (sequence_number & (journal_size - 1)); 80 | } 81 | 82 | #endif /* RECOVERY_JOURNAL_FORMAT_H */ 83 | -------------------------------------------------------------------------------- /vdo/recovery-utils.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef RECOVERY_UTILS_H 7 | #define RECOVERY_UTILS_H 8 | 9 | #include "constants.h" 10 | #include "packed-recovery-journal-block.h" 11 | #include "recovery-journal-entry.h" 12 | #include "recovery-journal.h" 13 | #include "types.h" 14 | 15 | /** 16 | * vdo_get_recovery_journal_block_header() - Get the block header for a block 17 | * at a position in the journal 18 | * data. 19 | * @journal: The recovery journal. 20 | * @journal_data: The recovery journal data. 21 | * @sequence: The sequence number. 22 | * 23 | * Return: A pointer to a packed recovery journal block header. 24 | */ 25 | static inline struct packed_journal_header * __must_check 26 | vdo_get_recovery_journal_block_header(struct recovery_journal *journal, 27 | char *journal_data, 28 | sequence_number_t sequence) 29 | { 30 | off_t block_offset = 31 | (vdo_get_recovery_journal_block_number(journal, sequence) 32 | * VDO_BLOCK_SIZE); 33 | return (struct packed_journal_header *) &journal_data[block_offset]; 34 | } 35 | 36 | /** 37 | * vdo_is_valid_recovery_journal_block() - Determine whether the given header 38 | * describes a valid block for the 39 | * given journal. 40 | * @journal: The journal to use. 41 | * @header: The unpacked block header to check. 42 | * 43 | * A block is not valid if it is unformatted, or if it is older than the last 44 | * successful recovery or reformat. 45 | * 46 | * Return: True if the header is valid. 47 | */ 48 | static inline bool __must_check 49 | vdo_is_valid_recovery_journal_block(const struct recovery_journal *journal, 50 | const struct recovery_block_header *header) 51 | { 52 | return ((header->metadata_type == VDO_METADATA_RECOVERY_JOURNAL) 53 | && (header->nonce == journal->nonce) 54 | && (header->recovery_count == journal->recovery_count)); 55 | } 56 | 57 | /** 58 | * vdo_is_exact_recovery_journal_block() - Determine whether the given header 59 | * describes the exact block indicated. 60 | * @journal: The journal to use. 61 | * @header: The unpacked block header to check. 62 | * @sequence: The expected sequence number. 63 | * 64 | * Return: True if the block matches. 65 | */ 66 | static inline bool __must_check 67 | vdo_is_exact_recovery_journal_block(const struct recovery_journal *journal, 68 | const struct recovery_block_header *header, 69 | sequence_number_t sequence) 70 | { 71 | return ((header->sequence_number == sequence) 72 | && vdo_is_valid_recovery_journal_block(journal, header)); 73 | } 74 | 75 | void vdo_load_recovery_journal(struct recovery_journal *journal, 76 | struct vdo_completion *parent, 77 | char **journal_data_ptr); 78 | 79 | bool 80 | vdo_find_recovery_journal_head_and_tail(struct recovery_journal *journal, 81 | char *journal_data, 82 | sequence_number_t *tail_ptr, 83 | sequence_number_t *block_map_head_ptr, 84 | sequence_number_t *slab_journal_head_ptr); 85 | 86 | int __must_check 87 | vdo_validate_recovery_journal_entry(const struct vdo *vdo, 88 | const struct recovery_journal_entry *entry); 89 | 90 | #endif /* RECOVERY_UTILS_H */ 91 | -------------------------------------------------------------------------------- /vdo/reference-count-rebuild.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef REFERENCE_COUNT_REBUILD_H 7 | #define REFERENCE_COUNT_REBUILD_H 8 | 9 | #include "kernel-types.h" 10 | #include "types.h" 11 | 12 | void vdo_rebuild_reference_counts(struct vdo *vdo, 13 | struct vdo_completion *parent, 14 | block_count_t *logical_blocks_used, 15 | block_count_t *block_map_data_blocks); 16 | 17 | #endif /* REFERENCE_COUNT_REBUILD_H */ 18 | -------------------------------------------------------------------------------- /vdo/reference-operation.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #include "reference-operation.h" 7 | 8 | #include "physical-zone.h" 9 | #include "types.h" 10 | 11 | static struct pbn_lock *return_pbn_lock(struct reference_operation operation) 12 | { 13 | return (struct pbn_lock *) operation.context; 14 | } 15 | 16 | /** 17 | * vdo_set_up_reference_operation_with_lock() - Set up a reference_operation 18 | * for which we already have the 19 | * lock. 20 | * @type: The type of operation. 21 | * @pbn: The PBN of the block on which to operate. 22 | * @state: The mapping state of the block on which to operate. 23 | * @lock: The pbn_lock to associate with the operation. 24 | * @operation: The reference_operation to set up. 25 | */ 26 | void 27 | vdo_set_up_reference_operation_with_lock(enum journal_operation type, 28 | physical_block_number_t pbn, 29 | enum block_mapping_state state, 30 | struct pbn_lock *lock, 31 | struct reference_operation *operation) 32 | { 33 | *operation = (struct reference_operation) { 34 | .type = type, 35 | .pbn = pbn, 36 | .state = state, 37 | .lock_getter = return_pbn_lock, 38 | .context = lock, 39 | }; 40 | } 41 | 42 | static struct pbn_lock *look_up_pbn_lock(struct reference_operation operation) 43 | { 44 | return ((operation.context == NULL) 45 | ? NULL 46 | : vdo_get_physical_zone_pbn_lock(operation.context, 47 | operation.pbn)); 48 | } 49 | 50 | /** 51 | * vdo_set_up_reference_operation_with_zone() - Set up a reference_operation 52 | * for which we will need to look 53 | * up the lock later. 54 | * @type: The type of operation. 55 | * @pbn: The PBN of the block on which to operate. 56 | * @state: The mapping state of the block on which to operate. 57 | * @zone: The physical_zone from which the pbn_lock can be retrieved when 58 | * needed. 59 | * @operation: The reference_operation to set up. 60 | */ 61 | void 62 | vdo_set_up_reference_operation_with_zone(enum journal_operation type, 63 | physical_block_number_t pbn, 64 | enum block_mapping_state state, 65 | struct physical_zone *zone, 66 | struct reference_operation *operation) 67 | { 68 | *operation = (struct reference_operation) { 69 | .type = type, 70 | .pbn = pbn, 71 | .state = state, 72 | .lock_getter = look_up_pbn_lock, 73 | .context = zone, 74 | }; 75 | } 76 | -------------------------------------------------------------------------------- /vdo/reference-operation.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef REFERENCE_OPERATION_H 7 | #define REFERENCE_OPERATION_H 8 | 9 | #include "kernel-types.h" 10 | #include "types.h" 11 | 12 | struct reference_operation; 13 | 14 | /** 15 | * typedef pbn_lock_getter - Get the pbn_lock associated with a 16 | * reference_operation. 17 | * @operation: The reference_operation. 18 | * 19 | * Return: The pbn_lock on the block of a reference_operation or NULL if there 20 | * isn't one. 21 | */ 22 | typedef struct pbn_lock *pbn_lock_getter(struct reference_operation operation); 23 | 24 | /* 25 | * The current operation on a physical block (from the point of view of the 26 | * data_vio doing the operation) 27 | */ 28 | struct reference_operation { 29 | /* The operation being performed */ 30 | enum journal_operation type; 31 | /* The PBN of the block being operated on */ 32 | physical_block_number_t pbn; 33 | /* The mapping state of the block being operated on */ 34 | enum block_mapping_state state; 35 | /* 36 | * A function to use to get any pbn_lock associated with this operation 37 | */ 38 | pbn_lock_getter *lock_getter; 39 | /* The context to pass to the pbn_lock_getter */ 40 | void *context; 41 | }; 42 | 43 | /** 44 | * vdo_get_reference_operation_pbn_lock() - Get the pbn_lock associated with 45 | * the current reference_operation. 46 | * @operation: The reference operation. 47 | * 48 | * Return: The pbn_lock on the block of the current operation or NULL if there 49 | * isn't one. 50 | */ 51 | static inline struct pbn_lock * __must_check 52 | vdo_get_reference_operation_pbn_lock(struct reference_operation operation) 53 | { 54 | return ((operation.lock_getter == NULL) 55 | ? NULL 56 | : operation.lock_getter(operation)); 57 | } 58 | 59 | void 60 | vdo_set_up_reference_operation_with_lock(enum journal_operation type, 61 | physical_block_number_t pbn, 62 | enum block_mapping_state state, 63 | struct pbn_lock *lock, 64 | struct reference_operation *operation); 65 | 66 | void 67 | vdo_set_up_reference_operation_with_zone(enum journal_operation type, 68 | physical_block_number_t pbn, 69 | enum block_mapping_state state, 70 | struct physical_zone *zone, 71 | struct reference_operation *operation); 72 | 73 | #endif /* REFERENCE_OPERATION_H */ 74 | -------------------------------------------------------------------------------- /vdo/release-versions.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | * 5 | * This program is free software; you can redistribute it and/or 6 | * modify it under the terms of the GNU General Public License 7 | * as published by the Free Software Foundation; either version 2 8 | * of the License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program; if not, write to the Free Software 17 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 18 | * 02110-1301, USA. 19 | */ 20 | 21 | #ifndef RELEASE_VERSIONS_H 22 | #define RELEASE_VERSIONS_H 23 | 24 | enum { 25 | VDO_OXYGEN_RELEASE_VERSION_NUMBER = 109583, 26 | VDO_FLUORINE_RELEASE_VERSION_NUMBER = 115838, 27 | VDO_NEON_RELEASE_VERSION_NUMBER = 120965, 28 | VDO_SODIUM_RELEASE_VERSION_NUMBER = 127441, 29 | VDO_MAGNESIUM_RELEASE_VERSION_NUMBER = 131337, 30 | VDO_ALUMINUM_RELEASE_VERSION_NUMBER = 133524, 31 | VDO_HEAD_RELEASE_VERSION_NUMBER = 0, 32 | VDO_CURRENT_RELEASE_VERSION_NUMBER = VDO_HEAD_RELEASE_VERSION_NUMBER, 33 | }; 34 | 35 | #endif /* not RELEASE_VERSIONS_H */ 36 | -------------------------------------------------------------------------------- /vdo/request-queue.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef REQUEST_QUEUE_H 7 | #define REQUEST_QUEUE_H 8 | 9 | #include "compiler.h" 10 | #include "type-defs.h" 11 | #include "uds.h" 12 | 13 | struct uds_request_queue; 14 | 15 | /* void return value because this function will process its own errors */ 16 | typedef void uds_request_queue_processor_t(struct uds_request *); 17 | 18 | /** 19 | * Allocate a new request processing queue and start a worker thread to 20 | * consume and service requests in the queue. 21 | * 22 | * @param queue_name the name of the queue and the worker thread 23 | * @param process_one the function the worker will invoke on each request 24 | * @param queue_ptr a pointer to receive the new queue 25 | * 26 | * @return UDS_SUCCESS or an error code 27 | **/ 28 | int __must_check 29 | make_uds_request_queue(const char *queue_name, 30 | uds_request_queue_processor_t *process_one, 31 | struct uds_request_queue **queue_ptr); 32 | 33 | /** 34 | * Add a request to the end of the queue for processing by the worker thread. 35 | * If the requeued flag is set on the request, it will be processed before 36 | * any non-requeued requests under most circumstances. 37 | * 38 | * @param queue the request queue that should process the request 39 | * @param request the request to be processed on the queue's worker thread 40 | **/ 41 | void uds_request_queue_enqueue(struct uds_request_queue *queue, 42 | struct uds_request *request); 43 | 44 | /** 45 | * Shut down the request queue worker thread, then destroy and free the queue. 46 | * 47 | * @param queue the queue to shut down and free 48 | **/ 49 | void uds_request_queue_finish(struct uds_request_queue *queue); 50 | 51 | #endif /* REQUEST_QUEUE_H */ 52 | -------------------------------------------------------------------------------- /vdo/slab-depot-format.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef SLAB_DEPOT_FORMAT_H 7 | #define SLAB_DEPOT_FORMAT_H 8 | 9 | #include "buffer.h" 10 | 11 | #include "header.h" 12 | #include "types.h" 13 | 14 | struct slab_depot_state_2_0 { 15 | struct slab_config slab_config; 16 | physical_block_number_t first_block; 17 | physical_block_number_t last_block; 18 | zone_count_t zone_count; 19 | } __packed; 20 | 21 | extern const struct header VDO_SLAB_DEPOT_HEADER_2_0; 22 | 23 | slab_count_t __must_check 24 | vdo_compute_slab_count(physical_block_number_t first_block, 25 | physical_block_number_t last_block, 26 | unsigned int slab_size_shift); 27 | 28 | size_t __must_check vdo_get_slab_depot_encoded_size(void); 29 | 30 | int __must_check 31 | vdo_encode_slab_depot_state_2_0(struct slab_depot_state_2_0 state, 32 | struct buffer *buffer); 33 | 34 | int __must_check 35 | vdo_decode_slab_depot_state_2_0(struct buffer *buffer, 36 | struct slab_depot_state_2_0 *state); 37 | 38 | int __must_check vdo_configure_slab_depot(block_count_t block_count, 39 | physical_block_number_t first_block, 40 | struct slab_config slab_config, 41 | zone_count_t zone_count, 42 | struct slab_depot_state_2_0 *state); 43 | 44 | int __must_check vdo_configure_slab(block_count_t slab_size, 45 | block_count_t slab_journal_blocks, 46 | struct slab_config *slab_config); 47 | 48 | block_count_t __must_check 49 | vdo_get_saved_reference_count_size(block_count_t block_count); 50 | 51 | #endif /* SLAB_DEPOT_FORMAT_H */ 52 | -------------------------------------------------------------------------------- /vdo/slab-iterator.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef SLAB_ITERATOR_H 7 | #define SLAB_ITERATOR_H 8 | 9 | #include "slab.h" 10 | #include "types.h" 11 | 12 | /* 13 | * A slab_iterator is a structure for iterating over a set of slabs. 14 | */ 15 | struct slab_iterator { 16 | struct vdo_slab **slabs; 17 | struct vdo_slab *next; 18 | slab_count_t end; 19 | slab_count_t stride; 20 | }; 21 | 22 | /** 23 | * vdo_iterate_slabs() - Return a slab_iterator initialized to iterate 24 | * over an array of slabs with a given stride. 25 | * @slabs: The array of slabs. 26 | * @start: The number of the slab to start iterating from. 27 | * @end: The number of the last slab which may be returned. 28 | * @stride: The difference in slab number between successive slabs. 29 | * 30 | * Iteration always occurs from higher to lower numbered slabs. 31 | * 32 | * Return: An initialized iterator structure. 33 | */ 34 | static inline struct slab_iterator vdo_iterate_slabs(struct vdo_slab **slabs, 35 | slab_count_t start, 36 | slab_count_t end, 37 | slab_count_t stride) 38 | { 39 | return (struct slab_iterator) { 40 | .slabs = slabs, 41 | .next = (((slabs == NULL) || (start < end)) ? NULL 42 | : slabs[start]), 43 | .end = end, 44 | .stride = stride, 45 | }; 46 | } 47 | 48 | /** 49 | * vdo_has_next_slab() - Check whether another vdo_slab would be returned by 50 | * the iterator. 51 | * @iterator: The iterator to poll. 52 | * 53 | * Return: true if the next call to vdo_next_slab will return a vdo_slab. 54 | */ 55 | static inline bool vdo_has_next_slab(const struct slab_iterator *iterator) 56 | { 57 | return (iterator->next != NULL); 58 | } 59 | 60 | /** 61 | * vdo_next_slab() - Get the next vdo_slab, advancing the iterator. 62 | * @iterator: The iterator over the vdo_slab chain. 63 | * 64 | * Return: The next vdo_slab or NULL if the array of slabs is empty or if all 65 | * the appropriate Slabs have been returned. 66 | */ 67 | static inline struct vdo_slab *vdo_next_slab(struct slab_iterator *iterator) 68 | { 69 | struct vdo_slab *slab = iterator->next; 70 | 71 | if ((slab == NULL) 72 | || (slab->slab_number < iterator->end + iterator->stride)) { 73 | iterator->next = NULL; 74 | } else { 75 | iterator->next = 76 | iterator->slabs[slab->slab_number - iterator->stride]; 77 | } 78 | return slab; 79 | } 80 | 81 | #endif /* SLAB_ITERATOR_H */ 82 | -------------------------------------------------------------------------------- /vdo/slab-journal-format.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #include "slab-journal-format.h" 7 | 8 | #include "journal-point.h" 9 | #include "types.h" 10 | 11 | /** 12 | * vdo_decode_slab_journal_entry() - Decode a slab journal entry. 13 | * @block: The journal block holding the entry. 14 | * @entry_count: The number of the entry. 15 | * 16 | * Return: The decoded entry. 17 | */ 18 | struct slab_journal_entry 19 | vdo_decode_slab_journal_entry(struct packed_slab_journal_block *block, 20 | journal_entry_count_t entry_count) 21 | { 22 | struct slab_journal_entry entry = 23 | vdo_unpack_slab_journal_entry(&block->payload.entries[entry_count]); 24 | if (block->header.has_block_map_increments && 25 | ((block->payload.full_entries.entry_types[entry_count / 8] & 26 | ((byte)1 << (entry_count % 8))) != 0)) { 27 | entry.operation = VDO_JOURNAL_BLOCK_MAP_INCREMENT; 28 | } 29 | return entry; 30 | } 31 | 32 | -------------------------------------------------------------------------------- /vdo/slab-scrubber.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef SLAB_SCRUBBER_H 7 | #define SLAB_SCRUBBER_H 8 | 9 | #include 10 | 11 | #include "admin-state.h" 12 | #include "completion.h" 13 | #include "types.h" 14 | #include "wait-queue.h" 15 | 16 | struct slab_scrubber { 17 | struct vdo_completion completion; 18 | /* The queue of slabs to scrub first */ 19 | struct list_head high_priority_slabs; 20 | /* The queue of slabs to scrub once there are no high_priority_slabs */ 21 | struct list_head slabs; 22 | /* The queue of VIOs waiting for a slab to be scrubbed */ 23 | struct wait_queue waiters; 24 | 25 | /* 26 | * The number of slabs that are unrecovered or being scrubbed. This 27 | * field is modified by the physical zone thread, but is queried by 28 | * other threads. 29 | */ 30 | slab_count_t slab_count; 31 | 32 | /* The administrative state of the scrubber */ 33 | struct admin_state admin_state; 34 | /* Whether to only scrub high-priority slabs */ 35 | bool high_priority_only; 36 | /* The context for entering read-only mode */ 37 | struct read_only_notifier *read_only_notifier; 38 | /* The slab currently being scrubbed */ 39 | struct vdo_slab *slab; 40 | /* The vio for loading slab journal blocks */ 41 | struct vio *vio; 42 | /* A buffer to store the slab journal blocks */ 43 | char *journal_data; 44 | }; 45 | 46 | int __must_check 47 | vdo_make_slab_scrubber(struct vdo *vdo, 48 | block_count_t slab_journal_size, 49 | struct read_only_notifier *read_only_notifier, 50 | struct slab_scrubber **scrubber_ptr); 51 | 52 | void vdo_free_slab_scrubber(struct slab_scrubber *scrubber); 53 | 54 | void vdo_register_slab_for_scrubbing(struct slab_scrubber *scrubber, 55 | struct vdo_slab *slab, 56 | bool high_priority); 57 | 58 | void vdo_scrub_slabs(struct slab_scrubber *scrubber, 59 | void *parent, 60 | vdo_action *callback, 61 | vdo_action *error_handler); 62 | 63 | void vdo_scrub_high_priority_slabs(struct slab_scrubber *scrubber, 64 | bool scrub_at_least_one, 65 | struct vdo_completion *parent, 66 | vdo_action *callback, 67 | vdo_action *error_handler); 68 | 69 | void vdo_stop_slab_scrubbing(struct slab_scrubber *scrubber, 70 | struct vdo_completion *parent); 71 | 72 | void vdo_resume_slab_scrubbing(struct slab_scrubber *scrubber, 73 | struct vdo_completion *parent); 74 | 75 | int vdo_enqueue_clean_slab_waiter(struct slab_scrubber *scrubber, 76 | struct waiter *waiter); 77 | 78 | slab_count_t __must_check 79 | vdo_get_scrubber_slab_count(const struct slab_scrubber *scrubber); 80 | 81 | void vdo_dump_slab_scrubber(const struct slab_scrubber *scrubber); 82 | 83 | #endif /* SLAB_SCRUBBER_H */ 84 | -------------------------------------------------------------------------------- /vdo/slab-summary-format.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef SLAB_SUMMARY_FORMAT_H 7 | #define SLAB_SUMMARY_FORMAT_H 8 | 9 | #include "constants.h" 10 | #include "types.h" 11 | 12 | /** 13 | * typedef tail_block_offset_t - The offset of a slab journal tail block. 14 | */ 15 | typedef uint8_t tail_block_offset_t; 16 | 17 | enum { 18 | VDO_SLAB_SUMMARY_FULLNESS_HINT_BITS = 6, 19 | }; 20 | 21 | struct slab_summary_entry { 22 | /* Bits 7..0: The offset of the tail block within the slab journal */ 23 | tail_block_offset_t tail_block_offset; 24 | 25 | #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 26 | /* Bits 13..8: A hint about the fullness of the slab */ 27 | unsigned int fullness_hint : 6; 28 | /* Bit 14: Whether the ref_counts must be loaded from the layer */ 29 | unsigned int load_ref_counts : 1; 30 | /* Bit 15: The believed cleanliness of this slab */ 31 | unsigned int is_dirty : 1; 32 | #else 33 | /* Bit 15: The believed cleanliness of this slab */ 34 | unsigned int is_dirty : 1; 35 | /* Bit 14: Whether the ref_counts must be loaded from the layer */ 36 | unsigned int load_ref_counts : 1; 37 | /* Bits 13..8: A hint about the fullness of the slab */ 38 | unsigned int fullness_hint : 6; 39 | #endif 40 | } __packed; 41 | 42 | /* XXX: These methods shouldn't take a block_size parameter. */ 43 | 44 | /** 45 | * vdo_get_slab_summary_zone_size() - Returns the size on disk of a single 46 | * zone of the slab_summary. 47 | * @block_size: The block size of the physical layer. 48 | * 49 | * Return: the number of blocks required to store a single zone of the 50 | * slab_summary on disk. 51 | */ 52 | static inline block_count_t __must_check 53 | vdo_get_slab_summary_zone_size(block_size_t block_size) 54 | { 55 | slab_count_t entries_per_block = 56 | block_size / sizeof(struct slab_summary_entry); 57 | block_count_t blocks_needed = MAX_VDO_SLABS / entries_per_block; 58 | return blocks_needed; 59 | } 60 | 61 | /** 62 | * vdo_get_slab_summary_size() - Return the size on disk of the slab_summary 63 | * structure. 64 | * @block_size: The block size of the physical layer. 65 | * 66 | * Return: The blocks required to store the slab_summary on disk. 67 | */ 68 | static inline block_count_t __must_check 69 | vdo_get_slab_summary_size(block_size_t block_size) 70 | { 71 | return vdo_get_slab_summary_zone_size(block_size) * MAX_VDO_PHYSICAL_ZONES; 72 | } 73 | 74 | /** 75 | * vdo_get_slab_summary_hint_shift() - Compute the shift for slab summary 76 | * hints. 77 | * @slab_size_shift: Exponent for the number of blocks per slab. 78 | * 79 | * Return: The hint shift. 80 | */ 81 | static inline uint8_t __must_check 82 | vdo_get_slab_summary_hint_shift(unsigned int slab_size_shift) 83 | { 84 | return ((slab_size_shift > VDO_SLAB_SUMMARY_FULLNESS_HINT_BITS) ? 85 | (slab_size_shift - VDO_SLAB_SUMMARY_FULLNESS_HINT_BITS) : 0); 86 | } 87 | 88 | #endif /* SLAB_SUMMARY_FORMAT_H */ 89 | -------------------------------------------------------------------------------- /vdo/status-codes.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef STATUS_CODES_H 7 | #define STATUS_CODES_H 8 | 9 | #include "errors.h" 10 | 11 | enum { 12 | UDS_BLOCK_SIZE = UDS_ERROR_CODE_BLOCK_END - UDS_ERROR_CODE_BASE, 13 | VDO_BLOCK_START = UDS_ERROR_CODE_BLOCK_END, 14 | VDO_BLOCK_END = VDO_BLOCK_START + UDS_BLOCK_SIZE, 15 | PRP_BLOCK_START = VDO_BLOCK_END, 16 | PRP_BLOCK_END = PRP_BLOCK_START + UDS_BLOCK_SIZE, 17 | }; 18 | 19 | /* 20 | * VDO-specific status codes. 21 | */ 22 | enum vdo_status_codes { 23 | /* successful result */ 24 | VDO_SUCCESS, 25 | /* base of all VDO errors */ 26 | VDO_STATUS_CODE_BASE = VDO_BLOCK_START, 27 | /* we haven't written this yet */ 28 | VDO_NOT_IMPLEMENTED = VDO_STATUS_CODE_BASE, 29 | /* input out of range */ 30 | VDO_OUT_OF_RANGE, 31 | /* an invalid reference count would result */ 32 | VDO_REF_COUNT_INVALID, 33 | /* a free block could not be allocated */ 34 | VDO_NO_SPACE, 35 | /* unexpected EOF on block read */ 36 | VDO_UNEXPECTED_EOF, 37 | /* improper or missing configuration option */ 38 | VDO_BAD_CONFIGURATION, 39 | /* socket opening or binding problem */ 40 | VDO_SOCKET_ERROR, 41 | /* read or write on non-aligned offset */ 42 | VDO_BAD_ALIGNMENT, 43 | /* prior operation still in progress */ 44 | VDO_COMPONENT_BUSY, 45 | /* page contents incorrect or corrupt data */ 46 | VDO_BAD_PAGE, 47 | /* unsupported version of some component */ 48 | VDO_UNSUPPORTED_VERSION, 49 | /* component id mismatch in decoder */ 50 | VDO_INCORRECT_COMPONENT, 51 | /* parameters have conflicting values */ 52 | VDO_PARAMETER_MISMATCH, 53 | /* the block size is too small */ 54 | VDO_BLOCK_SIZE_TOO_SMALL, 55 | /* no partition exists with a given id */ 56 | VDO_UNKNOWN_PARTITION, 57 | /* a partition already exists with a given id */ 58 | VDO_PARTITION_EXISTS, 59 | /* the VDO is not in read-only mode */ 60 | VDO_NOT_READ_ONLY, 61 | /* physical block growth of too few blocks */ 62 | VDO_INCREMENT_TOO_SMALL, 63 | /* incorrect checksum */ 64 | VDO_CHECKSUM_MISMATCH, 65 | /* the recovery journal is full */ 66 | VDO_RECOVERY_JOURNAL_FULL, 67 | /* a lock is held incorrectly */ 68 | VDO_LOCK_ERROR, 69 | /* the VDO is in read-only mode */ 70 | VDO_READ_ONLY, 71 | /* the VDO is shutting down */ 72 | VDO_SHUTTING_DOWN, 73 | /* the recovery journal has corrupt entries */ 74 | VDO_CORRUPT_JOURNAL, 75 | /* exceeds maximum number of slabs supported */ 76 | VDO_TOO_MANY_SLABS, 77 | /* a compressed block fragment is invalid */ 78 | VDO_INVALID_FRAGMENT, 79 | /* action is unsupported while rebuilding */ 80 | VDO_RETRY_AFTER_REBUILD, 81 | /* the extended command is not known */ 82 | VDO_UNKNOWN_COMMAND, 83 | /* bad extended command parameters */ 84 | VDO_COMMAND_ERROR, 85 | /* cannot determine sizes to fit */ 86 | VDO_CANNOT_DETERMINE_SIZE, 87 | /* a block map entry is invalid */ 88 | VDO_BAD_MAPPING, 89 | /* read cache has no free slots */ 90 | VDO_READ_CACHE_BUSY, 91 | /* bio_add_page failed */ 92 | VDO_BIO_CREATION_FAILED, 93 | /* bad magic number */ 94 | VDO_BAD_MAGIC, 95 | /* bad nonce */ 96 | VDO_BAD_NONCE, 97 | /* sequence number overflow */ 98 | VDO_JOURNAL_OVERFLOW, 99 | /* the VDO is not in a state to perform an admin operation */ 100 | VDO_INVALID_ADMIN_STATE, 101 | /* failure adding a sysfs node */ 102 | VDO_CANT_ADD_SYSFS_NODE, 103 | /* one more than last error code */ 104 | VDO_STATUS_CODE_LAST, 105 | VDO_STATUS_CODE_BLOCK_END = VDO_BLOCK_END 106 | }; 107 | 108 | extern const struct error_info vdo_status_list[]; 109 | 110 | int vdo_register_status_codes(void); 111 | 112 | int vdo_map_to_system_error(int error); 113 | 114 | #endif /* STATUS_CODES_H */ 115 | -------------------------------------------------------------------------------- /vdo/string-utils.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #include "string-utils.h" 7 | 8 | #include "errors.h" 9 | #include "logger.h" 10 | #include "memory-alloc.h" 11 | #include "permassert.h" 12 | #include "uds.h" 13 | 14 | int uds_wrap_vsnprintf(const char *what, 15 | char *buf, 16 | size_t buf_size, 17 | int error, 18 | const char *fmt, 19 | va_list ap, 20 | size_t *needed) 21 | { 22 | int n; 23 | 24 | if (buf == NULL) { 25 | static char nobuf[1]; 26 | 27 | buf = nobuf; 28 | buf_size = 0; 29 | } 30 | n = vsnprintf(buf, buf_size, fmt, ap); 31 | if (n < 0) { 32 | return uds_log_error_strerror(UDS_UNEXPECTED_RESULT, 33 | "%s: vsnprintf failed", what); 34 | } 35 | if (needed) { 36 | *needed = n; 37 | } 38 | if (((size_t) n >= buf_size) && (buf != NULL) && 39 | (error != UDS_SUCCESS)) { 40 | return uds_log_error_strerror(error, 41 | "%s: string too long", what); 42 | } 43 | return UDS_SUCCESS; 44 | } 45 | 46 | int uds_fixed_sprintf(const char *what, 47 | char *buf, 48 | size_t buf_size, 49 | int error, 50 | const char *fmt, 51 | ...) 52 | { 53 | va_list args; 54 | int result; 55 | 56 | if (buf == NULL) { 57 | return UDS_INVALID_ARGUMENT; 58 | } 59 | va_start(args, fmt); 60 | result = uds_wrap_vsnprintf(what, buf, buf_size, error, fmt, args, 61 | NULL); 62 | va_end(args); 63 | return result; 64 | } 65 | 66 | char *uds_v_append_to_buffer(char *buffer, char *buf_end, const char *fmt, 67 | va_list args) 68 | { 69 | size_t n = vsnprintf(buffer, buf_end - buffer, fmt, args); 70 | 71 | if (n >= (size_t)(buf_end - buffer)) { 72 | buffer = buf_end; 73 | } else { 74 | buffer += n; 75 | } 76 | return buffer; 77 | } 78 | 79 | char *uds_append_to_buffer(char *buffer, char *buf_end, const char *fmt, ...) 80 | { 81 | va_list ap; 82 | char *pos; 83 | 84 | va_start(ap, fmt); 85 | pos = uds_v_append_to_buffer(buffer, buf_end, fmt, ap); 86 | va_end(ap); 87 | return pos; 88 | } 89 | 90 | -------------------------------------------------------------------------------- /vdo/string-utils.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef STRING_UTILS_H 7 | #define STRING_UTILS_H 8 | 9 | #include 10 | #include 11 | 12 | #include "compiler.h" 13 | #include "type-defs.h" 14 | 15 | /** 16 | * Convert a boolean value to its corresponding "true" or "false" string. 17 | * 18 | * @param value The boolean value to convert 19 | * 20 | * @return "true" if value is true, "false" otherwise. 21 | **/ 22 | static INLINE const char *uds_bool_to_string(bool value) 23 | { 24 | return (value ? "true" : "false"); 25 | } 26 | 27 | /** 28 | * Write a printf-style string into a fixed-size buffer, returning 29 | * errors if it would not fit. (our version of snprintf) 30 | * 31 | * @param [in] what A description of what is being written, for error 32 | * logging; if NULL doesn't log anything. 33 | * @param [out] buf The target buffer 34 | * @param [in] buf_size The size of buf 35 | * @param [in] error Error code to return on overflow 36 | * @param [in] fmt The sprintf format parameter. 37 | * 38 | * @return UDS_SUCCESS or error 39 | **/ 40 | int __must_check uds_fixed_sprintf(const char *what, 41 | char *buf, 42 | size_t buf_size, 43 | int error, 44 | const char *fmt, ...) 45 | __printf(5, 6); 46 | 47 | /** 48 | * Write printf-style string into an existing buffer, returning a specified 49 | * error code if it would not fit, and setting ``needed`` to the amount of 50 | * space that would be required. 51 | * 52 | * @param [in] what A description of what is being written, for logging. 53 | * @param [in] buf The buffer in which to write the string, or NULL to 54 | * merely determine the required space. 55 | * @param [in] buf_size The size of buf. 56 | * @param [in] error The error code to return for exceeding the specified 57 | * space, UDS_SUCCESS if no logging required. 58 | * @param [in] fmt The sprintf format specification. 59 | * @param [in] ap The variable argument pointer (see ). 60 | * @param [out] needed If non-NULL, the actual amount of string space 61 | * required, which may be smaller or larger than 62 | * buf_size. 63 | * 64 | * @return UDS_SUCCESS if the string fits, the value of the error parameter if 65 | * the string does not fit and a buffer was supplied, or 66 | * UDS_UNEXPECTED_RESULT if vsnprintf fails in some other undocumented 67 | * way. 68 | **/ 69 | int __must_check uds_wrap_vsnprintf(const char *what, 70 | char *buf, 71 | size_t buf_size, 72 | int error, 73 | const char *fmt, 74 | va_list ap, 75 | size_t *needed) 76 | __printf(5, 0); 77 | 78 | /** 79 | * Helper to append a string to a buffer. 80 | * 81 | * @param buffer the place at which to append the string 82 | * @param buf_end pointer to the end of the buffer 83 | * @param fmt a printf format string 84 | * 85 | * @return the updated buffer position after the append 86 | * 87 | * if insufficient space is available, the contents are silently truncated 88 | **/ 89 | char *uds_append_to_buffer(char *buffer, char *buf_end, const char *fmt, ...) 90 | __printf(3, 4); 91 | 92 | /** 93 | * Variable-arglist helper to append a string to a buffer. 94 | * 95 | * @param buffer the place at which to append the string 96 | * @param buf_end pointer to the end of the buffer 97 | * @param fmt a printf format string 98 | * @param args printf arguments 99 | * 100 | * @return the updated buffer position after the append 101 | * 102 | * if insufficient space is available, the contents are silently truncated 103 | **/ 104 | char *uds_v_append_to_buffer(char *buffer, char *buf_end, const char *fmt, 105 | va_list args) 106 | __printf(3, 0); 107 | 108 | 109 | #endif /* STRING_UTILS_H */ 110 | -------------------------------------------------------------------------------- /vdo/super-block-codec.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef SUPER_BLOCK_CODEC_H 7 | #define SUPER_BLOCK_CODEC_H 8 | 9 | #include "buffer.h" 10 | 11 | #include "header.h" 12 | #include "types.h" 13 | 14 | /* 15 | * The machinery for encoding and decoding super blocks. 16 | */ 17 | struct super_block_codec { 18 | /* The buffer for encoding and decoding component data */ 19 | struct buffer *component_buffer; 20 | /* 21 | * A sector-sized buffer wrapping the first sector of 22 | * encoded_super_block, for encoding and decoding the entire super 23 | * block. 24 | */ 25 | struct buffer *block_buffer; 26 | /* A 1-block buffer holding the encoded on-disk super block */ 27 | byte *encoded_super_block; 28 | }; 29 | 30 | int __must_check vdo_initialize_super_block_codec(struct super_block_codec *codec); 31 | 32 | void vdo_destroy_super_block_codec(struct super_block_codec *codec); 33 | 34 | int __must_check vdo_encode_super_block(struct super_block_codec *codec); 35 | 36 | int __must_check vdo_decode_super_block(struct super_block_codec *codec); 37 | 38 | #endif /* SUPER_BLOCK_CODEC_H */ 39 | -------------------------------------------------------------------------------- /vdo/super-block.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef SUPER_BLOCK_H 7 | #define SUPER_BLOCK_H 8 | 9 | #include "kernel-types.h" 10 | #include "types.h" 11 | 12 | struct vdo_super_block; 13 | 14 | void vdo_free_super_block(struct vdo_super_block *super_block); 15 | 16 | void vdo_save_super_block(struct vdo_super_block *super_block, 17 | physical_block_number_t super_block_offset, 18 | struct vdo_completion *parent); 19 | 20 | void vdo_load_super_block(struct vdo *vdo, 21 | struct vdo_completion *parent, 22 | physical_block_number_t super_block_offset, 23 | struct vdo_super_block **super_block_ptr); 24 | 25 | struct super_block_codec * __must_check 26 | vdo_get_super_block_codec(struct vdo_super_block *super_block); 27 | 28 | #endif /* SUPER_BLOCK_H */ 29 | -------------------------------------------------------------------------------- /vdo/sync-completion.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #include "sync-completion.h" 7 | 8 | #include 9 | 10 | #include "completion.h" 11 | 12 | struct sync_completion { 13 | struct vdo_completion vdo_completion; 14 | struct completion completion; 15 | vdo_action *action; 16 | }; 17 | 18 | /** 19 | * as_sync_completion() - Convert a vdo_completion to a sync completion. 20 | * @completion: The completion to convert. 21 | * 22 | * Return: The completion as a sync completion. 23 | */ 24 | static inline struct sync_completion * __must_check 25 | as_sync_completion(struct vdo_completion *completion) 26 | { 27 | vdo_assert_completion_type(completion->type, VDO_SYNC_COMPLETION); 28 | return container_of(completion, 29 | struct sync_completion, 30 | vdo_completion); 31 | } 32 | 33 | /** 34 | * complete_synchronous_action() - A vdo_action to signal the waiting thread 35 | * that a synchronous action is complete. 36 | * @completion: The sync completion. 37 | * 38 | * This callback is registered in run_synchrnous_action(). 39 | */ 40 | static void complete_synchronous_action(struct vdo_completion *completion) 41 | { 42 | complete(&(as_sync_completion(completion)->completion)); 43 | } 44 | 45 | /** 46 | * run_synchronous_action() - A vdo_action to perform a synchronous action 47 | * registered in vdo_perform_synchronous_action(). 48 | * @completion: The sync completion. 49 | */ 50 | static void run_synchronous_action(struct vdo_completion *completion) 51 | { 52 | completion->callback = complete_synchronous_action; 53 | as_sync_completion(completion)->action(completion); 54 | } 55 | 56 | /** 57 | * vdo_perform_synchronous_action() - Launch an action on a VDO thread and 58 | * wait for it to complete. 59 | * @vdo: The vdo. 60 | * @action: The callback to launch. 61 | * @thread_id: The thread on which to run the action. 62 | * @parent: The parent of the sync completion (may be NULL). 63 | */ 64 | int vdo_perform_synchronous_action(struct vdo *vdo, 65 | vdo_action *action, 66 | thread_id_t thread_id, 67 | void *parent) 68 | { 69 | struct sync_completion sync; 70 | 71 | vdo_initialize_completion(&sync.vdo_completion, vdo, VDO_SYNC_COMPLETION); 72 | init_completion(&sync.completion); 73 | sync.action = action; 74 | vdo_launch_completion_callback_with_parent(&sync.vdo_completion, 75 | run_synchronous_action, 76 | thread_id, 77 | parent); 78 | wait_for_completion(&sync.completion); 79 | return sync.vdo_completion.result; 80 | } 81 | -------------------------------------------------------------------------------- /vdo/sync-completion.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef SYNC_COMPLETION_H 7 | #define SYNC_COMPLETION_H 8 | 9 | #include "completion.h" 10 | #include "types.h" 11 | 12 | int vdo_perform_synchronous_action(struct vdo *vdo, 13 | vdo_action * action, 14 | thread_id_t thread_id, 15 | void *parent); 16 | 17 | #endif /* SYNC_COMPLETION_H */ 18 | -------------------------------------------------------------------------------- /vdo/sysfs.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #include 7 | 8 | #include "logger.h" 9 | 10 | #include "constants.h" 11 | #include "dedupe.h" 12 | #include "vdo.h" 13 | 14 | static int vdo_log_level_show(char *buf, 15 | const struct kernel_param *kp) 16 | { 17 | return sprintf(buf, "%s\n", uds_log_priority_to_string(get_uds_log_level())); 18 | } 19 | 20 | static int vdo_log_level_store(const char *buf, 21 | const struct kernel_param *kp) 22 | { 23 | static char internal_buf[11]; 24 | 25 | int n = strlen(buf); 26 | 27 | if (n > 10) { 28 | return -EINVAL; 29 | } 30 | 31 | memset(internal_buf, '\000', sizeof(internal_buf)); 32 | memcpy(internal_buf, buf, n); 33 | if (internal_buf[n - 1] == '\n') { 34 | internal_buf[n - 1] = '\000'; 35 | } 36 | set_uds_log_level(uds_log_string_to_priority(internal_buf)); 37 | return 0; 38 | } 39 | 40 | 41 | static int vdo_dedupe_timeout_interval_store(const char *buf, 42 | const struct kernel_param *kp) 43 | { 44 | int result = param_set_uint(buf, kp); 45 | 46 | if (result != 0) { 47 | return result; 48 | } 49 | vdo_set_dedupe_index_timeout_interval(*(uint *)kp->arg); 50 | return 0; 51 | } 52 | 53 | static int vdo_min_dedupe_timer_interval_store(const char *buf, 54 | const struct kernel_param *kp) 55 | { 56 | int result = param_set_uint(buf, kp); 57 | 58 | if (result != 0) { 59 | return result; 60 | } 61 | vdo_set_dedupe_index_min_timer_interval(*(uint *)kp->arg); 62 | return 0; 63 | } 64 | 65 | static const struct kernel_param_ops log_level_ops = { 66 | .set = vdo_log_level_store, 67 | .get = vdo_log_level_show, 68 | }; 69 | 70 | 71 | static const struct kernel_param_ops dedupe_timeout_ops = { 72 | .set = vdo_dedupe_timeout_interval_store, 73 | .get = param_get_uint, 74 | }; 75 | 76 | static const struct kernel_param_ops dedupe_timer_ops = { 77 | .set = vdo_min_dedupe_timer_interval_store, 78 | .get = param_get_uint, 79 | }; 80 | 81 | module_param_cb(log_level, &log_level_ops, NULL, 0644); 82 | 83 | 84 | module_param_cb(deduplication_timeout_interval, &dedupe_timeout_ops, 85 | &vdo_dedupe_index_timeout_interval, 0644); 86 | 87 | module_param_cb(min_deduplication_timer_interval, &dedupe_timer_ops, 88 | &vdo_dedupe_index_min_timer_interval, 0644); 89 | -------------------------------------------------------------------------------- /vdo/thread-cond-var.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #include "errors.h" 7 | #include "time-utils.h" 8 | #include "uds-threads.h" 9 | 10 | int uds_init_cond(struct cond_var *cv) 11 | { 12 | cv->event_count = NULL; 13 | return make_event_count(&cv->event_count); 14 | } 15 | 16 | int uds_signal_cond(struct cond_var *cv) 17 | { 18 | event_count_broadcast(cv->event_count); 19 | return UDS_SUCCESS; 20 | } 21 | 22 | int uds_broadcast_cond(struct cond_var *cv) 23 | { 24 | event_count_broadcast(cv->event_count); 25 | return UDS_SUCCESS; 26 | } 27 | 28 | int uds_wait_cond(struct cond_var *cv, struct mutex *mutex) 29 | { 30 | event_token_t token = event_count_prepare(cv->event_count); 31 | 32 | uds_unlock_mutex(mutex); 33 | event_count_wait(cv->event_count, token, NULL); 34 | uds_lock_mutex(mutex); 35 | return UDS_SUCCESS; 36 | } 37 | 38 | int uds_timed_wait_cond(struct cond_var *cv, 39 | struct mutex *mutex, 40 | ktime_t timeout) 41 | { 42 | bool happened; 43 | event_token_t token = event_count_prepare(cv->event_count); 44 | 45 | uds_unlock_mutex(mutex); 46 | happened = event_count_wait(cv->event_count, token, &timeout); 47 | uds_lock_mutex(mutex); 48 | return happened ? UDS_SUCCESS : ETIMEDOUT; 49 | } 50 | 51 | int uds_destroy_cond(struct cond_var *cv) 52 | { 53 | free_event_count(cv->event_count); 54 | cv->event_count = NULL; 55 | return UDS_SUCCESS; 56 | } 57 | -------------------------------------------------------------------------------- /vdo/thread-config.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef THREAD_CONFIG_H 7 | #define THREAD_CONFIG_H 8 | 9 | #include "permassert.h" 10 | 11 | #include "kernel-types.h" 12 | #include "types.h" 13 | 14 | struct thread_config { 15 | zone_count_t logical_zone_count; 16 | zone_count_t physical_zone_count; 17 | zone_count_t hash_zone_count; 18 | thread_count_t bio_thread_count; 19 | thread_count_t thread_count; 20 | thread_id_t admin_thread; 21 | thread_id_t journal_thread; 22 | thread_id_t packer_thread; 23 | thread_id_t dedupe_thread; 24 | thread_id_t bio_ack_thread; 25 | thread_id_t cpu_thread; 26 | thread_id_t *logical_threads; 27 | thread_id_t *physical_threads; 28 | thread_id_t *hash_zone_threads; 29 | thread_id_t *bio_threads; 30 | }; 31 | 32 | int __must_check 33 | vdo_make_thread_config(struct thread_count_config counts, 34 | struct thread_config **config_ptr); 35 | 36 | void vdo_free_thread_config(struct thread_config *config); 37 | 38 | /** 39 | * vdo_get_logical_zone_thread() - Get the thread id for a given logical zone. 40 | * @thread_config: The thread config. 41 | * @logical_zone: The number of the logical zone. 42 | * 43 | * Return: The thread id for the given zone. 44 | */ 45 | static inline thread_id_t __must_check 46 | vdo_get_logical_zone_thread(const struct thread_config *thread_config, 47 | zone_count_t logical_zone) 48 | { 49 | ASSERT_LOG_ONLY((logical_zone <= thread_config->logical_zone_count), 50 | "logical zone valid"); 51 | return thread_config->logical_threads[logical_zone]; 52 | } 53 | 54 | /** 55 | * vdo_get_physical_zone_thread() - Get the thread id for a given physical 56 | * zone. 57 | * @thread_config: The thread config. 58 | * @physical_zone: The number of the physical zone. 59 | * 60 | * Return: The thread id for the given zone. 61 | */ 62 | static inline thread_id_t __must_check 63 | vdo_get_physical_zone_thread(const struct thread_config *thread_config, 64 | zone_count_t physical_zone) 65 | { 66 | ASSERT_LOG_ONLY((physical_zone <= thread_config->physical_zone_count), 67 | "physical zone valid"); 68 | return thread_config->physical_threads[physical_zone]; 69 | } 70 | 71 | /** 72 | * vdo_get_hash_zone_thread() - Get the thread id for a given hash zone. 73 | * @thread_config: The thread config. 74 | * @hash_zone: The number of the hash zone. 75 | * 76 | * Return: The thread id for the given zone. 77 | */ 78 | static inline thread_id_t __must_check 79 | vdo_get_hash_zone_thread(const struct thread_config *thread_config, 80 | zone_count_t hash_zone) 81 | { 82 | ASSERT_LOG_ONLY((hash_zone <= thread_config->hash_zone_count), 83 | "hash zone valid"); 84 | return thread_config->hash_zone_threads[hash_zone]; 85 | } 86 | 87 | void vdo_get_thread_name(const struct thread_config *thread_config, 88 | thread_id_t thread_id, 89 | char *buffer, 90 | size_t buffer_length); 91 | 92 | #endif /* THREAD_CONFIG_H */ 93 | -------------------------------------------------------------------------------- /vdo/thread-device.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #include "thread-device.h" 7 | 8 | #include "thread-registry.h" 9 | 10 | /* A registry of threads associated with device id numbers. */ 11 | static struct thread_registry device_id_thread_registry; 12 | 13 | /* 14 | * Associate the current thread with a device id number for logging. 15 | * 16 | * A registered thread must be unregistered via 17 | * unregister_thread_device_id. 18 | */ 19 | void uds_register_thread_device_id(struct registered_thread *new_thread, 20 | unsigned int *id_ptr) 21 | { 22 | uds_register_thread(&device_id_thread_registry, new_thread, id_ptr); 23 | } 24 | 25 | void uds_unregister_thread_device_id(void) 26 | { 27 | uds_unregister_thread(&device_id_thread_registry); 28 | } 29 | 30 | int uds_get_thread_device_id(void) 31 | { 32 | const unsigned int *pointer = 33 | uds_lookup_thread(&device_id_thread_registry); 34 | 35 | return pointer ? *pointer : -1; 36 | } 37 | 38 | void uds_initialize_thread_device_registry(void) 39 | { 40 | uds_initialize_thread_registry(&device_id_thread_registry); 41 | } 42 | -------------------------------------------------------------------------------- /vdo/thread-device.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef UDS_THREAD_DEVICE_H 7 | #define UDS_THREAD_DEVICE_H 8 | 9 | #include "thread-registry.h" 10 | 11 | void uds_register_thread_device_id(struct registered_thread *new_thread, 12 | unsigned int *id_ptr); 13 | 14 | void uds_unregister_thread_device_id(void); 15 | 16 | int uds_get_thread_device_id(void); 17 | 18 | void uds_initialize_thread_device_registry(void); 19 | 20 | #endif /* UDS_THREAD_DEVICE_H */ 21 | -------------------------------------------------------------------------------- /vdo/thread-registry.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #include "thread-registry.h" 7 | 8 | #include 9 | 10 | #include "permassert.h" 11 | 12 | /* 13 | * We need to be careful when using other facilities that may use 14 | * thread registry functions in their normal operation. For example, 15 | * we do not want to invoke the logger while holding a lock. 16 | */ 17 | 18 | void uds_initialize_thread_registry(struct thread_registry *registry) 19 | { 20 | INIT_LIST_HEAD(®istry->links); 21 | spin_lock_init(®istry->lock); 22 | } 23 | 24 | /* Register the current thread and associate it with a data pointer. */ 25 | void uds_register_thread(struct thread_registry *registry, 26 | struct registered_thread *new_thread, 27 | const void *pointer) 28 | { 29 | struct registered_thread *thread; 30 | bool found_it = false; 31 | 32 | INIT_LIST_HEAD(&new_thread->links); 33 | new_thread->pointer = pointer; 34 | new_thread->task = current; 35 | 36 | spin_lock(®istry->lock); 37 | list_for_each_entry(thread, ®istry->links, links) { 38 | if (thread->task == current) { 39 | /* 40 | * This should not have been there. 41 | * We'll complain after releasing the lock. 42 | */ 43 | list_del_rcu(&thread->links); 44 | found_it = true; 45 | break; 46 | } 47 | } 48 | list_add_tail_rcu(&new_thread->links, ®istry->links); 49 | spin_unlock(®istry->lock); 50 | 51 | ASSERT_LOG_ONLY(!found_it, "new thread not already in registry"); 52 | if (found_it) { 53 | /* Ensure no RCU iterators see it before re-initializing. */ 54 | synchronize_rcu(); 55 | INIT_LIST_HEAD(&thread->links); 56 | } 57 | } 58 | 59 | void uds_unregister_thread(struct thread_registry *registry) 60 | { 61 | struct registered_thread *thread; 62 | bool found_it = false; 63 | 64 | spin_lock(®istry->lock); 65 | list_for_each_entry(thread, ®istry->links, links) { 66 | if (thread->task == current) { 67 | list_del_rcu(&thread->links); 68 | found_it = true; 69 | break; 70 | } 71 | } 72 | spin_unlock(®istry->lock); 73 | 74 | ASSERT_LOG_ONLY(found_it, "thread found in registry"); 75 | if (found_it) { 76 | /* Ensure no RCU iterators see it before re-initializing. */ 77 | synchronize_rcu(); 78 | INIT_LIST_HEAD(&thread->links); 79 | } 80 | } 81 | 82 | const void *uds_lookup_thread(struct thread_registry *registry) 83 | { 84 | struct registered_thread *thread; 85 | const void *result = NULL; 86 | 87 | rcu_read_lock(); 88 | list_for_each_entry_rcu(thread, ®istry->links, links) { 89 | if (thread->task == current) { 90 | result = thread->pointer; 91 | break; 92 | } 93 | } 94 | rcu_read_unlock(); 95 | 96 | return result; 97 | } 98 | -------------------------------------------------------------------------------- /vdo/thread-registry.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef THREAD_REGISTRY_H 7 | #define THREAD_REGISTRY_H 8 | 9 | #include 10 | #include 11 | 12 | struct thread_registry { 13 | struct list_head links; 14 | spinlock_t lock; 15 | }; 16 | 17 | struct registered_thread { 18 | struct list_head links; 19 | const void *pointer; 20 | struct task_struct *task; 21 | }; 22 | 23 | void uds_initialize_thread_registry(struct thread_registry *registry); 24 | 25 | void uds_register_thread(struct thread_registry *registry, 26 | struct registered_thread *new_thread, 27 | const void *pointer); 28 | 29 | void uds_unregister_thread(struct thread_registry *registry); 30 | 31 | const void *uds_lookup_thread(struct thread_registry *registry); 32 | 33 | #endif /* THREAD_REGISTRY_H */ 34 | -------------------------------------------------------------------------------- /vdo/time-utils.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #include "permassert.h" 7 | #include "string-utils.h" 8 | #include "time-utils.h" 9 | 10 | #include 11 | #include 12 | 13 | 14 | int64_t current_time_us(void) 15 | { 16 | return current_time_ns(CLOCK_REALTIME) / NSEC_PER_USEC; 17 | } 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /vdo/time-utils.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef TIME_UTILS_H 7 | #define TIME_UTILS_H 8 | 9 | #include "compiler.h" 10 | #include "type-defs.h" 11 | 12 | #include 13 | #include 14 | 15 | /* Some constants that are defined in kernel headers. */ 16 | 17 | /** 18 | * Return the current nanosecond time according to the specified clock 19 | * type. 20 | * 21 | * @param clock Either CLOCK_REALTIME or CLOCK_MONOTONIC 22 | * 23 | * @return the current time according to the clock in question 24 | **/ 25 | static INLINE ktime_t current_time_ns(clockid_t clock) 26 | { 27 | /* clock is always a constant, so gcc reduces this to a single call */ 28 | return clock == CLOCK_MONOTONIC ? ktime_get_ns() : ktime_get_real_ns(); 29 | } 30 | 31 | 32 | 33 | 34 | 35 | 36 | /** 37 | * Convert seconds to a ktime_t value 38 | * 39 | * @param seconds A number of seconds 40 | * 41 | * @return the equivalent number of seconds as a ktime_t 42 | **/ 43 | static INLINE ktime_t seconds_to_ktime(int64_t seconds) 44 | { 45 | return (ktime_t) seconds * NSEC_PER_SEC; 46 | } 47 | 48 | 49 | /** 50 | * Convert microseconds to a ktime_t value 51 | * 52 | * @param microseconds A number of microseconds 53 | * 54 | * @return the equivalent number of microseconds as a ktime_t 55 | **/ 56 | static INLINE ktime_t us_to_ktime(int64_t microseconds) 57 | { 58 | return (ktime_t) microseconds * NSEC_PER_USEC; 59 | } 60 | 61 | /** 62 | * Convert a ktime_t value to seconds 63 | * 64 | * @param reltime The time value 65 | * 66 | * @return the equivalent number of seconds, truncated 67 | **/ 68 | static INLINE int64_t ktime_to_seconds(ktime_t reltime) 69 | { 70 | return reltime / NSEC_PER_SEC; 71 | } 72 | 73 | 74 | /** 75 | * Return the wall clock time in microseconds. The actual value is time 76 | * since the epoch (see "man gettimeofday"), but the typical use is to call 77 | * this twice and compute the difference, giving the elapsed time between 78 | * the two calls. 79 | * 80 | * @return the time in microseconds 81 | **/ 82 | int64_t __must_check current_time_us(void); 83 | 84 | 85 | #endif /* TIME_UTILS_H */ 86 | -------------------------------------------------------------------------------- /vdo/type-defs.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef TYPE_DEFS_H 7 | #define TYPE_DEFS_H 8 | 9 | /* 10 | * General system type definitions. 11 | */ 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | typedef unsigned char byte; 18 | 19 | #define CHAR_BIT 8 20 | 21 | #define INT64_MAX (9223372036854775807L) 22 | #define UCHAR_MAX ((unsigned char)~0ul) 23 | #define UINT8_MAX ((uint8_t)~0ul) 24 | #define UINT16_MAX ((uint16_t)~0ul) 25 | #define UINT64_MAX ((uint64_t)~0ul) 26 | #endif /* TYPE_DEFS_H */ 27 | -------------------------------------------------------------------------------- /vdo/uds-sysfs.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef UDS_SYSFS_H 7 | #define UDS_SYSFS_H 8 | 9 | /** 10 | * Called when the module is loaded to initialize the /sys/\ 11 | * tree. 12 | * 13 | * @return 0 on success, or non-zero on error 14 | **/ 15 | int uds_init_sysfs(void); 16 | 17 | /** 18 | * Called when the module is being unloaded to terminate the 19 | * /sys/\ tree. 20 | **/ 21 | void uds_put_sysfs(void); 22 | 23 | #endif /* UDS_SYSFS_H */ 24 | -------------------------------------------------------------------------------- /vdo/vdo-component-states.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef VDO_COMPONENT_STATES_H 7 | #define VDO_COMPONENT_STATES_H 8 | 9 | #include "block-map-format.h" 10 | #include "recovery-journal-format.h" 11 | #include "slab-depot-format.h" 12 | #include "types.h" 13 | #include "vdo-component.h" 14 | #include "vdo-layout.h" 15 | 16 | /* 17 | * The version of the on-disk format of a VDO volume. This should be 18 | * incremented any time the on-disk representation of any VDO structure 19 | * changes. Changes which require only online upgrade steps should increment 20 | * the minor version. Changes which require an offline upgrade or which can not 21 | * be upgraded to at all should increment the major version and set the minor 22 | * version to 0. 23 | */ 24 | extern const struct version_number VDO_VOLUME_VERSION_67_0; 25 | 26 | /* 27 | * The entirety of the component data encoded in the VDO super block. 28 | */ 29 | struct vdo_component_states { 30 | /* The release version */ 31 | release_version_number_t release_version; 32 | 33 | /* The VDO volume version */ 34 | struct version_number volume_version; 35 | 36 | /* Components */ 37 | struct vdo_component vdo; 38 | struct block_map_state_2_0 block_map; 39 | struct recovery_journal_state_7_0 recovery_journal; 40 | struct slab_depot_state_2_0 slab_depot; 41 | 42 | /* Our partitioning of the underlying storage */ 43 | struct fixed_layout *layout; 44 | }; 45 | 46 | void vdo_destroy_component_states(struct vdo_component_states *states); 47 | 48 | int __must_check 49 | vdo_decode_component_states(struct buffer *buffer, 50 | release_version_number_t expected_release_version, 51 | struct vdo_component_states *states); 52 | 53 | int __must_check 54 | vdo_validate_component_states(struct vdo_component_states *states, 55 | nonce_t geometry_nonce, 56 | block_count_t physical_size, 57 | block_count_t logical_size); 58 | 59 | /** 60 | * vdo_encode() - Encode a VDO super block into a buffer for writing in the 61 | * super block. 62 | * @buffer: The buffer to encode into. 63 | * @states: The states of the vdo to be encoded. 64 | */ 65 | int __must_check 66 | vdo_encode(struct buffer *buffer, struct vdo_component_states *states); 67 | 68 | int vdo_encode_component_states(struct buffer *buffer, 69 | const struct vdo_component_states *states); 70 | 71 | #endif /* VDO_COMPONENT_STATES_H */ 72 | -------------------------------------------------------------------------------- /vdo/vdo-component.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef VDO_COMPONENT_H 7 | #define VDO_COMPONENT_H 8 | 9 | #include "buffer.h" 10 | 11 | #include "types.h" 12 | 13 | /* 14 | * The configuration of the VDO service. 15 | */ 16 | struct vdo_config { 17 | block_count_t logical_blocks; /* number of logical blocks */ 18 | block_count_t physical_blocks; /* number of physical blocks */ 19 | block_count_t slab_size; /* number of blocks in a slab */ 20 | block_count_t recovery_journal_size; /* number of recovery journal blocks */ 21 | block_count_t slab_journal_blocks; /* number of slab journal blocks */ 22 | }; 23 | 24 | /* 25 | * This is the structure that captures the vdo fields saved as a super block 26 | * component. 27 | */ 28 | struct vdo_component { 29 | enum vdo_state state; 30 | uint64_t complete_recoveries; 31 | uint64_t read_only_recoveries; 32 | struct vdo_config config; 33 | nonce_t nonce; 34 | }; 35 | 36 | size_t __must_check vdo_get_component_encoded_size(void); 37 | 38 | int __must_check 39 | vdo_encode_component(struct vdo_component component, struct buffer *buffer); 40 | 41 | int __must_check 42 | vdo_decode_component(struct buffer *buffer, struct vdo_component *component); 43 | 44 | int vdo_validate_config(const struct vdo_config *config, 45 | block_count_t physical_block_count, 46 | block_count_t logical_block_count); 47 | 48 | #endif /* VDO_COMPONENT_H */ 49 | -------------------------------------------------------------------------------- /vdo/vdo-load.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef VDO_LOAD_H 7 | #define VDO_LOAD_H 8 | 9 | #include "kernel-types.h" 10 | 11 | int __must_check vdo_load(struct vdo *vdo); 12 | 13 | int __must_check 14 | vdo_prepare_to_load(struct vdo *vdo); 15 | 16 | #endif /* VDO_LOAD_H */ 17 | -------------------------------------------------------------------------------- /vdo/vdo-recovery.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef VDO_RECOVERY_H 7 | #define VDO_RECOVERY_H 8 | 9 | #include "completion.h" 10 | #include "vdo.h" 11 | 12 | void vdo_replay_into_slab_journals(struct block_allocator *allocator, 13 | struct vdo_completion *completion, 14 | void *context); 15 | 16 | void vdo_launch_recovery(struct vdo *vdo, struct vdo_completion *parent); 17 | 18 | #endif /* VDO_RECOVERY_H */ 19 | -------------------------------------------------------------------------------- /vdo/vdo-resize-logical.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef VDO_RESIZE_LOGICAL_H 7 | #define VDO_RESIZE_LOGICAL_H 8 | 9 | #include "kernel-types.h" 10 | #include "types.h" 11 | 12 | int vdo_perform_grow_logical(struct vdo *vdo, block_count_t new_logical_blocks); 13 | 14 | int vdo_prepare_to_grow_logical(struct vdo *vdo, block_count_t new_logical_blocks); 15 | 16 | #endif /* VDO_RESIZE_LOGICAL_H */ 17 | -------------------------------------------------------------------------------- /vdo/vdo-resize.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef VDO_RESIZE_H 7 | #define VDO_RESIZE_H 8 | 9 | #include "kernel-types.h" 10 | #include "types.h" 11 | 12 | int vdo_perform_grow_physical(struct vdo *vdo, 13 | block_count_t new_physical_blocks); 14 | 15 | int __must_check 16 | vdo_prepare_to_grow_physical(struct vdo *vdo, 17 | block_count_t new_physical_blocks); 18 | 19 | #endif /* VDO_RESIZE_H */ 20 | -------------------------------------------------------------------------------- /vdo/vdo-resume.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef VDO_RESUME_H 7 | #define VDO_RESUME_H 8 | 9 | #include "kernel-types.h" 10 | #include "types.h" 11 | 12 | int vdo_preresume_internal(struct vdo *vdo, 13 | struct device_config *config, 14 | const char *device_name); 15 | 16 | #endif /* VDO_RESUME_H */ 17 | -------------------------------------------------------------------------------- /vdo/vdo-suspend.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef VDO_SUSPEND_H 7 | #define VDO_SUSPEND_H 8 | 9 | #include "kernel-types.h" 10 | 11 | int vdo_suspend(struct vdo *vdo); 12 | 13 | #endif /* VDO_SUSPEND_H */ 14 | -------------------------------------------------------------------------------- /vdo/vio-pool.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef VIO_POOL_H 7 | #define VIO_POOL_H 8 | 9 | #include 10 | 11 | #include "permassert.h" 12 | 13 | #include "completion.h" 14 | #include "types.h" 15 | #include "wait-queue.h" 16 | 17 | /* 18 | * A vio_pool is a collection of preallocated vios used to write arbitrary 19 | * metadata blocks. 20 | */ 21 | 22 | /* 23 | * A vio_pool_entry is the pair of vio and buffer whether in use or not. 24 | */ 25 | struct vio_pool_entry { 26 | struct list_head available_entry; 27 | struct vio *vio; 28 | void *buffer; 29 | void *parent; 30 | void *context; 31 | }; 32 | 33 | /** 34 | * typedef vio_constructor - A function which constructs a vio for a pool. 35 | * @vdo: The vdo in which the vio will operate. 36 | * @parent: The parent of the vio. 37 | * @buffer: The data buffer for the vio. 38 | * @vio_ptr: A pointer to hold the new vio. 39 | */ 40 | typedef int vio_constructor(struct vdo *vdo, 41 | void *parent, 42 | void *buffer, 43 | struct vio **vio_ptr); 44 | 45 | int __must_check make_vio_pool(struct vdo *vdo, 46 | size_t pool_size, 47 | thread_id_t thread_id, 48 | vio_constructor *constructor, 49 | void *context, 50 | struct vio_pool **pool_ptr); 51 | 52 | void free_vio_pool(struct vio_pool *pool); 53 | 54 | bool __must_check is_vio_pool_busy(struct vio_pool *pool); 55 | 56 | int acquire_vio_from_pool(struct vio_pool *pool, struct waiter *waiter); 57 | 58 | void return_vio_to_pool(struct vio_pool *pool, struct vio_pool_entry *entry); 59 | 60 | /** 61 | * as_vio_pool_entry() - Convert a list entry to the vio_pool_entry 62 | * that contains it. 63 | * @entry: The list entry to convert. 64 | * 65 | * Return: The vio_pool_entry wrapping the list entry. 66 | */ 67 | static inline struct vio_pool_entry *as_vio_pool_entry(struct list_head *entry) 68 | { 69 | return list_entry(entry, struct vio_pool_entry, available_entry); 70 | } 71 | 72 | #endif /* VIO_POOL_H */ 73 | -------------------------------------------------------------------------------- /vdo/vio-read.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef VIO_READ_H 7 | #define VIO_READ_H 8 | 9 | #include "kernel-types.h" 10 | 11 | void launch_read_data_vio(struct data_vio *data_vio); 12 | 13 | void cleanup_read_data_vio(struct data_vio *data_vio); 14 | 15 | #endif /* VIO_READ_H */ 16 | -------------------------------------------------------------------------------- /vdo/vio-write.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef VIO_WRITE_H 7 | #define VIO_WRITE_H 8 | 9 | #include "kernel-types.h" 10 | 11 | void launch_write_data_vio(struct data_vio *data_vio); 12 | 13 | void cleanup_write_data_vio(struct data_vio *data_vio); 14 | 15 | void continue_write_after_compression(struct data_vio *data_vio); 16 | 17 | void launch_compress_data_vio(struct data_vio *data_vio); 18 | 19 | void launch_deduplicate_data_vio(struct data_vio *data_vio); 20 | 21 | #endif /* VIO_WRITE_H */ 22 | -------------------------------------------------------------------------------- /vdo/volume-geometry.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef VOLUME_GEOMETRY_H 7 | #define VOLUME_GEOMETRY_H 8 | 9 | 10 | #include 11 | #include 12 | 13 | #include "uds.h" 14 | 15 | #include "kernel-types.h" 16 | #include "types.h" 17 | 18 | enum { 19 | VDO_GEOMETRY_BLOCK_LOCATION = 0, 20 | }; 21 | 22 | struct index_config { 23 | uint32_t mem; 24 | uint32_t unused; 25 | bool sparse; 26 | } __packed; 27 | 28 | enum volume_region_id { 29 | VDO_INDEX_REGION = 0, 30 | VDO_DATA_REGION = 1, 31 | VDO_VOLUME_REGION_COUNT, 32 | }; 33 | 34 | struct volume_region { 35 | /* The ID of the region */ 36 | enum volume_region_id id; 37 | /* 38 | * The absolute starting offset on the device. The region continues 39 | * until the next region begins. 40 | */ 41 | physical_block_number_t start_block; 42 | } __packed; 43 | 44 | struct volume_geometry { 45 | /* The release version number of this volume */ 46 | release_version_number_t release_version; 47 | /* The nonce of this volume */ 48 | nonce_t nonce; 49 | /* The uuid of this volume */ 50 | uuid_t uuid; 51 | /* The block offset to be applied to bios */ 52 | block_count_t bio_offset; 53 | /* The regions in ID order */ 54 | struct volume_region regions[VDO_VOLUME_REGION_COUNT]; 55 | /* The index config */ 56 | struct index_config index_config; 57 | } __packed; 58 | 59 | /* This volume geometry struct is used for sizing only */ 60 | struct volume_geometry_4_0 { 61 | /* The release version number of this volume */ 62 | release_version_number_t release_version; 63 | /* The nonce of this volume */ 64 | nonce_t nonce; 65 | /* The uuid of this volume */ 66 | uuid_t uuid; 67 | /* The regions in ID order */ 68 | struct volume_region regions[VDO_VOLUME_REGION_COUNT]; 69 | /* The index config */ 70 | struct index_config index_config; 71 | } __packed; 72 | 73 | /** 74 | * vdo_get_index_region_start() - Get the start of the index region from a 75 | * geometry. 76 | * @geometry: The geometry. 77 | * 78 | * Return: The start of the index region. 79 | */ 80 | static inline physical_block_number_t __must_check 81 | vdo_get_index_region_start(struct volume_geometry geometry) 82 | { 83 | return geometry.regions[VDO_INDEX_REGION].start_block; 84 | } 85 | 86 | /** 87 | * vdo_get_data_region_start() - Get the start of the data region from a 88 | * geometry. 89 | * @geometry: The geometry. 90 | * 91 | * Return: The start of the data region. 92 | */ 93 | static inline physical_block_number_t __must_check 94 | vdo_get_data_region_start(struct volume_geometry geometry) 95 | { 96 | return geometry.regions[VDO_DATA_REGION].start_block; 97 | } 98 | 99 | /** 100 | * vdo_get_index_region_size() - Get the size of the index region from a 101 | * geometry. 102 | * @geometry: The geometry. 103 | * 104 | * Return: The size of the index region. 105 | */ 106 | static inline physical_block_number_t __must_check 107 | vdo_get_index_region_size(struct volume_geometry geometry) 108 | { 109 | return vdo_get_data_region_start(geometry) - 110 | vdo_get_index_region_start(geometry); 111 | } 112 | 113 | int __must_check 114 | vdo_read_geometry_block(struct block_device *bdev, 115 | struct volume_geometry *geometry); 116 | 117 | #endif /* VOLUME_GEOMETRY_H */ 118 | -------------------------------------------------------------------------------- /vdo/volume-index-ops.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | #include "volume-index-ops.h" 6 | 7 | #include "compiler.h" 8 | #include "config.h" 9 | #include "errors.h" 10 | #include "geometry.h" 11 | #include "logger.h" 12 | #include "volume-index005.h" 13 | #include "volume-index006.h" 14 | #include "memory-alloc.h" 15 | #include "permassert.h" 16 | #include "uds.h" 17 | 18 | static INLINE bool uses_sparse(const struct configuration *config) 19 | { 20 | return is_sparse_geometry(config->geometry); 21 | } 22 | 23 | void get_volume_index_combined_stats(const struct volume_index *volume_index, 24 | struct volume_index_stats *stats) 25 | { 26 | struct volume_index_stats dense, sparse; 27 | 28 | get_volume_index_stats(volume_index, &dense, &sparse); 29 | stats->memory_allocated = 30 | dense.memory_allocated + sparse.memory_allocated; 31 | stats->rebalance_time = dense.rebalance_time + sparse.rebalance_time; 32 | stats->rebalance_count = 33 | dense.rebalance_count + sparse.rebalance_count; 34 | stats->record_count = dense.record_count + sparse.record_count; 35 | stats->collision_count = 36 | dense.collision_count + sparse.collision_count; 37 | stats->discard_count = dense.discard_count + sparse.discard_count; 38 | stats->overflow_count = dense.overflow_count + sparse.overflow_count; 39 | stats->num_lists = dense.num_lists + sparse.num_lists; 40 | stats->early_flushes = dense.early_flushes + sparse.early_flushes; 41 | } 42 | 43 | int make_volume_index(const struct configuration *config, 44 | uint64_t volume_nonce, 45 | struct volume_index **volume_index) 46 | { 47 | if (uses_sparse(config)) { 48 | return make_volume_index006(config, volume_nonce, 49 | volume_index); 50 | } else { 51 | return make_volume_index005(config, volume_nonce, 52 | volume_index); 53 | } 54 | } 55 | 56 | int compute_volume_index_save_blocks(const struct configuration *config, 57 | size_t block_size, 58 | uint64_t *block_count) 59 | { 60 | size_t num_bytes; 61 | int result = (uses_sparse(config) ? 62 | compute_volume_index_save_bytes006(config, 63 | &num_bytes) : 64 | compute_volume_index_save_bytes005(config, 65 | &num_bytes)); 66 | if (result != UDS_SUCCESS) { 67 | return result; 68 | } 69 | num_bytes += sizeof(struct delta_list_save_info); 70 | *block_count = DIV_ROUND_UP(num_bytes, block_size) + MAX_ZONES; 71 | return UDS_SUCCESS; 72 | } 73 | 74 | int save_volume_index(struct volume_index *volume_index, 75 | struct buffered_writer **writers, 76 | unsigned int num_writers) 77 | { 78 | int result = UDS_SUCCESS; 79 | unsigned int zone; 80 | 81 | for (zone = 0; zone < num_writers; ++zone) { 82 | result = start_saving_volume_index(volume_index, 83 | zone, 84 | writers[zone]); 85 | if (result != UDS_SUCCESS) { 86 | break; 87 | } 88 | 89 | result = finish_saving_volume_index(volume_index, zone); 90 | if (result != UDS_SUCCESS) { 91 | break; 92 | } 93 | 94 | result = write_guard_delta_list(writers[zone]); 95 | if (result != UDS_SUCCESS) { 96 | break; 97 | } 98 | 99 | result = flush_buffered_writer(writers[zone]); 100 | if (result != UDS_SUCCESS) { 101 | break; 102 | } 103 | } 104 | 105 | return result; 106 | } 107 | 108 | int load_volume_index(struct volume_index *volume_index, 109 | struct buffered_reader **readers, 110 | unsigned int num_readers) 111 | { 112 | /* Start by reading the "header" section of the stream */ 113 | int result = start_restoring_volume_index(volume_index, 114 | readers, 115 | num_readers); 116 | if (result != UDS_SUCCESS) { 117 | return result; 118 | } 119 | 120 | result = finish_restoring_volume_index(volume_index, 121 | readers, 122 | num_readers); 123 | if (result != UDS_SUCCESS) { 124 | abort_restoring_volume_index(volume_index); 125 | return result; 126 | } 127 | 128 | /* Check the final guard lists to make sure we read everything. */ 129 | result = check_guard_delta_lists(readers, num_readers); 130 | if (result != UDS_SUCCESS) { 131 | abort_restoring_volume_index(volume_index); 132 | } 133 | 134 | return result; 135 | } 136 | -------------------------------------------------------------------------------- /vdo/volume-index005.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef VOLUMEINDEX005_H 7 | #define VOLUMEINDEX005_H 1 8 | 9 | #include "volume-index-ops.h" 10 | 11 | /** 12 | * Make a new volume index. 13 | * 14 | * @param config The configuration of the volume index 15 | * @param volume_nonce The nonce used to authenticate the index 16 | * @param volume_index Location to hold new volume index ptr 17 | * 18 | * @return error code or UDS_SUCCESS 19 | **/ 20 | int __must_check make_volume_index005(const struct configuration *config, 21 | uint64_t volume_nonce, 22 | struct volume_index **volume_index); 23 | 24 | /** 25 | * Compute the number of bytes required to save a volume index of a given 26 | * configuration. 27 | * 28 | * @param config The configuration of the volume index 29 | * @param num_bytes The number of bytes required to save the volume index 30 | * 31 | * @return UDS_SUCCESS or an error code. 32 | **/ 33 | int __must_check 34 | compute_volume_index_save_bytes005(const struct configuration *config, 35 | size_t *num_bytes); 36 | 37 | #endif /* VOLUMEINDEX005_H */ 38 | -------------------------------------------------------------------------------- /vdo/volume-index006.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef VOLUMEINDEX006_H 7 | #define VOLUMEINDEX006_H 1 8 | 9 | #include "volume-index-ops.h" 10 | 11 | /** 12 | * Make a new volume index. 13 | * 14 | * @param config The configuration of the volume index 15 | * @param volume_nonce The nonce used to authenticate the index 16 | * @param volume_index Location to hold new volume index ptr 17 | * 18 | * @return error code or UDS_SUCCESS 19 | **/ 20 | int __must_check make_volume_index006(const struct configuration *config, 21 | uint64_t volume_nonce, 22 | struct volume_index **volume_index); 23 | 24 | /** 25 | * Compute the number of bytes required to save a volume index of a given 26 | * configuration. 27 | * 28 | * @param config The configuration of the volume index 29 | * @param num_bytes The number of bytes required to save the volume index 30 | * 31 | * @return UDS_SUCCESS or an error code. 32 | **/ 33 | int __must_check 34 | compute_volume_index_save_bytes006(const struct configuration *config, 35 | size_t *num_bytes); 36 | 37 | #endif /* VOLUMEINDEX006_H */ 38 | -------------------------------------------------------------------------------- /vdo/volume-store.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #include "geometry.h" 7 | #include "index-layout.h" 8 | #include "logger.h" 9 | #include "volume-store.h" 10 | 11 | 12 | void close_volume_store(struct volume_store *volume_store) 13 | { 14 | if (volume_store->vs_client != NULL) { 15 | dm_bufio_client_destroy(volume_store->vs_client); 16 | volume_store->vs_client = NULL; 17 | } 18 | } 19 | 20 | void destroy_volume_page(struct volume_page *volume_page) 21 | { 22 | release_volume_page(volume_page); 23 | } 24 | 25 | int initialize_volume_page(size_t page_size, struct volume_page *volume_page) 26 | { 27 | volume_page->vp_buffer = NULL; 28 | return UDS_SUCCESS; 29 | } 30 | 31 | int open_volume_store(struct volume_store *volume_store, 32 | struct index_layout *layout, 33 | unsigned int reserved_buffers __maybe_unused, 34 | size_t bytes_per_page) 35 | { 36 | return open_uds_volume_bufio(layout, bytes_per_page, reserved_buffers, 37 | &volume_store->vs_client); 38 | } 39 | 40 | void prefetch_volume_pages(const struct volume_store *vs __maybe_unused, 41 | unsigned int physical_page __maybe_unused, 42 | unsigned int page_count __maybe_unused) 43 | { 44 | dm_bufio_prefetch(vs->vs_client, physical_page, page_count); 45 | } 46 | 47 | int prepare_to_write_volume_page(const struct volume_store *volume_store 48 | __maybe_unused, 49 | unsigned int physical_page __maybe_unused, 50 | struct volume_page *volume_page 51 | __maybe_unused) 52 | { 53 | struct dm_buffer *buffer = NULL; 54 | byte *data; 55 | 56 | release_volume_page(volume_page); 57 | data = dm_bufio_new(volume_store->vs_client, physical_page, &buffer); 58 | if (IS_ERR(data)) { 59 | return -PTR_ERR(data); 60 | } 61 | volume_page->vp_buffer = buffer; 62 | return UDS_SUCCESS; 63 | } 64 | 65 | int read_volume_page(const struct volume_store *volume_store, 66 | unsigned int physical_page, 67 | struct volume_page *volume_page) 68 | { 69 | byte *data; 70 | 71 | release_volume_page(volume_page); 72 | data = dm_bufio_read(volume_store->vs_client, physical_page, 73 | &volume_page->vp_buffer); 74 | if (IS_ERR(data)) { 75 | return uds_log_warning_strerror(-PTR_ERR(data), 76 | "error reading physical page %u", 77 | physical_page); 78 | } 79 | return UDS_SUCCESS; 80 | } 81 | 82 | void release_volume_page(struct volume_page *volume_page __maybe_unused) 83 | { 84 | if (volume_page->vp_buffer != NULL) { 85 | dm_bufio_release(volume_page->vp_buffer); 86 | volume_page->vp_buffer = NULL; 87 | } 88 | } 89 | 90 | void swap_volume_pages(struct volume_page *volume_page1, 91 | struct volume_page *volume_page2) 92 | { 93 | struct volume_page temp = *volume_page1; 94 | *volume_page1 = *volume_page2; 95 | *volume_page2 = temp; 96 | } 97 | 98 | int sync_volume_store(const struct volume_store *volume_store) 99 | { 100 | int result = -dm_bufio_write_dirty_buffers(volume_store->vs_client); 101 | if (result != UDS_SUCCESS) { 102 | return uds_log_error_strerror(result, 103 | "cannot sync chapter to volume"); 104 | } 105 | return UDS_SUCCESS; 106 | } 107 | 108 | int write_volume_page(const struct volume_store *volume_store, 109 | unsigned int physical_page, 110 | struct volume_page *volume_page) 111 | { 112 | dm_bufio_mark_buffer_dirty(volume_page->vp_buffer); 113 | return UDS_SUCCESS; 114 | } 115 | -------------------------------------------------------------------------------- /vdo/workQueue.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright Red Hat 4 | */ 5 | 6 | #ifndef VDO_WORK_QUEUE_H 7 | #define VDO_WORK_QUEUE_H 8 | 9 | #include /* for TASK_COMM_LEN */ 10 | 11 | #include "funnel-queue.h" 12 | 13 | #include "kernel-types.h" 14 | #include "types.h" 15 | 16 | enum { 17 | MAX_VDO_WORK_QUEUE_NAME_LEN = TASK_COMM_LEN, 18 | }; 19 | 20 | struct vdo_work_queue_type { 21 | void (*start)(void *); 22 | void (*finish)(void *); 23 | enum vdo_completion_priority max_priority; 24 | enum vdo_completion_priority default_priority; 25 | }; 26 | 27 | int make_work_queue(const char *thread_name_prefix, 28 | const char *name, 29 | struct vdo_thread *owner, 30 | const struct vdo_work_queue_type *type, 31 | unsigned int thread_count, 32 | void *thread_privates[], 33 | struct vdo_work_queue **queue_ptr); 34 | 35 | void enqueue_work_queue(struct vdo_work_queue *queue, 36 | struct vdo_completion *completion); 37 | 38 | void finish_work_queue(struct vdo_work_queue *queue); 39 | 40 | void free_work_queue(struct vdo_work_queue *queue); 41 | 42 | void dump_work_queue(struct vdo_work_queue *queue); 43 | 44 | void dump_completion_to_buffer(struct vdo_completion *completion, 45 | char *buffer, 46 | size_t length); 47 | 48 | void *get_work_queue_private_data(void); 49 | struct vdo_work_queue *get_current_work_queue(void); 50 | struct vdo_thread *get_work_queue_owner(struct vdo_work_queue *queue); 51 | 52 | bool __must_check 53 | vdo_work_queue_type_is(struct vdo_work_queue *queue, 54 | const struct vdo_work_queue_type *type); 55 | 56 | #endif /* VDO_WORK_QUEUE_H */ 57 | --------------------------------------------------------------------------------