├── .gitignore ├── LICENSE ├── Makefile ├── README.rst ├── apfs.h ├── apfs_raw.h ├── btree.c ├── compress.c ├── dir.c ├── dkms.conf ├── extents.c ├── file.c ├── genver.sh ├── inode.c ├── key.c ├── libzbitmap.c ├── libzbitmap.h ├── lzfse ├── lzfse.h ├── lzfse_decode.c ├── lzfse_decode_base.c ├── lzfse_fse.c ├── lzfse_fse.h ├── lzfse_internal.h ├── lzvn_decode_base.c └── lzvn_decode_base.h ├── message.c ├── namei.c ├── node.c ├── object.c ├── snapshot.c ├── spaceman.c ├── super.c ├── symlink.c ├── transaction.c ├── unicode.c ├── unicode.h ├── xattr.c └── xfield.c /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | *.swp 3 | *.d 4 | *.o 5 | *.symvers 6 | *.mod.c 7 | *.order 8 | *.ko 9 | *.a 10 | *.mod 11 | tags 12 | version.h 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc. 5 | 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Library General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License 307 | along with this program; if not, write to the Free Software 308 | Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 309 | 310 | 311 | Also add information on how to contact you by electronic and paper mail. 312 | 313 | If the program is interactive, make it output a short notice like this 314 | when it starts in an interactive mode: 315 | 316 | Gnomovision version 69, Copyright (C) year name of author 317 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 318 | This is free software, and you are welcome to redistribute it 319 | under certain conditions; type `show c' for details. 320 | 321 | The hypothetical commands `show w' and `show c' should show the appropriate 322 | parts of the General Public License. Of course, the commands you use may 323 | be called something other than `show w' and `show c'; they could even be 324 | mouse-clicks or menu items--whatever suits your program. 325 | 326 | You should also get your employer (if you work as a programmer) or your 327 | school, if any, to sign a "copyright disclaimer" for the program, if 328 | necessary. Here is a sample; alter the names: 329 | 330 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 331 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 332 | 333 | , 1 April 1989 334 | Ty Coon, President of Vice 335 | 336 | This General Public License does not permit incorporating your program into 337 | proprietary programs. If your program is a subroutine library, you may 338 | consider it more useful to permit linking proprietary applications with the 339 | library. If this is what you want to do, use the GNU Library General 340 | Public License instead of this License. 341 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-2.0-only 2 | # 3 | # Makefile for the out-of-tree Linux APFS module. 4 | # 5 | 6 | KERNELRELEASE ?= $(shell uname -r) 7 | KERNEL_DIR ?= /lib/modules/$(KERNELRELEASE)/build 8 | PWD := $(shell pwd) 9 | 10 | obj-m = apfs.o 11 | apfs-y := btree.o compress.o dir.o extents.o file.o inode.o key.o libzbitmap.o \ 12 | lzfse/lzfse_decode.o lzfse/lzfse_decode_base.o lzfse/lzfse_fse.o \ 13 | lzfse/lzvn_decode_base.o message.o namei.o node.o object.o snapshot.o \ 14 | spaceman.o super.o symlink.o transaction.o unicode.o xattr.o xfield.o 15 | 16 | # If you want mounts to be writable by default, run the build as: 17 | # make CONFIG=-DCONFIG_APFS_RW_ALWAYS 18 | # This is risky and not generally recommended. 19 | ccflags-y += $(CONFIG) 20 | 21 | default: 22 | ./genver.sh 23 | make -C $(KERNEL_DIR) M=$(PWD) 24 | install: 25 | make -C $(KERNEL_DIR) M=$(PWD) modules_install 26 | clean: 27 | rm -f version.h 28 | make -C $(KERNEL_DIR) M=$(PWD) clean 29 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | =========================== 2 | Apple File System for Linux 3 | =========================== 4 | 5 | The Apple File System (APFS) is the copy-on-write filesystem currently used on 6 | all Apple devices. This module provides a degree of experimental support on 7 | Linux. It's intended to work with all kernel versions since 4.12, but testing 8 | is focused on 5.18 and above. 9 | 10 | To help test write support, a set of userland tools is also under development. 11 | The git tree can be retrieved from https://github.com/eafer/apfsprogs.git. 12 | 13 | Known limitations 14 | ================= 15 | 16 | This module is the result of reverse engineering and testing has been limited. 17 | If you make use of the write support, there is a real risk of data corruption, 18 | so mounts are read-only by default. Please report any issues that you find. 19 | 20 | Writes to fusion drives will likely never be supported, though I might 21 | reconsider this if people actually need it. Encryption is not yet implemented 22 | even in read-only mode. 23 | 24 | Reporting bugs 25 | ============== 26 | 27 | If you encounter any problem, the first thing you should do is run (as root):: 28 | 29 | dmesg | grep -i apfs 30 | 31 | to see all the error messages. If that doesn't help you, please file a github 32 | issue at https://github.com/eafer/linux-apfs-rw/issues. Or send me an email if 33 | you prefer. 34 | 35 | Build 36 | ===== 37 | 38 | Note that at this point the driver has been packaged for several distros, so 39 | it may be more convenient to install it from there instead of building it 40 | yourself. The distro packages are often outdated though, so be sure to check 41 | the version. 42 | 43 | In order to build a module out-of-tree, you will first need the Linux kernel 44 | headers. Depending on your distro, you can get them by running (as root):: 45 | 46 | # Debian/Ubuntu 47 | apt-get install linux-headers-$(uname -r) 48 | 49 | # Arch/Manjaro 50 | pacman -Sy linux-headers 51 | 52 | # RHEL/Rocky/CentOS/Fedora 53 | yum install kernel-headers kernel-devel 54 | 55 | Now you can just cd to the linux-apfs-rw directory and run:: 56 | 57 | make 58 | 59 | The resulting module is the apfs.ko file. Before you can use it you must insert 60 | it into the kernel, as well as its dependencies. Again as root:: 61 | 62 | modprobe libcrc32c 63 | insmod apfs.ko 64 | 65 | Mount 66 | ===== 67 | 68 | Like all filesystems, apfs is mounted with:: 69 | 70 | mount [-o options] device dir 71 | 72 | where ``device`` is the path to your device file or filesystem image, and 73 | ``dir`` is the mount point. The following options are accepted: 74 | 75 | ============ ================================================================= 76 | vol=n Volume number to mount. The default is volume 0. 77 | 78 | snap=label Volume snapshot to mount (in read-only mode). 79 | 80 | tier2=path Path to the tier 2 device. For fusion drives only. 81 | 82 | uid=n, gid=n Override on-disk inode ownership data with given uid/gid. 83 | 84 | cknodes Verify the checksum on all metadata nodes. Right now this has a 85 | severe performance cost, so it's not recommended. 86 | 87 | readwrite Enable the experimental write support. This may corrupt your 88 | container. 89 | ============ ================================================================= 90 | 91 | So for instance, if you want to mount volume number 2, and you want the metadata 92 | to be checked, you should run (as root):: 93 | 94 | mount -o cknodes,vol=2 device dir 95 | 96 | To unmount it, run:: 97 | 98 | umount dir 99 | 100 | Credits 101 | ======= 102 | 103 | Originally written by Ernesto A. Fernández , with 104 | several contributions from Gabriel Krisman Bertazi , 105 | Arnaud Ferraris and Stan Skowronek 106 | . For attribution details see the historical 107 | git tree at https://github.com/eafer/linux-apfs.git. 108 | 109 | Work was first based on reverse engineering done by others [1]_ [2]_, and later 110 | on the (very incomplete) official specification [3]_. Some parts of the code 111 | imitate the ext2 module, and to a lesser degree xfs, udf, gfs2 and hfsplus. 112 | 113 | .. [1] Hansen, K.H., Toolan, F., Decoding the APFS file system, Digital 114 | Investigation (2017), https://dx.doi.org/10.1016/j.diin.2017.07.003 115 | .. [2] https://github.com/sgan81/apfs-fuse 116 | .. [3] https://developer.apple.com/support/apple-file-system/Apple-File-System-Reference.pdf 117 | -------------------------------------------------------------------------------- /compress.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright (C) 2020 Corellium LLC 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include "apfs.h" 11 | #include "libzbitmap.h" 12 | #include "lzfse/lzfse.h" 13 | #include "lzfse/lzvn_decode_base.h" 14 | 15 | #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0) 16 | 17 | #include 18 | 19 | static inline void *kvmalloc(size_t size, gfp_t flags) 20 | { 21 | gfp_t kmalloc_flags = flags; 22 | void *ret; 23 | 24 | if ((flags & GFP_KERNEL) != GFP_KERNEL) 25 | return kmalloc(size, flags); 26 | 27 | if (size > PAGE_SIZE) 28 | kmalloc_flags |= __GFP_NOWARN | __GFP_NORETRY; 29 | 30 | ret = kmalloc(size, flags); 31 | if (ret || size < PAGE_SIZE) 32 | return ret; 33 | 34 | return vmalloc(size); 35 | } 36 | 37 | #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0) */ 38 | 39 | struct apfs_compress_file_data { 40 | struct apfs_compress_hdr hdr; 41 | u8 *buf; 42 | ssize_t bufblk; 43 | size_t bufsize; 44 | struct mutex mtx; 45 | struct super_block *sb; 46 | struct apfs_compressed_data cdata; 47 | }; 48 | 49 | static inline int apfs_compress_is_rsrc(u32 algo) 50 | { 51 | return (algo & 1) == 0; 52 | } 53 | 54 | static inline bool apfs_compress_is_supported(u32 algo) 55 | { 56 | switch (algo) { 57 | case APFS_COMPRESS_ZLIB_RSRC: 58 | case APFS_COMPRESS_ZLIB_ATTR: 59 | case APFS_COMPRESS_LZVN_RSRC: 60 | case APFS_COMPRESS_LZVN_ATTR: 61 | case APFS_COMPRESS_PLAIN_RSRC: 62 | case APFS_COMPRESS_PLAIN_ATTR: 63 | case APFS_COMPRESS_LZFSE_RSRC: 64 | case APFS_COMPRESS_LZFSE_ATTR: 65 | case APFS_COMPRESS_LZBITMAP_RSRC: 66 | case APFS_COMPRESS_LZBITMAP_ATTR: 67 | return true; 68 | default: 69 | /* Once will usually be enough, don't flood the console */ 70 | pr_err_once("APFS: unsupported compression algorithm (%u)\n", algo); 71 | return false; 72 | } 73 | } 74 | 75 | static int apfs_compress_file_open(struct inode *inode, struct file *filp) 76 | { 77 | struct super_block *sb = inode->i_sb; 78 | struct apfs_nxsb_info *nxi = APFS_NXI(sb); 79 | struct apfs_compress_file_data *fd; 80 | ssize_t res; 81 | bool is_rsrc; 82 | 83 | /* 84 | * The official implementation seems to transparently decompress files 85 | * when you write to them. Doing that atomically inside the kernel is 86 | * probably a chore, so for now I'll just leave it to the user to make 87 | * an uncompressed copy themselves and replace the original. I might 88 | * fix this in the future, but only if people complain (TODO). 89 | */ 90 | if (filp->f_mode & FMODE_WRITE) { 91 | apfs_warn(sb, "writes to compressed files are not supported"); 92 | apfs_warn(sb, "you can work with a copy of the file instead"); 93 | return -EOPNOTSUPP; 94 | } 95 | 96 | if (!(filp->f_flags & O_LARGEFILE) && i_size_read(inode) > MAX_NON_LFS) 97 | return -EOVERFLOW; 98 | 99 | fd = kzalloc(sizeof(*fd), GFP_KERNEL); 100 | if (!fd) 101 | return -ENOMEM; 102 | mutex_init(&fd->mtx); 103 | fd->sb = sb; 104 | 105 | down_read(&nxi->nx_big_sem); 106 | 107 | res = ____apfs_xattr_get(inode, APFS_XATTR_NAME_COMPRESSED, &fd->hdr, sizeof(fd->hdr), 0); 108 | if (res != sizeof(fd->hdr)) { 109 | apfs_err(sb, "decmpfs header read failed"); 110 | goto fail; 111 | } 112 | 113 | if (!apfs_compress_is_supported(le32_to_cpu(fd->hdr.algo))) { 114 | res = -EOPNOTSUPP; 115 | goto fail; 116 | } 117 | 118 | fd->buf = kvmalloc(APFS_COMPRESS_BLOCK, GFP_KERNEL); 119 | if (!fd->buf) { 120 | res = -ENOMEM; 121 | goto fail; 122 | } 123 | fd->bufblk = -1; 124 | 125 | is_rsrc = apfs_compress_is_rsrc(le32_to_cpu(fd->hdr.algo)); 126 | res = apfs_xattr_get_compressed_data(inode, is_rsrc ? APFS_XATTR_NAME_RSRC_FORK : APFS_XATTR_NAME_COMPRESSED, &fd->cdata); 127 | if (res) { 128 | apfs_err(sb, "failed to get compressed data"); 129 | goto fail; 130 | } 131 | 132 | up_read(&nxi->nx_big_sem); 133 | 134 | filp->private_data = fd; 135 | return 0; 136 | 137 | fail: 138 | apfs_release_compressed_data(&fd->cdata); 139 | if (fd->buf) 140 | kvfree(fd->buf); 141 | up_read(&nxi->nx_big_sem); 142 | kfree(fd); 143 | if (res > 0) 144 | res = -EINVAL; 145 | return res; 146 | } 147 | 148 | static int apfs_compress_file_read_block(struct apfs_compress_file_data *fd, loff_t block) 149 | { 150 | struct super_block *sb = fd->sb; 151 | struct apfs_compressed_data *comp_data = &fd->cdata; 152 | u8 *cdata = NULL; 153 | u8 *tmp = fd->buf; 154 | u32 doffs = 0, coffs; 155 | size_t csize, bsize; 156 | int res = 0; 157 | 158 | if (apfs_compress_is_rsrc(le32_to_cpu(fd->hdr.algo)) && 159 | le32_to_cpu(fd->hdr.algo) != APFS_COMPRESS_LZBITMAP_RSRC && 160 | le32_to_cpu(fd->hdr.algo) != APFS_COMPRESS_LZVN_RSRC && 161 | le32_to_cpu(fd->hdr.algo) != APFS_COMPRESS_LZFSE_RSRC) { 162 | struct apfs_compress_rsrc_hdr hdr = {0}; 163 | struct apfs_compress_rsrc_data cd = {0}; 164 | struct apfs_compress_rsrc_block blk = {0}; 165 | u32 blk_off; 166 | 167 | res = apfs_compressed_data_read(comp_data, &hdr, sizeof(hdr), 0 /* offset */); 168 | if (res) { 169 | apfs_err(sb, "failed to read resource header"); 170 | return res; 171 | } 172 | 173 | doffs = be32_to_cpu(hdr.data_offs); 174 | res = apfs_compressed_data_read(comp_data, &cd, sizeof(cd), doffs); 175 | if (res) { 176 | apfs_err(sb, "failed to read resource data header"); 177 | return res; 178 | } 179 | if (block >= le32_to_cpu(cd.num)) 180 | return 0; 181 | 182 | blk_off = doffs + sizeof(cd) + sizeof(blk) * block; 183 | res = apfs_compressed_data_read(comp_data, &blk, sizeof(blk), blk_off); 184 | if (res) { 185 | apfs_err(sb, "failed to read resource block metadata"); 186 | return res; 187 | } 188 | 189 | bsize = le64_to_cpu(fd->hdr.size) - block * APFS_COMPRESS_BLOCK; 190 | if (bsize > APFS_COMPRESS_BLOCK) 191 | bsize = APFS_COMPRESS_BLOCK; 192 | 193 | csize = le32_to_cpu(blk.size); 194 | coffs = le32_to_cpu(blk.offs) + 4; 195 | } else if (apfs_compress_is_rsrc(le32_to_cpu(fd->hdr.algo))) { 196 | __le32 blks[2]; 197 | u32 blk_off; 198 | 199 | blk_off = doffs + sizeof(__le32) * block; 200 | res = apfs_compressed_data_read(comp_data, blks, sizeof(blks), blk_off); 201 | if (res) { 202 | apfs_err(sb, "failed to read resource block metadata"); 203 | return res; 204 | } 205 | 206 | bsize = le64_to_cpu(fd->hdr.size) - block * APFS_COMPRESS_BLOCK; 207 | if (bsize > APFS_COMPRESS_BLOCK) 208 | bsize = APFS_COMPRESS_BLOCK; 209 | 210 | coffs = le32_to_cpu(blks[0]); 211 | csize = le32_to_cpu(blks[1]) - coffs; 212 | } else { 213 | /* 214 | * I think attr compression is only for single-block files, in 215 | * fact none of these files ever seem to decompress to more than 216 | * 2048 bytes. 217 | */ 218 | bsize = le64_to_cpu(fd->hdr.size); 219 | if (block != 0 || bsize > APFS_COMPRESS_BLOCK) { 220 | apfs_err(sb, "file too big for inline compression"); 221 | return -EFSCORRUPTED; 222 | } 223 | 224 | /* The first few bytes are the decmpfs header */ 225 | coffs = sizeof(struct apfs_compress_hdr); 226 | csize = comp_data->size - sizeof(struct apfs_compress_hdr); 227 | } 228 | 229 | cdata = kvmalloc(csize, GFP_KERNEL); 230 | if (!cdata) 231 | return -ENOMEM; 232 | res = apfs_compressed_data_read(comp_data, cdata, csize, doffs + coffs); 233 | if (res) { 234 | apfs_err(sb, "failed to read compressed block"); 235 | goto fail; 236 | } 237 | 238 | switch (le32_to_cpu(fd->hdr.algo)) { 239 | case APFS_COMPRESS_ZLIB_RSRC: 240 | case APFS_COMPRESS_ZLIB_ATTR: 241 | if (cdata[0] == 0x78 && csize >= 2) { 242 | res = zlib_inflate_blob(tmp, bsize, cdata + 2, csize - 2); 243 | if (res <= 0) { 244 | apfs_err(sb, "zlib decompression failed"); 245 | goto fail; 246 | } 247 | bsize = res; 248 | res = 0; 249 | } else if ((cdata[0] & 0x0F) == 0x0F) { 250 | memcpy(tmp, &cdata[1], csize - 1); 251 | bsize = csize - 1; 252 | } else { 253 | apfs_err(sb, "zlib decompression failed"); 254 | res = -EINVAL; 255 | goto fail; 256 | } 257 | break; 258 | case APFS_COMPRESS_LZVN_RSRC: 259 | case APFS_COMPRESS_LZVN_ATTR: 260 | if (cdata[0] == 0x06) { 261 | memcpy(tmp, &cdata[1], csize - 1); 262 | bsize = csize - 1; 263 | } else { 264 | lzvn_decoder_state dstate = {0}; 265 | 266 | dstate.src = cdata; 267 | dstate.src_end = dstate.src + csize; 268 | dstate.dst = dstate.dst_begin = tmp; 269 | dstate.dst_end = dstate.dst + bsize; 270 | lzvn_decode(&dstate); 271 | bsize = dstate.dst - tmp; 272 | } 273 | break; 274 | case APFS_COMPRESS_LZBITMAP_RSRC: 275 | case APFS_COMPRESS_LZBITMAP_ATTR: 276 | if (cdata[0] == 0x5a) { 277 | res = zbm_decompress(tmp, bsize, cdata, csize, &bsize); 278 | if (res < 0) { 279 | apfs_err(sb, "lzbitmap decompression failed"); 280 | goto fail; 281 | } 282 | res = 0; 283 | } else if ((cdata[0] & 0x0F) == 0x0F) { 284 | memcpy(tmp, &cdata[1], csize - 1); 285 | bsize = csize - 1; 286 | } else { 287 | apfs_err(sb, "lzbitmap decompression failed"); 288 | res = -EINVAL; 289 | goto fail; 290 | } 291 | break; 292 | case APFS_COMPRESS_LZFSE_RSRC: 293 | case APFS_COMPRESS_LZFSE_ATTR: 294 | if (cdata[0] == 0x62 && csize >= 2) { 295 | res = lzfse_decode_buffer(tmp, bsize, cdata, csize, NULL); 296 | if (res == 0) { 297 | apfs_err(sb, "lzfse decompression failed"); 298 | /* Could be ENOMEM too... */ 299 | res = -EINVAL; 300 | goto fail; 301 | } 302 | bsize = res; 303 | res = 0; 304 | } else { 305 | /* cdata[0] == 0xff, apparently */ 306 | memcpy(tmp, &cdata[1], csize - 1); 307 | bsize = csize - 1; 308 | } 309 | break; 310 | case APFS_COMPRESS_PLAIN_RSRC: 311 | case APFS_COMPRESS_PLAIN_ATTR: 312 | memcpy(tmp, &cdata[1], csize - 1); 313 | bsize = csize - 1; 314 | break; 315 | default: 316 | res = -EINVAL; 317 | goto fail; 318 | } 319 | fd->bufblk = block; 320 | fd->bufsize = bsize; 321 | fail: 322 | kvfree(cdata); 323 | return res; 324 | } 325 | 326 | static int apfs_compress_file_release(struct inode *inode, struct file *filp) 327 | { 328 | struct apfs_compress_file_data *fd = filp->private_data; 329 | 330 | apfs_release_compressed_data(&fd->cdata); 331 | if (fd->buf) 332 | kvfree(fd->buf); 333 | kfree(fd); 334 | return 0; 335 | } 336 | 337 | static ssize_t apfs_compress_file_read_from_block(struct apfs_compress_file_data *fd, char *buf, size_t size, loff_t off) 338 | { 339 | struct super_block *sb = fd->sb; 340 | struct apfs_nxsb_info *nxi = APFS_NXI(sb); 341 | struct apfs_compressed_data cdata = fd->cdata; 342 | loff_t block; 343 | size_t bsize; 344 | ssize_t res; 345 | 346 | /* 347 | * Request reads of all blocks before actually working with any of them. 348 | * The compressed data is typically small enough that this is effective. 349 | * It would be much better to make an inode for the xattr dstream and 350 | * work with readahead as usual, but I'm not confident I can get that 351 | * right (TODO). 352 | */ 353 | if (cdata.has_dstream && off == 0) { 354 | down_read(&nxi->nx_big_sem); 355 | apfs_nonsparse_dstream_preread(cdata.dstream); 356 | up_read(&nxi->nx_big_sem); 357 | } 358 | 359 | if (off >= le64_to_cpu(fd->hdr.size)) 360 | return 0; 361 | if (size > le64_to_cpu(fd->hdr.size) - off) 362 | size = le64_to_cpu(fd->hdr.size) - off; 363 | 364 | block = off / APFS_COMPRESS_BLOCK; 365 | off -= block * APFS_COMPRESS_BLOCK; 366 | if (block != fd->bufblk) { 367 | down_read(&nxi->nx_big_sem); 368 | res = apfs_compress_file_read_block(fd, block); 369 | up_read(&nxi->nx_big_sem); 370 | if (res) { 371 | apfs_err(sb, "failed to read block into buffer"); 372 | return res; 373 | } 374 | } 375 | bsize = fd->bufsize; 376 | 377 | if (bsize < off) 378 | return 0; 379 | bsize -= off; 380 | if (size > bsize) 381 | size = bsize; 382 | memcpy(buf, fd->buf + off, size); 383 | return size; 384 | } 385 | 386 | static ssize_t apfs_compress_file_read_page(struct file *filp, char *buf, loff_t off) 387 | { 388 | struct apfs_compress_file_data *fd = filp->private_data; 389 | loff_t step; 390 | ssize_t block, res; 391 | size_t size = PAGE_SIZE; 392 | 393 | step = 0; 394 | while (step < size) { 395 | block = APFS_COMPRESS_BLOCK - ((off + step) & (APFS_COMPRESS_BLOCK - 1)); 396 | if (block > size - step) 397 | block = size - step; 398 | mutex_lock(&fd->mtx); 399 | res = apfs_compress_file_read_from_block(fd, buf + step, block, off + step); 400 | mutex_unlock(&fd->mtx); 401 | if (res < block) { 402 | if (res < 0 && !step) 403 | return res; 404 | step += res > 0 ? res : 0; 405 | break; 406 | } 407 | step += block; 408 | } 409 | return step; 410 | } 411 | 412 | #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0) || RHEL_VERSION_GE(9, 3) 413 | static int apfs_compress_read_folio(struct file *filp, struct folio *folio) 414 | { 415 | struct page *page = &folio->page; 416 | #else 417 | static int apfs_compress_readpage(struct file *filp, struct page *page) 418 | { 419 | #endif 420 | char *addr = NULL; 421 | ssize_t ret; 422 | loff_t off; 423 | 424 | /* Mostly copied from ext4_read_inline_page() */ 425 | off = page->index << PAGE_SHIFT; 426 | addr = kmap(page); 427 | ret = apfs_compress_file_read_page(filp, addr, off); 428 | flush_dcache_page(page); 429 | kunmap(page); 430 | if (ret >= 0) { 431 | zero_user_segment(page, ret, PAGE_SIZE); 432 | SetPageUptodate(page); 433 | ret = 0; 434 | } 435 | 436 | unlock_page(page); 437 | return ret; 438 | } 439 | 440 | const struct address_space_operations apfs_compress_aops = { 441 | #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0) || RHEL_VERSION_GE(9, 3) 442 | .read_folio = apfs_compress_read_folio, 443 | #else 444 | .readpage = apfs_compress_readpage, 445 | #endif 446 | }; 447 | 448 | /* TODO: these operations are all happening without proper locks */ 449 | const struct file_operations apfs_compress_file_operations = { 450 | .open = apfs_compress_file_open, 451 | .llseek = generic_file_llseek, 452 | .read_iter = generic_file_read_iter, 453 | .release = apfs_compress_file_release, 454 | .mmap = apfs_file_mmap, 455 | }; 456 | 457 | int apfs_compress_get_size(struct inode *inode, loff_t *size) 458 | { 459 | struct apfs_compress_hdr hdr; 460 | int res = ____apfs_xattr_get(inode, APFS_XATTR_NAME_COMPRESSED, &hdr, sizeof(hdr), 0); 461 | 462 | if (res < 0) 463 | return res; 464 | if (res != sizeof(hdr)) { 465 | apfs_err(inode->i_sb, "decmpfs header read failed"); 466 | return 1; 467 | } 468 | 469 | if (!apfs_compress_is_supported(le32_to_cpu(hdr.algo))) 470 | return 1; 471 | 472 | *size = le64_to_cpu(hdr.size); 473 | return 0; 474 | } 475 | -------------------------------------------------------------------------------- /dkms.conf: -------------------------------------------------------------------------------- 1 | PACKAGE_NAME="linux-apfs-rw" 2 | PACKAGE_VERSION="0.3.14" 3 | 4 | BUILT_MODULE_NAME[0]="apfs" 5 | DEST_MODULE_LOCATION[0]="/extra" 6 | 7 | AUTOINSTALL="yes" 8 | PRE_BUILD="genver.sh" 9 | -------------------------------------------------------------------------------- /file.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright (C) 2018 Ernesto A. Fernández 4 | */ 5 | 6 | #include "apfs.h" 7 | 8 | #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 8, 0) 9 | #include 10 | #endif 11 | #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 0) 12 | typedef int vm_fault_t; 13 | #endif 14 | 15 | #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0) 16 | static void wait_for_stable_page(struct page *page) 17 | { 18 | return folio_wait_stable(page_folio(page)); 19 | } 20 | #endif 21 | 22 | #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) 23 | static vm_fault_t apfs_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf) 24 | { 25 | #else 26 | static vm_fault_t apfs_page_mkwrite(struct vm_fault *vmf) 27 | { 28 | struct vm_area_struct *vma = vmf->vma; 29 | #endif 30 | struct page *page = vmf->page; 31 | #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 7, 0) 32 | struct folio *folio; 33 | #endif 34 | struct inode *inode = file_inode(vma->vm_file); 35 | struct super_block *sb = inode->i_sb; 36 | struct buffer_head *bh, *head; 37 | vm_fault_t ret = VM_FAULT_LOCKED; 38 | unsigned int blocksize, block_start, len; 39 | u64 size; 40 | int err = 0; 41 | 42 | sb_start_pagefault(inode->i_sb); 43 | file_update_time(vma->vm_file); 44 | 45 | err = apfs_transaction_start(sb, APFS_TRANS_REG); 46 | if (err) 47 | goto out; 48 | apfs_inode_join_transaction(sb, inode); 49 | 50 | err = apfs_inode_create_exclusive_dstream(inode); 51 | if (err) { 52 | apfs_err(sb, "dstream creation failed for ino 0x%llx", apfs_ino(inode)); 53 | goto out_abort; 54 | } 55 | 56 | lock_page(page); 57 | wait_for_stable_page(page); 58 | if (page->mapping != inode->i_mapping) { 59 | ret = VM_FAULT_NOPAGE; 60 | goto out_unlock; 61 | } 62 | 63 | if (!page_has_buffers(page)) { 64 | #if LINUX_VERSION_CODE < KERNEL_VERSION(6, 7, 0) 65 | create_empty_buffers(page, sb->s_blocksize, 0); 66 | #else 67 | folio = page_folio(page); 68 | bh = folio_buffers(folio); 69 | if (!bh) 70 | bh = create_empty_buffers(folio, sb->s_blocksize, 0); 71 | #endif 72 | } 73 | 74 | size = i_size_read(inode); 75 | if (page->index == size >> PAGE_SHIFT) 76 | len = size & ~PAGE_MASK; 77 | else 78 | len = PAGE_SIZE; 79 | 80 | /* The blocks were read on the fault, mark them as unmapped for CoW */ 81 | head = page_buffers(page); 82 | blocksize = head->b_size; 83 | for (bh = head, block_start = 0; bh != head || !block_start; 84 | block_start += blocksize, bh = bh->b_this_page) { 85 | if (len > block_start) { 86 | /* If it's not a hole, the fault read it already */ 87 | ASSERT(!buffer_mapped(bh) || buffer_uptodate(bh)); 88 | if (buffer_trans(bh)) 89 | continue; 90 | clear_buffer_mapped(bh); 91 | } 92 | } 93 | unlock_page(page); /* XXX: race? */ 94 | 95 | err = block_page_mkwrite(vma, vmf, apfs_get_new_block); 96 | if (err) { 97 | apfs_err(sb, "mkwrite failed for ino 0x%llx", apfs_ino(inode)); 98 | goto out_abort; 99 | } 100 | set_page_dirty(page); 101 | 102 | /* An immediate commit would leave the page unlocked */ 103 | APFS_SB(sb)->s_nxi->nx_transaction.t_state |= APFS_NX_TRANS_DEFER_COMMIT; 104 | 105 | err = apfs_transaction_commit(sb); 106 | if (err) 107 | goto out_unlock; 108 | goto out; 109 | 110 | out_unlock: 111 | unlock_page(page); 112 | out_abort: 113 | apfs_transaction_abort(sb); 114 | out: 115 | if (err) 116 | #if LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0) && !RHEL_VERSION_GE(9, 5) 117 | ret = block_page_mkwrite_return(err); 118 | #else 119 | ret = vmf_fs_error(err); 120 | #endif 121 | sb_end_pagefault(inode->i_sb); 122 | return ret; 123 | } 124 | 125 | static const struct vm_operations_struct apfs_file_vm_ops = { 126 | .fault = filemap_fault, 127 | .map_pages = filemap_map_pages, 128 | .page_mkwrite = apfs_page_mkwrite, 129 | }; 130 | 131 | int apfs_file_mmap(struct file *file, struct vm_area_struct *vma) 132 | { 133 | struct address_space *mapping = file->f_mapping; 134 | 135 | #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0) || RHEL_VERSION_GE(9, 3) 136 | if (!mapping->a_ops->read_folio) 137 | #else 138 | if (!mapping->a_ops->readpage) 139 | #endif 140 | return -ENOEXEC; 141 | file_accessed(file); 142 | vma->vm_ops = &apfs_file_vm_ops; 143 | return 0; 144 | } 145 | 146 | /* 147 | * Just flush the whole transaction for now (TODO), since that's technically 148 | * correct and easy to implement. 149 | */ 150 | int apfs_fsync(struct file *file, loff_t start, loff_t end, int datasync) 151 | { 152 | struct inode *inode = file->f_mapping->host; 153 | struct super_block *sb = inode->i_sb; 154 | 155 | return apfs_sync_fs(sb, true /* wait */); 156 | } 157 | 158 | #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 8, 0) 159 | static ssize_t apfs_copy_file_range(struct file *src_file, loff_t src_off, 160 | struct file *dst_file, loff_t dst_off, 161 | size_t len, unsigned int flags) 162 | { 163 | return (splice_copy_file_range(src_file, src_off, 164 | dst_file, dst_off, len)); 165 | } 166 | #endif 167 | 168 | const struct file_operations apfs_file_operations = { 169 | .llseek = generic_file_llseek, 170 | .read_iter = generic_file_read_iter, 171 | .write_iter = generic_file_write_iter, 172 | .mmap = apfs_file_mmap, 173 | .open = generic_file_open, 174 | .fsync = apfs_fsync, 175 | .unlocked_ioctl = apfs_file_ioctl, 176 | 177 | #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 8, 0) 178 | .copy_file_range = apfs_copy_file_range, 179 | #elif LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0) 180 | .copy_file_range = generic_copy_file_range, 181 | #endif 182 | 183 | #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0) 184 | .remap_file_range = apfs_remap_file_range, 185 | #else 186 | .clone_file_range = apfs_clone_file_range, 187 | #endif 188 | 189 | #if LINUX_VERSION_CODE < KERNEL_VERSION(6, 5, 0) 190 | .splice_read = generic_file_splice_read, 191 | #else 192 | .splice_read = filemap_splice_read, 193 | #endif 194 | .splice_write = iter_file_splice_write, 195 | }; 196 | 197 | #if LINUX_VERSION_CODE == KERNEL_VERSION(5, 3, 0) 198 | /* 199 | * This is needed mainly to test clones with xfstests, so we only support the 200 | * kernel version I use during testing. TODO: support all kernel versions. 201 | */ 202 | int apfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len) 203 | { 204 | return generic_block_fiemap(inode, fieinfo, start, len, apfs_get_block); 205 | } 206 | #endif 207 | 208 | const struct inode_operations apfs_file_inode_operations = { 209 | .getattr = apfs_getattr, 210 | .listxattr = apfs_listxattr, 211 | .setattr = apfs_setattr, 212 | .update_time = apfs_update_time, 213 | #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 13, 0) 214 | .fileattr_get = apfs_fileattr_get, 215 | .fileattr_set = apfs_fileattr_set, 216 | #endif 217 | #if LINUX_VERSION_CODE == KERNEL_VERSION(5, 3, 0) 218 | .fiemap = apfs_fiemap, 219 | #endif 220 | }; 221 | -------------------------------------------------------------------------------- /genver.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # SPDX-License-Identifier: GPL-2.0-only 3 | # 4 | # Script to generate the module version header 5 | # 6 | 7 | if command -v git >/dev/null 2>&1 && [ -d .git ]; then 8 | GIT_COMMIT=$(git describe HEAD | tail -c 9) 9 | else 10 | GIT_COMMIT="$(grep PACKAGE_VERSION dkms.conf | cut -d '"' -f2)?" 11 | fi 12 | 13 | printf '#define GIT_COMMIT\t"%s"\n' "$GIT_COMMIT" > version.h 14 | -------------------------------------------------------------------------------- /key.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright (C) 2018 Ernesto A. Fernández 4 | */ 5 | 6 | #include 7 | #include "apfs.h" 8 | #include "unicode.h" 9 | 10 | /** 11 | * apfs_filename_cmp - Normalize and compare two APFS filenames 12 | * @sb: filesystem superblock 13 | * @name1: first name to compare 14 | * @len1: length of @name1 15 | * @name2: second name to compare 16 | * @len2: length of the @name2 17 | * 18 | * Returns 0 if @name1 and @name2 are equal, or non-zero otherwise. 19 | */ 20 | int apfs_filename_cmp(struct super_block *sb, 21 | const char *name1, unsigned int len1, 22 | const char *name2, unsigned int len2) 23 | { 24 | struct apfs_unicursor cursor1, cursor2; 25 | bool case_fold = apfs_is_case_insensitive(sb); 26 | 27 | if (!apfs_is_normalization_insensitive(sb)) { 28 | if (len1 != len2) 29 | return -1; 30 | return memcmp(name1, name2, len1); 31 | } 32 | 33 | apfs_init_unicursor(&cursor1, name1, len1); 34 | apfs_init_unicursor(&cursor2, name2, len2); 35 | 36 | while (1) { 37 | unicode_t uni1, uni2; 38 | 39 | uni1 = apfs_normalize_next(&cursor1, case_fold); 40 | uni2 = apfs_normalize_next(&cursor2, case_fold); 41 | 42 | if (uni1 != uni2) 43 | return uni1 < uni2 ? -1 : 1; 44 | if (!uni1) 45 | return 0; 46 | } 47 | } 48 | 49 | /** 50 | * apfs_keycmp - Compare two keys 51 | * @k1: first key to compare 52 | * @k2: second key to compare 53 | * 54 | * returns 0 if @k1 and @k2 are equal 55 | * < 0 if @k1 comes before @k2 in the btree 56 | * > 0 if @k1 comes after @k2 in the btree 57 | */ 58 | int apfs_keycmp(struct apfs_key *k1, struct apfs_key *k2) 59 | { 60 | if (k1->id != k2->id) 61 | return k1->id < k2->id ? -1 : 1; 62 | if (k1->type != k2->type) 63 | return k1->type < k2->type ? -1 : 1; 64 | if (k1->number != k2->number) 65 | return k1->number < k2->number ? -1 : 1; 66 | if (!k1->name || !k2->name) 67 | return 0; 68 | 69 | /* Normalization seems to be ignored here, even for directory records */ 70 | return strcmp(k1->name, k2->name); 71 | } 72 | 73 | /** 74 | * apfs_read_cat_key - Parse an on-disk catalog key 75 | * @raw: pointer to the raw key 76 | * @size: size of the raw key 77 | * @key: apfs_key structure to store the result 78 | * @hashed: are the directory records hashed? 79 | * 80 | * Returns 0 on success, or a negative error code otherwise. 81 | */ 82 | int apfs_read_cat_key(void *raw, int size, struct apfs_key *key, bool hashed) 83 | { 84 | if (size < sizeof(struct apfs_key_header)) { 85 | apfs_err(NULL, "bad key length (%d)", size); 86 | return -EFSCORRUPTED; 87 | } 88 | key->id = apfs_cat_cnid((struct apfs_key_header *)raw); 89 | key->type = apfs_cat_type((struct apfs_key_header *)raw); 90 | 91 | switch (key->type) { 92 | case APFS_TYPE_DIR_REC: 93 | if (hashed) { 94 | if (size < sizeof(struct apfs_drec_hashed_key) + 1 || 95 | *((char *)raw + size - 1) != 0) { 96 | /* Filename must have NULL-termination */ 97 | apfs_err(NULL, "invalid drec key (%d)", size); 98 | return -EFSCORRUPTED; 99 | } 100 | /* Name length is not used in key comparisons, only the hash */ 101 | key->number = le32_to_cpu( 102 | ((struct apfs_drec_hashed_key *)raw)->name_len_and_hash) & 103 | APFS_DREC_HASH_MASK; 104 | key->name = ((struct apfs_drec_hashed_key *)raw)->name; 105 | } else { 106 | if (size < sizeof(struct apfs_drec_key) + 1 || 107 | *((char *)raw + size - 1) != 0) { 108 | /* Filename must have NULL-termination */ 109 | apfs_err(NULL, "invalid drec key (%d)", size); 110 | return -EFSCORRUPTED; 111 | } 112 | /* There's no hash */ 113 | key->number = 0; 114 | key->name = ((struct apfs_drec_key *)raw)->name; 115 | } 116 | break; 117 | case APFS_TYPE_XATTR: 118 | if (size < sizeof(struct apfs_xattr_key) + 1 || 119 | *((char *)raw + size - 1) != 0) { 120 | /* xattr name must have NULL-termination */ 121 | apfs_err(NULL, "invalid xattr key (%d)", size); 122 | return -EFSCORRUPTED; 123 | } 124 | key->number = 0; 125 | key->name = ((struct apfs_xattr_key *)raw)->name; 126 | break; 127 | case APFS_TYPE_FILE_EXTENT: 128 | if (size != sizeof(struct apfs_file_extent_key)) { 129 | apfs_err(NULL, "bad key length (%d)", size); 130 | return -EFSCORRUPTED; 131 | } 132 | key->number = le64_to_cpu( 133 | ((struct apfs_file_extent_key *)raw)->logical_addr); 134 | key->name = NULL; 135 | break; 136 | case APFS_TYPE_SIBLING_LINK: 137 | if (size != sizeof(struct apfs_sibling_link_key)) { 138 | apfs_err(NULL, "bad key length (%d)", size); 139 | return -EFSCORRUPTED; 140 | } 141 | key->number = le64_to_cpu( 142 | ((struct apfs_sibling_link_key *)raw)->sibling_id); 143 | key->name = NULL; 144 | break; 145 | default: 146 | key->number = 0; 147 | key->name = NULL; 148 | break; 149 | } 150 | 151 | return 0; 152 | } 153 | 154 | int apfs_read_fext_key(void *raw, int size, struct apfs_key *key) 155 | { 156 | struct apfs_fext_tree_key *raw_key; 157 | 158 | if (size != sizeof(*raw_key)) { 159 | apfs_err(NULL, "bad key length (%d)", size); 160 | return -EFSCORRUPTED; 161 | } 162 | raw_key = raw; 163 | 164 | key->id = le64_to_cpu(raw_key->private_id); 165 | key->type = 0; 166 | key->number = le64_to_cpu(raw_key->logical_addr); 167 | key->name = NULL; 168 | return 0; 169 | } 170 | 171 | /** 172 | * apfs_read_free_queue_key - Parse an on-disk free queue key 173 | * @raw: pointer to the raw key 174 | * @size: size of the raw key 175 | * @key: apfs_key structure to store the result 176 | * 177 | * Returns 0 on success, or a negative error code otherwise. 178 | */ 179 | int apfs_read_free_queue_key(void *raw, int size, struct apfs_key *key) 180 | { 181 | struct apfs_spaceman_free_queue_key *raw_key; 182 | 183 | if (size != sizeof(*raw_key)) { 184 | apfs_err(NULL, "bad key length (%d)", size); 185 | return -EFSCORRUPTED; 186 | } 187 | raw_key = raw; 188 | 189 | key->id = le64_to_cpu(raw_key->sfqk_xid); 190 | key->type = 0; 191 | key->number = le64_to_cpu(raw_key->sfqk_paddr); 192 | key->name = NULL; 193 | 194 | return 0; 195 | } 196 | 197 | /** 198 | * apfs_read_omap_key - Parse an on-disk object map key 199 | * @raw: pointer to the raw key 200 | * @size: size of the raw key 201 | * @key: apfs_key structure to store the result 202 | * 203 | * Returns 0 on success, or a negative error code otherwise. 204 | */ 205 | int apfs_read_omap_key(void *raw, int size, struct apfs_key *key) 206 | { 207 | struct apfs_omap_key *raw_key; 208 | 209 | if (size != sizeof(*raw_key)) { 210 | apfs_err(NULL, "bad key length (%d)", size); 211 | return -EFSCORRUPTED; 212 | } 213 | raw_key = raw; 214 | 215 | key->id = le64_to_cpu(raw_key->ok_oid); 216 | key->type = 0; 217 | key->number = le64_to_cpu(raw_key->ok_xid); 218 | key->name = NULL; 219 | 220 | return 0; 221 | } 222 | 223 | /** 224 | * apfs_read_extentref_key - Parse an on-disk extent reference tree key 225 | * @raw: pointer to the raw key 226 | * @size: size of the raw key 227 | * @key: apfs_key structure to store the result 228 | * 229 | * Returns 0 on success, or a negative error code otherwise. 230 | */ 231 | int apfs_read_extentref_key(void *raw, int size, struct apfs_key *key) 232 | { 233 | if (size != sizeof(struct apfs_phys_ext_key)) { 234 | apfs_err(NULL, "bad key length (%d)", size); 235 | return -EFSCORRUPTED; 236 | } 237 | key->id = apfs_cat_cnid((struct apfs_key_header *)raw); 238 | key->type = apfs_cat_type((struct apfs_key_header *)raw); 239 | key->number = 0; 240 | key->name = NULL; 241 | return 0; 242 | } 243 | 244 | int apfs_read_snap_meta_key(void *raw, int size, struct apfs_key *key) 245 | { 246 | if (size < sizeof(struct apfs_key_header)) { 247 | apfs_err(NULL, "bad key length (%d)", size); 248 | return -EFSCORRUPTED; 249 | } 250 | key->id = apfs_cat_cnid((struct apfs_key_header *)raw); 251 | key->type = apfs_cat_type((struct apfs_key_header *)raw); 252 | key->number = 0; 253 | 254 | switch (key->type) { 255 | case APFS_TYPE_SNAP_METADATA: 256 | if (size != sizeof(struct apfs_snap_metadata_key)) { 257 | apfs_err(NULL, "bad key length (%d)", size); 258 | return -EFSCORRUPTED; 259 | } 260 | key->name = NULL; 261 | return 0; 262 | case APFS_TYPE_SNAP_NAME: 263 | if (size < sizeof(struct apfs_snap_name_key) + 1 || *((char *)raw + size - 1) != 0) { 264 | /* snapshot name must have NULL-termination */ 265 | apfs_err(NULL, "invalid snap name key (%d)", size); 266 | return -EFSCORRUPTED; 267 | } 268 | key->name = ((struct apfs_snap_name_key *)raw)->name; 269 | return 0; 270 | default: 271 | return -EFSCORRUPTED; 272 | } 273 | } 274 | 275 | int apfs_read_omap_snap_key(void *raw, int size, struct apfs_key *key) 276 | { 277 | __le64 *xid = NULL; 278 | 279 | if (size != sizeof(*xid)) { 280 | apfs_err(NULL, "bad key length (%d)", size); 281 | return -EFSCORRUPTED; 282 | } 283 | xid = raw; 284 | 285 | key->id = le64_to_cpup(xid); 286 | key->number = 0; 287 | key->name = NULL; 288 | key->type = 0; 289 | return 0; 290 | } 291 | 292 | /** 293 | * apfs_init_drec_key - Initialize an in-memory key for a dentry query 294 | * @sb: filesystem superblock 295 | * @ino: inode number of the parent directory 296 | * @name: filename (NULL for a multiple query) 297 | * @name_len: filename length (0 if NULL) 298 | * @key: apfs_key structure to initialize 299 | */ 300 | void apfs_init_drec_key(struct super_block *sb, u64 ino, const char *name, 301 | unsigned int name_len, struct apfs_key *key) 302 | { 303 | struct apfs_unicursor cursor; 304 | bool case_fold = apfs_is_case_insensitive(sb); 305 | u32 hash = 0xFFFFFFFF; 306 | 307 | key->id = ino; 308 | key->type = APFS_TYPE_DIR_REC; 309 | if (!apfs_is_normalization_insensitive(sb)) { 310 | key->name = name; 311 | key->number = 0; 312 | return; 313 | } 314 | 315 | /* To respect normalization, queries can only consider the hash */ 316 | key->name = NULL; 317 | 318 | if (!name) { 319 | key->number = 0; 320 | return; 321 | } 322 | 323 | apfs_init_unicursor(&cursor, name, name_len); 324 | 325 | while (1) { 326 | unicode_t utf32; 327 | 328 | utf32 = apfs_normalize_next(&cursor, case_fold); 329 | if (!utf32) 330 | break; 331 | 332 | hash = crc32c(hash, &utf32, sizeof(utf32)); 333 | } 334 | 335 | /* The filename length doesn't matter, so it's left as zero */ 336 | key->number = hash << APFS_DREC_HASH_SHIFT; 337 | } 338 | -------------------------------------------------------------------------------- /libzbitmap.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0+ OR MIT 2 | /* 3 | * Copyright (C) 2022 Corellium LLC 4 | * 5 | * Author: Ernesto A. Fernández 6 | * 7 | * Ported from libzbitmap (https://github.com/eafer/libzbitmap). Only the 8 | * decompression code is included. 9 | */ 10 | 11 | #include 12 | #include 13 | #include "libzbitmap.h" 14 | 15 | #define ZBM_MAGIC "ZBM\x09" 16 | #define ZBM_MAGIC_SZ 4 17 | 18 | #define ZBM_MAX_DECMP_CHUNK_SIZE 0x8000 19 | #define ZBM_MAX_DECMP_CHUNK_SIZE_BITS 15 20 | 21 | struct uint24 { 22 | uint8_t low; 23 | uint8_t mid; 24 | uint8_t hig; 25 | }; 26 | 27 | /* This header is shared by both compressed and decompressed chunks */ 28 | struct zbm_chunk_hdr { 29 | struct uint24 len; /* Length of the chunk */ 30 | struct uint24 decmp_len; /* Length of the chunk after decompression */ 31 | }; 32 | 33 | /* The full header for compressed chunks */ 34 | struct zbm_cmp_chunk_hdr { 35 | /* Shared with decompressed chunks */ 36 | struct zbm_chunk_hdr hdr; 37 | 38 | /* Offset for each of the three metadata areas */ 39 | struct uint24 meta_off_1; 40 | struct uint24 meta_off_2; 41 | struct uint24 meta_off_3; 42 | }; 43 | 44 | /* Pointer to a half-byte */ 45 | struct nybl_ptr { 46 | uint8_t *addr; /* Address of the byte */ 47 | int nibble; /* Which of the two nibbles? */ 48 | }; 49 | 50 | /* 0-2 and 0xf are not real bitmap indexes */ 51 | #define ZBM_BITMAP_COUNT (16 - 1 - 3) 52 | #define ZBM_BITMAP_BASE 3 53 | #define ZBM_BITMAP_BYTECNT 17 54 | #define ZBM_MAX_PERIOD_BYTECNT 2 55 | 56 | struct zbm_bmap { 57 | uint8_t bitmap; /* The bitmap */ 58 | uint8_t period_bytecnt; /* Read this many bytes to get the new period */ 59 | }; 60 | 61 | struct zbm_state { 62 | /* Updated during a chunk read */ 63 | uint8_t *dest; /* Write the next byte here */ 64 | size_t dest_left; /* Room left in destination buffer */ 65 | uint32_t written; /* Bytes written so far for current chunk */ 66 | uint16_t period; /* Repetition period for decompression, in bytes */ 67 | 68 | /* Updated right before a chunk read */ 69 | const uint8_t *src_end; /* End of current chunk */ 70 | uint32_t len; /* Length of the chunk */ 71 | uint32_t decmp_len; /* Expected chunk length after decompression */ 72 | 73 | /* Updated after a chunk read */ 74 | const uint8_t *src; /* Start of buffer, or current chunk if any */ 75 | size_t src_left; /* Room left in the source buffer */ 76 | size_t prewritten; /* Bytes written for previous chunks */ 77 | 78 | /* Current position in data and metadata areas for this chunk */ 79 | const uint8_t *data; 80 | const uint8_t *meta_1; 81 | const uint8_t *meta_2; 82 | struct nybl_ptr meta_3; 83 | 84 | /* Array of bitmaps for the current chunk */ 85 | struct zbm_bmap bitmaps[ZBM_BITMAP_COUNT]; 86 | }; 87 | 88 | static int zbm_check_magic(struct zbm_state *state) 89 | { 90 | if(state->src_left < ZBM_MAGIC_SZ) 91 | return -EINVAL; 92 | 93 | if(memcmp(state->src, ZBM_MAGIC, ZBM_MAGIC_SZ)) 94 | return -EINVAL; 95 | 96 | state->src += ZBM_MAGIC_SZ; 97 | state->src_left -= ZBM_MAGIC_SZ; 98 | return 0; 99 | } 100 | 101 | static uint32_t zbm_u24_to_u32(struct uint24 n) 102 | { 103 | uint32_t res; 104 | 105 | res = n.hig; 106 | res <<= 8; 107 | res += n.mid; 108 | res <<= 8; 109 | res += n.low; 110 | return res; 111 | } 112 | 113 | /* Some chunks just have regular uncompressed data, but with a header */ 114 | static int zbm_chunk_is_uncompressed(struct zbm_state *state) 115 | { 116 | return state->len == state->decmp_len + sizeof(struct zbm_chunk_hdr); 117 | } 118 | 119 | static int zbm_handle_uncompressed_chunk(struct zbm_state *state) 120 | { 121 | state->meta_1 = state->meta_2 = NULL; 122 | state->meta_3.addr = NULL; 123 | state->meta_3.nibble = 0; 124 | state->data = state->src + sizeof(struct zbm_chunk_hdr); 125 | memcpy(state->dest, state->data, state->decmp_len); 126 | 127 | state->dest += state->decmp_len; 128 | state->dest_left -= state->decmp_len; 129 | state->written = state->decmp_len; 130 | return 0; 131 | } 132 | 133 | static int zbm_read_nibble(struct nybl_ptr *nybl, const uint8_t *limit, uint8_t *result) 134 | { 135 | if(nybl->addr >= limit) 136 | return -EINVAL; 137 | 138 | if(nybl->nibble == 0) { 139 | *result = *nybl->addr & 0xf; 140 | nybl->nibble = 1; 141 | } else { 142 | *result = (*nybl->addr >> 4) & 0xf; 143 | nybl->nibble = 0; 144 | ++nybl->addr; 145 | } 146 | return 0; 147 | } 148 | 149 | static void zbm_rewind_nibble(struct nybl_ptr *nybl) 150 | { 151 | if(nybl->nibble == 0) { 152 | nybl->nibble = 1; 153 | --nybl->addr; 154 | } else { 155 | nybl->nibble = 0; 156 | } 157 | } 158 | 159 | static int zbm_apply_bitmap(struct zbm_state *state, struct zbm_bmap *bitmap) 160 | { 161 | int i; 162 | 163 | /* The periods are stored in the first metadata area */ 164 | if(bitmap->period_bytecnt) { 165 | state->period = 0; 166 | for(i = 0; i < bitmap->period_bytecnt; ++i) { 167 | if(state->meta_1 >= state->src_end) 168 | return -EINVAL; 169 | state->period |= *state->meta_1 << i * 8; 170 | ++state->meta_1; 171 | } 172 | } 173 | if(state->period == 0) 174 | return -EINVAL; 175 | 176 | for(i = 0; i < 8; ++i) { 177 | if(state->written == state->decmp_len) 178 | break; 179 | if(bitmap->bitmap & 1 << i) { 180 | if(state->data >= state->src_end) 181 | return -EINVAL; 182 | *state->dest = *state->data; 183 | ++state->data; 184 | } else { 185 | if(state->prewritten + state->written < state->period) 186 | return -EINVAL; 187 | *state->dest = *(state->dest - state->period); 188 | } 189 | ++state->dest; 190 | --state->dest_left; 191 | ++state->written; 192 | } 193 | 194 | return 0; 195 | } 196 | 197 | static int zbm_apply_bitmap_number(struct zbm_state *state, uint8_t bmp_num) 198 | { 199 | struct zbm_bmap next = {0}; 200 | 201 | /* Not a valid bitmap number (it signals a repetition) */ 202 | if(bmp_num == 0xf) 203 | return -EINVAL; 204 | 205 | /* An actual index in the bitmap array */ 206 | if(bmp_num > ZBM_MAX_PERIOD_BYTECNT) 207 | return zbm_apply_bitmap(state, &state->bitmaps[bmp_num - ZBM_BITMAP_BASE]); 208 | 209 | /* For < 2, use the next bitmap in the second metadata area */ 210 | if(state->meta_2 >= state->src_end) 211 | return -EINVAL; 212 | next.bitmap = *state->meta_2; 213 | next.period_bytecnt = bmp_num; 214 | ++state->meta_2; 215 | return zbm_apply_bitmap(state, &next); 216 | } 217 | 218 | /* Find out how many times we need to repeat the current bitmap operation */ 219 | static int zbm_read_repetition_count(struct zbm_state *state, uint16_t *repeat) 220 | { 221 | uint8_t nibble; 222 | uint16_t total; 223 | int err; 224 | 225 | /* Don't confuse the trailing bitmaps with a repetition count */ 226 | if(state->decmp_len - state->written <= 8) { 227 | *repeat = 1; 228 | return 0; 229 | } 230 | 231 | err = zbm_read_nibble(&state->meta_3, state->src_end, &nibble); 232 | if(err) 233 | return err; 234 | 235 | if(nibble != 0xf) { 236 | /* No repetition count: the previous bitmap number gets applied once */ 237 | zbm_rewind_nibble(&state->meta_3); 238 | *repeat = 1; 239 | return 0; 240 | } 241 | 242 | /* 243 | * Under this scheme, repeating a bitmap number 3 times wouldn't save any 244 | * space, so the repetition count starts from 4. 245 | */ 246 | total = 4; 247 | while(nibble == 0xf) { 248 | err = zbm_read_nibble(&state->meta_3, state->src_end, &nibble); 249 | if(err) 250 | return err; 251 | total += nibble; 252 | if(total < nibble) 253 | return -EINVAL; 254 | } 255 | 256 | *repeat = total; 257 | return 0; 258 | } 259 | 260 | static int zbm_decompress_single_bitmap(struct zbm_state *state) 261 | { 262 | uint8_t bmp_num; 263 | uint16_t repeat; 264 | int i; 265 | int err; 266 | 267 | /* The current nibble is the offset of the next bitmap to apply */ 268 | err = zbm_read_nibble(&state->meta_3, state->src_end, &bmp_num); 269 | if(err) 270 | return err; 271 | 272 | err = zbm_read_repetition_count(state, &repeat); 273 | if(err) 274 | return err; 275 | 276 | for(i = 0; i < repeat; ++i) { 277 | err = zbm_apply_bitmap_number(state, bmp_num); 278 | if(err) 279 | return err; 280 | } 281 | return 0; 282 | } 283 | 284 | /* Pointer to a bit */ 285 | struct bit_ptr { 286 | uint8_t *addr; /* Address of the byte */ 287 | int offset; /* Bit number */ 288 | }; 289 | 290 | /* This function does not perform boundary checks, the caller must do it */ 291 | static int zbm_read_single_bit(struct bit_ptr *bit) 292 | { 293 | int res = *bit->addr >> bit->offset & 1; 294 | 295 | ++bit->offset; 296 | if(bit->offset != 8) 297 | return res; 298 | bit->offset = 0; 299 | ++bit->addr; 300 | return res; 301 | } 302 | 303 | static int zbm_read_single_bitmap(struct bit_ptr *bit, const uint8_t *limit, struct zbm_bmap *result) 304 | { 305 | int i; 306 | 307 | result->bitmap = 0; 308 | result->period_bytecnt = 0; 309 | 310 | /* The bitmap itself */ 311 | for(i = 0; i < 8; ++i) { 312 | if(bit->addr >= limit) 313 | return -EINVAL; 314 | result->bitmap |= zbm_read_single_bit(bit) << i; 315 | } 316 | 317 | /* 318 | * The two trailing bits tell us how many bytes to read for the next 319 | * repetition period 320 | */ 321 | for(i = 0; i < 2; ++i) { 322 | if(bit->addr >= limit) 323 | return -EINVAL; 324 | result->period_bytecnt |= zbm_read_single_bit(bit) << i; 325 | } 326 | 327 | return 0; 328 | } 329 | 330 | static int zbm_read_bitmaps(struct zbm_state *state) 331 | { 332 | struct bit_ptr bmap = {0}; 333 | int err, i; 334 | 335 | if(state->len < ZBM_BITMAP_BYTECNT) 336 | return -EINVAL; 337 | 338 | bmap.addr = (uint8_t *)state->src_end - ZBM_BITMAP_BYTECNT; 339 | bmap.offset = 0; 340 | 341 | for(i = 0; i < ZBM_BITMAP_COUNT; ++i) { 342 | err = zbm_read_single_bitmap(&bmap, state->src_end, &state->bitmaps[i]); 343 | if(err) 344 | return err; 345 | if(state->bitmaps[i].period_bytecnt > ZBM_MAX_PERIOD_BYTECNT) 346 | return -EINVAL; 347 | } 348 | return 0; 349 | } 350 | 351 | static int zbm_handle_compressed_chunk(struct zbm_state *state) 352 | { 353 | const struct zbm_cmp_chunk_hdr *hdr = NULL; 354 | uint32_t meta_off_1, meta_off_2, meta_off_3; 355 | int err; 356 | 357 | state->written = 0; 358 | state->period = 8; 359 | 360 | if(state->len < sizeof(*hdr)) 361 | return -EINVAL; 362 | hdr = (struct zbm_cmp_chunk_hdr *)state->src; 363 | state->data = state->src + sizeof(*hdr); 364 | 365 | meta_off_1 = zbm_u24_to_u32(hdr->meta_off_1); 366 | meta_off_2 = zbm_u24_to_u32(hdr->meta_off_2); 367 | meta_off_3 = zbm_u24_to_u32(hdr->meta_off_3); 368 | if(meta_off_1 >= state->len || meta_off_2 >= state->len || meta_off_3 >= state->len) 369 | return -EINVAL; 370 | state->meta_1 = state->src + meta_off_1; 371 | state->meta_2 = state->src + meta_off_2; 372 | state->meta_3.addr = (uint8_t *)state->src + meta_off_3; 373 | state->meta_3.nibble = 0; 374 | 375 | err = zbm_read_bitmaps(state); 376 | if(err) 377 | return err; 378 | 379 | while(state->written < state->decmp_len) { 380 | err = zbm_decompress_single_bitmap(state); 381 | if(err) 382 | return err; 383 | } 384 | 385 | return 0; 386 | } 387 | 388 | static int zbm_handle_chunk(struct zbm_state *state) 389 | { 390 | const struct zbm_chunk_hdr *decmp_hdr = NULL; 391 | 392 | if(state->src_left < sizeof(*decmp_hdr)) 393 | return -EINVAL; 394 | decmp_hdr = (struct zbm_chunk_hdr *)state->src; 395 | 396 | state->len = zbm_u24_to_u32(decmp_hdr->len); 397 | if(state->len > state->src_left) 398 | return -EINVAL; 399 | state->src_end = state->src + state->len; 400 | 401 | state->decmp_len = zbm_u24_to_u32(decmp_hdr->decmp_len); 402 | if(state->decmp_len > ZBM_MAX_DECMP_CHUNK_SIZE) 403 | return -EINVAL; 404 | if(!state->dest) /* We just wanted the length, so we are done */ 405 | return 0; 406 | if(state->decmp_len > state->dest_left) 407 | return -ERANGE; 408 | 409 | if(zbm_chunk_is_uncompressed(state)) 410 | return zbm_handle_uncompressed_chunk(state); 411 | 412 | return zbm_handle_compressed_chunk(state); 413 | } 414 | 415 | int zbm_decompress(void *dest, size_t dest_size, const void *src, size_t src_size, size_t *out_len) 416 | { 417 | struct zbm_state state = {0}; 418 | int err; 419 | 420 | state.src = src; 421 | state.src_left = src_size; 422 | state.dest = dest; 423 | state.dest_left = dest_size; 424 | state.prewritten = 0; 425 | 426 | err = zbm_check_magic(&state); 427 | if(err) 428 | return err; 429 | 430 | /* The final chunk has zero decompressed length */ 431 | do { 432 | err = zbm_handle_chunk(&state); 433 | if(err) 434 | return err; 435 | state.src += state.len; 436 | state.src_left -= state.len; 437 | state.prewritten += state.decmp_len; 438 | } while(state.decmp_len != 0); 439 | 440 | *out_len = state.prewritten; 441 | return 0; 442 | } 443 | -------------------------------------------------------------------------------- /libzbitmap.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0+ OR MIT */ 2 | /* 3 | * Copyright (c) 2022 Corellium LLC 4 | * 5 | * Author: Ernesto A. Fernández 6 | * 7 | * Ported from libzbitmap (https://github.com/eafer/libzbitmap). Only the 8 | * decompression code is included. 9 | */ 10 | 11 | #ifndef _LIBZBITMAP_H 12 | #define _LIBZBITMAP_H 13 | 14 | #include 15 | #include 16 | 17 | /** 18 | * zbm_decompress - Decompress an LZBITMAP buffer 19 | * @dest: destination buffer (may be NULL) 20 | * @dest_size: size of the destination buffer 21 | * @src: source buffer 22 | * @src_size: size of the source buffer 23 | * @out_len: on return, the length of the decompressed output 24 | * 25 | * May be called with a NULL destination buffer to retrieve the expected length 26 | * of the decompressed data. Returns 0 on success, or a negative error code in 27 | * case of failure. 28 | */ 29 | int zbm_decompress(void *dest, size_t dest_size, const void *src, size_t src_size, size_t *out_len); 30 | 31 | #endif /* _LIBZBITMAP_H */ 32 | -------------------------------------------------------------------------------- /lzfse/lzfse.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: BSD-3-Clause */ 2 | /* 3 | * Copyright (c) 2015-2016, Apple Inc. All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | * 7 | * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | * 9 | * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer 10 | * in the documentation and/or other materials provided with the distribution. 11 | * 12 | * 3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived 13 | * from this software without specific prior written permission. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 16 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 17 | * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 19 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 20 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 21 | */ 22 | 23 | #ifndef LZFSE_H 24 | #define LZFSE_H 25 | 26 | #include 27 | #include 28 | 29 | /*! @abstract Get the required scratch buffer size to decompress using LZFSE. */ 30 | size_t lzfse_decode_scratch_size(void); 31 | 32 | /*! @abstract Decompress a buffer using LZFSE. 33 | * 34 | * @param dst_buffer 35 | * Pointer to the first byte of the destination buffer. 36 | * 37 | * @param dst_size 38 | * Size of the destination buffer in bytes. 39 | * 40 | * @param src_buffer 41 | * Pointer to the first byte of the source buffer. 42 | * 43 | * @param src_size 44 | * Size of the source buffer in bytes. 45 | * 46 | * @param scratch_buffer 47 | * If non-NULL, a pointer to scratch space for the routine to use as workspace; 48 | * the routine may use up to lzfse_decode_scratch_size( ) bytes of workspace 49 | * during its operation, and will not perform any internal allocations. If 50 | * NULL, the routine may allocate its own memory to use during operation via 51 | * a single call to malloc( ), and will release it by calling free( ) prior 52 | * to returning. For most use, passing NULL is perfectly satisfactory, but if 53 | * you require strict control over allocation, you will want to pass an 54 | * explicit scratch buffer. 55 | * 56 | * @return 57 | * The number of bytes written to the destination buffer if the input is 58 | * successfully decompressed. If there is not enough space in the destination 59 | * buffer to hold the entire expanded output, only the first dst_size bytes 60 | * will be written to the buffer and dst_size is returned. Note that this 61 | * behavior differs from that of lzfse_encode_buffer. 62 | */ 63 | size_t lzfse_decode_buffer(uint8_t *__restrict dst_buffer, size_t dst_size, 64 | const uint8_t *__restrict src_buffer, size_t src_size, 65 | void *__restrict scratch_buffer); 66 | 67 | #endif /* LZFSE_H */ 68 | -------------------------------------------------------------------------------- /lzfse/lzfse_decode.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* 3 | * Copyright (c) 2015-2016, Apple Inc. All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | * 7 | * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | * 9 | * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer 10 | * in the documentation and/or other materials provided with the distribution. 11 | * 12 | * 3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived 13 | * from this software without specific prior written permission. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 16 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 17 | * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 19 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 20 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 21 | */ 22 | 23 | /* LZFSE decode API */ 24 | 25 | #include 26 | #include "lzfse.h" 27 | #include "lzfse_internal.h" 28 | 29 | size_t lzfse_decode_scratch_size(void) 30 | { 31 | return sizeof(lzfse_decoder_state); 32 | } 33 | 34 | static size_t lzfse_decode_buffer_with_scratch(uint8_t *__restrict dst_buffer, size_t dst_size, 35 | const uint8_t *__restrict src_buffer, 36 | size_t src_size, void *__restrict scratch_buffer) 37 | { 38 | int status; 39 | lzfse_decoder_state *s = (lzfse_decoder_state *)scratch_buffer; 40 | 41 | memset(s, 0x00, sizeof(*s)); 42 | 43 | /* Initialize state */ 44 | s->src = src_buffer; 45 | s->src_begin = src_buffer; 46 | s->src_end = s->src + src_size; 47 | s->dst = dst_buffer; 48 | s->dst_begin = dst_buffer; 49 | s->dst_end = dst_buffer + dst_size; 50 | 51 | /* Decode */ 52 | status = lzfse_decode(s); 53 | if (status == LZFSE_STATUS_DST_FULL) 54 | return dst_size; 55 | if (status != LZFSE_STATUS_OK) 56 | return 0; /* failed */ 57 | return (size_t)(s->dst - dst_buffer); /* bytes written */ 58 | } 59 | 60 | size_t lzfse_decode_buffer(uint8_t *__restrict dst_buffer, size_t dst_size, 61 | const uint8_t *__restrict src_buffer, size_t src_size, 62 | void *__restrict scratch_buffer) 63 | { 64 | int has_malloc = 0; 65 | size_t ret = 0; 66 | 67 | /* Deal with the possible NULL pointer */ 68 | if (scratch_buffer == NULL) { 69 | /* +1 in case scratch size could be zero */ 70 | scratch_buffer = kmalloc(lzfse_decode_scratch_size() + 1, GFP_KERNEL); 71 | has_malloc = 1; 72 | } 73 | if (scratch_buffer == NULL) 74 | return 0; 75 | ret = lzfse_decode_buffer_with_scratch(dst_buffer, dst_size, src_buffer, src_size, 76 | scratch_buffer); 77 | if (has_malloc) 78 | kfree(scratch_buffer); 79 | return ret; 80 | } 81 | -------------------------------------------------------------------------------- /lzfse/lzfse_decode_base.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* 3 | * Copyright (c) 2015-2016, Apple Inc. All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | * 7 | * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | * 9 | * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer 10 | * in the documentation and/or other materials provided with the distribution. 11 | * 12 | * 3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived 13 | * from this software without specific prior written permission. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 16 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 17 | * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 19 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 20 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 21 | */ 22 | 23 | #include "lzfse_internal.h" 24 | #include "lzvn_decode_base.h" 25 | 26 | /*! @abstract Decode an entry value from next bits of stream. 27 | * Return \p value, and set \p *nbits to the number of bits to consume 28 | * (starting with LSB). 29 | */ 30 | static inline int lzfse_decode_v1_freq_value(uint32_t bits, int *nbits) 31 | { 32 | static const int8_t lzfse_freq_nbits_table[32] = { 2, 3, 2, 5, 2, 3, 2, 8, 2, 3, 2, 33 | 5, 2, 3, 2, 14, 2, 3, 2, 5, 2, 3, 34 | 2, 8, 2, 3, 2, 5, 2, 3, 2, 14 }; 35 | static const int8_t lzfse_freq_value_table[32] = { 0, 2, 1, 4, 0, 3, 1, -1, 0, 2, 1, 36 | 5, 0, 3, 1, -1, 0, 2, 1, 6, 0, 3, 37 | 1, -1, 0, 2, 1, 7, 0, 3, 1, -1 }; 38 | 39 | uint32_t b = bits & 31; /* lower 5 bits */ 40 | int n = lzfse_freq_nbits_table[b]; 41 | *nbits = n; 42 | 43 | /* Special cases for > 5 bits encoding */ 44 | if (n == 8) 45 | return 8 + ((bits >> 4) & 0xf); 46 | if (n == 14) 47 | return 24 + ((bits >> 4) & 0x3ff); 48 | 49 | // <= 5 bits encoding from table 50 | return lzfse_freq_value_table[b]; 51 | } 52 | 53 | /*! @abstract Extracts up to 32 bits from a 64-bit field beginning at 54 | * \p offset, and zero-extends them to a \p uint32_t. 55 | * 56 | * If we number the bits of \p v from 0 (least significant) to 63 (most 57 | * significant), the result is bits \p offset to \p offset+nbits-1. 58 | */ 59 | static inline uint32_t get_field(uint64_t v, int offset, int nbits) 60 | { 61 | if (nbits == 32) 62 | return (uint32_t)(v >> offset); 63 | return (uint32_t)((v >> offset) & ((1 << nbits) - 1)); 64 | } 65 | 66 | /*! @abstract Return \c header_size field from a \c lzfse_compressed_block_header_v2. */ 67 | static inline uint32_t lzfse_decode_v2_header_size(const lzfse_compressed_block_header_v2 *in) 68 | { 69 | return get_field(in->packed_fields[2], 0, 32); 70 | } 71 | 72 | /*! @abstract Decode all fields from a \c lzfse_compressed_block_header_v2 to a 73 | * \c lzfse_compressed_block_header_v1. 74 | * @return 0 on success. 75 | * @return -1 on failure. 76 | */ 77 | static inline int lzfse_decode_v1(lzfse_compressed_block_header_v1 *out, 78 | const lzfse_compressed_block_header_v2 *in) 79 | { 80 | uint64_t v0; 81 | uint64_t v1; 82 | uint64_t v2; 83 | uint16_t *dst = NULL; 84 | const uint8_t *src = NULL; 85 | const uint8_t *src_end = NULL; 86 | uint32_t accum = 0; 87 | int accum_nbits = 0; 88 | int nbits = 0; 89 | int i; 90 | 91 | /* Clear all fields */ 92 | memset(out, 0x00, sizeof(lzfse_compressed_block_header_v1)); 93 | 94 | v0 = in->packed_fields[0]; 95 | v1 = in->packed_fields[1]; 96 | v2 = in->packed_fields[2]; 97 | 98 | out->magic = LZFSE_COMPRESSEDV1_BLOCK_MAGIC; 99 | out->n_raw_bytes = in->n_raw_bytes; 100 | 101 | /* Literal state */ 102 | out->n_literals = get_field(v0, 0, 20); 103 | out->n_literal_payload_bytes = get_field(v0, 20, 20); 104 | out->literal_bits = (int)get_field(v0, 60, 3) - 7; 105 | out->literal_state[0] = get_field(v1, 0, 10); 106 | out->literal_state[1] = get_field(v1, 10, 10); 107 | out->literal_state[2] = get_field(v1, 20, 10); 108 | out->literal_state[3] = get_field(v1, 30, 10); 109 | 110 | /* L,M,D state */ 111 | out->n_matches = get_field(v0, 40, 20); 112 | out->n_lmd_payload_bytes = get_field(v1, 40, 20); 113 | out->lmd_bits = (int)get_field(v1, 60, 3) - 7; 114 | out->l_state = get_field(v2, 32, 10); 115 | out->m_state = get_field(v2, 42, 10); 116 | out->d_state = get_field(v2, 52, 10); 117 | 118 | /* Total payload size */ 119 | out->n_payload_bytes = out->n_literal_payload_bytes + out->n_lmd_payload_bytes; 120 | 121 | /* Freq tables */ 122 | dst = &(out->l_freq[0]); 123 | src = &(in->freq[0]); 124 | src_end = (const uint8_t *)in + get_field(v2, 0, 32); /* first byte after header */ 125 | accum = 0; 126 | accum_nbits = 0; 127 | 128 | /* No freq tables? */ 129 | if (src_end == src) 130 | return 0; /* OK, freq tables were omitted */ 131 | 132 | for (i = 0; i < LZFSE_ENCODE_L_SYMBOLS + LZFSE_ENCODE_M_SYMBOLS + LZFSE_ENCODE_D_SYMBOLS + 133 | LZFSE_ENCODE_LITERAL_SYMBOLS; 134 | i++) { 135 | /* 136 | * Refill accum, one byte at a time, until we reach end of 137 | * header, or accum is full 138 | */ 139 | while (src < src_end && accum_nbits + 8 <= 32) { 140 | accum |= (uint32_t)(*src) << accum_nbits; 141 | accum_nbits += 8; 142 | src++; 143 | } 144 | 145 | /* Decode and store value */ 146 | nbits = 0; 147 | dst[i] = lzfse_decode_v1_freq_value(accum, &nbits); 148 | 149 | if (nbits > accum_nbits) 150 | return -1; /* failed */ 151 | 152 | /* Consume nbits bits */ 153 | accum >>= nbits; 154 | accum_nbits -= nbits; 155 | } 156 | 157 | if (accum_nbits >= 8 || src != src_end) 158 | return -1; /* we need to end up exactly at the end of header, with less than 8 bits in accumulator */ 159 | 160 | return 0; 161 | } 162 | 163 | static inline void copy(uint8_t *dst, const uint8_t *src, size_t length) 164 | { 165 | const uint8_t *dst_end = dst + length; 166 | 167 | do { 168 | copy8(dst, src); 169 | dst += 8; 170 | src += 8; 171 | } while (dst < dst_end); 172 | } 173 | 174 | static int lzfse_decode_lmd(lzfse_decoder_state *s) 175 | { 176 | lzfse_compressed_block_decoder_state *bs = &(s->compressed_lzfse_block_state); 177 | fse_state l_state = bs->l_state; 178 | fse_state m_state = bs->m_state; 179 | fse_state d_state = bs->d_state; 180 | fse_in_stream in = bs->lmd_in_stream; 181 | const uint8_t *src_start = s->src_begin; 182 | const uint8_t *src = s->src + bs->lmd_in_buf; 183 | const uint8_t *lit = bs->current_literal; 184 | uint8_t *dst = s->dst; 185 | uint32_t symbols = bs->n_matches; 186 | int32_t L = bs->l_value; 187 | int32_t M = bs->m_value; 188 | int32_t D = bs->d_value; 189 | int32_t new_d; 190 | 191 | /* 192 | * Number of bytes remaining in the destination buffer, minus 32 to 193 | * provide a margin of safety for using overlarge copies on the fast 194 | * path. This is a signed quantity, and may go negative when we are 195 | * close to the end of the buffer. That's OK; we're careful about how 196 | * we handle it in the slow-and-careful match execution path. 197 | */ 198 | ptrdiff_t remaining_bytes = s->dst_end - dst - 32; 199 | 200 | /* 201 | * If L or M is non-zero, that means that we have already started 202 | * decoding this block, and that we needed to interrupt decoding to get 203 | * more space from the caller. There's a pending L, M, D triplet that 204 | * we weren't able to completely process. Jump ahead to finish executing 205 | * that symbol before decoding new values. 206 | */ 207 | if (L || M) 208 | goto ExecuteMatch; 209 | 210 | while (symbols > 0) { 211 | int res; 212 | /* Decode the next L, M, D symbol from the input stream. */ 213 | res = fse_in_flush(&in, &src, src_start); 214 | if (res) 215 | return LZFSE_STATUS_ERROR; 216 | L = fse_value_decode(&l_state, bs->l_decoder, &in); 217 | if ((lit + L) >= (bs->literals + LZFSE_LITERALS_PER_BLOCK + 64)) 218 | return LZFSE_STATUS_ERROR; 219 | res = fse_in_flush2(&in, &src, src_start); 220 | if (res) 221 | return LZFSE_STATUS_ERROR; 222 | M = fse_value_decode(&m_state, bs->m_decoder, &in); 223 | res = fse_in_flush2(&in, &src, src_start); 224 | if (res) 225 | return LZFSE_STATUS_ERROR; 226 | new_d = fse_value_decode(&d_state, bs->d_decoder, &in); 227 | D = new_d ? new_d : D; 228 | symbols--; 229 | 230 | ExecuteMatch: 231 | /* 232 | * Error if D is out of range, so that we avoid passing through 233 | * uninitialized data or accesssing memory out of the 234 | * destination buffer. 235 | */ 236 | if ((uint32_t)D > dst + L - s->dst_begin) 237 | return LZFSE_STATUS_ERROR; 238 | 239 | if (L + M <= remaining_bytes) { 240 | size_t i; 241 | /* 242 | * If we have plenty of space remaining, we can copy the 243 | * literal and match with 16- and 32-byte operations, 244 | * without worrying about writing off the end of the 245 | * buffer. 246 | */ 247 | remaining_bytes -= L + M; 248 | copy(dst, lit, L); 249 | dst += L; 250 | lit += L; 251 | /* 252 | * For the match, we have two paths; a fast copy by 253 | * 16-bytes if the match distance is large enough to 254 | * allow it, and a more careful path that applies a 255 | * permutation to account for the possible overlap 256 | * between source and destination if the distance is 257 | * small. 258 | */ 259 | if (D >= 8 || D >= M) 260 | copy(dst, dst - D, M); 261 | else 262 | for (i = 0; i < M; i++) 263 | dst[i] = dst[i - D]; 264 | dst += M; 265 | } 266 | 267 | else { 268 | /* 269 | * Otherwise, we are very close to the end of the 270 | * destination buffer, so we cannot use wide copies that 271 | * slop off the end of the region that we are copying 272 | * to. First, we restore the true length remaining, 273 | * rather than the sham value we've been using so far. 274 | */ 275 | remaining_bytes += 32; 276 | /* 277 | * Now, we process the literal. Either there's space for 278 | * it or there isn't; if there is, we copy the whole 279 | * thing and update all the pointers and lengths to 280 | * reflect the copy. 281 | */ 282 | if (L <= remaining_bytes) { 283 | size_t i; 284 | 285 | for (i = 0; i < L; i++) 286 | dst[i] = lit[i]; 287 | dst += L; 288 | lit += L; 289 | remaining_bytes -= L; 290 | L = 0; 291 | } 292 | /* 293 | * There isn't enough space to fit the whole literal. 294 | * Copy as much of it as we can, update the pointers and 295 | * the value of L, and report that the destination 296 | * buffer is full. Note that we always write right up to 297 | * the end of the destination buffer. 298 | */ 299 | else { 300 | size_t i; 301 | 302 | for (i = 0; i < remaining_bytes; i++) 303 | dst[i] = lit[i]; 304 | dst += remaining_bytes; 305 | lit += remaining_bytes; 306 | L -= remaining_bytes; 307 | goto DestinationBufferIsFull; 308 | } 309 | /* 310 | * The match goes just like the literal does. We copy as 311 | * much as we can byte-by-byte, and if we reach the end 312 | * of the buffer before finishing, we return to the 313 | * caller indicating that the buffer is full. 314 | */ 315 | if (M <= remaining_bytes) { 316 | size_t i; 317 | 318 | for (i = 0; i < M; i++) 319 | dst[i] = dst[i - D]; 320 | dst += M; 321 | remaining_bytes -= M; 322 | M = 0; 323 | (void)M; /* no dead store warning */ 324 | /* 325 | * We don't need to update M = 0, because 326 | * there's no partial symbol to continue 327 | * executing. Either we're at the end of the 328 | * block, in which case we will never need to 329 | * resume with this state, or we're going to 330 | * decode another L, M, D set, which will 331 | * overwrite M anyway. 332 | * 333 | * But we still set M = 0, to maintain the 334 | * post-condition. 335 | */ 336 | } else { 337 | size_t i; 338 | 339 | for (i = 0; i < remaining_bytes; i++) 340 | dst[i] = dst[i - D]; 341 | dst += remaining_bytes; 342 | M -= remaining_bytes; 343 | DestinationBufferIsFull: 344 | /* 345 | * Because we want to be able to resume decoding 346 | * where we've left off (even in the middle of a 347 | * literal or match), we need to update all of 348 | * the block state fields with the current 349 | * values so that we can resume execution from 350 | * this point once the caller has given us more 351 | * space to write into. 352 | */ 353 | bs->l_value = L; 354 | bs->m_value = M; 355 | bs->d_value = D; 356 | bs->l_state = l_state; 357 | bs->m_state = m_state; 358 | bs->d_state = d_state; 359 | bs->lmd_in_stream = in; 360 | bs->n_matches = symbols; 361 | bs->lmd_in_buf = (uint32_t)(src - s->src); 362 | bs->current_literal = lit; 363 | s->dst = dst; 364 | return LZFSE_STATUS_DST_FULL; 365 | } 366 | /* 367 | * Restore the "sham" decremented value of 368 | * remaining_bytes and continue to the next L, M, D 369 | * triple. We'll just be back in the careful path again, 370 | * but this only happens at the very end of the buffer, 371 | * so a little minor inefficiency here is a good 372 | * tradeoff for simpler code. 373 | */ 374 | remaining_bytes -= 32; 375 | } 376 | } 377 | /* 378 | * Because we've finished with the whole block, we don't need to update 379 | * any of the blockstate fields; they will not be used again. We just 380 | * update the destination pointer in the state object and return. 381 | */ 382 | s->dst = dst; 383 | return LZFSE_STATUS_OK; 384 | } 385 | 386 | int lzfse_decode(lzfse_decoder_state *s) 387 | { 388 | while (1) { 389 | /* Are we inside a block? */ 390 | switch (s->block_magic) { 391 | case LZFSE_NO_BLOCK_MAGIC: { 392 | uint32_t magic; 393 | /* We need at least 4 bytes of magic number to identify next block */ 394 | if (s->src + 4 > s->src_end) 395 | return LZFSE_STATUS_SRC_EMPTY; /* SRC truncated */ 396 | magic = load4(s->src); 397 | 398 | if (magic == LZFSE_ENDOFSTREAM_BLOCK_MAGIC) { 399 | s->src += 4; 400 | s->end_of_stream = 1; 401 | return LZFSE_STATUS_OK; /* done */ 402 | } 403 | 404 | if (magic == LZFSE_UNCOMPRESSED_BLOCK_MAGIC) { 405 | uncompressed_block_decoder_state *bs = NULL; 406 | 407 | if (s->src + sizeof(uncompressed_block_header) > s->src_end) 408 | return LZFSE_STATUS_SRC_EMPTY; /* SRC truncated */ 409 | /* Setup state for uncompressed block */ 410 | bs = &(s->uncompressed_block_state); 411 | bs->n_raw_bytes = load4( 412 | s->src + offsetof(uncompressed_block_header, n_raw_bytes)); 413 | s->src += sizeof(uncompressed_block_header); 414 | s->block_magic = magic; 415 | break; 416 | } 417 | 418 | if (magic == LZFSE_COMPRESSEDLZVN_BLOCK_MAGIC) { 419 | lzvn_compressed_block_decoder_state *bs = NULL; 420 | 421 | if (s->src + sizeof(lzvn_compressed_block_header) > s->src_end) 422 | return LZFSE_STATUS_SRC_EMPTY; /* SRC truncated */ 423 | /* Setup state for compressed LZVN block */ 424 | bs = &(s->compressed_lzvn_block_state); 425 | bs->n_raw_bytes = 426 | load4(s->src + 427 | offsetof(lzvn_compressed_block_header, n_raw_bytes)); 428 | bs->n_payload_bytes = 429 | load4(s->src + offsetof(lzvn_compressed_block_header, 430 | n_payload_bytes)); 431 | bs->d_prev = 0; 432 | s->src += sizeof(lzvn_compressed_block_header); 433 | s->block_magic = magic; 434 | break; 435 | } 436 | 437 | if (magic == LZFSE_COMPRESSEDV1_BLOCK_MAGIC || 438 | magic == LZFSE_COMPRESSEDV2_BLOCK_MAGIC) { 439 | lzfse_compressed_block_header_v1 header1; 440 | size_t header_size = 0; 441 | lzfse_compressed_block_decoder_state *bs = NULL; 442 | 443 | /* Decode compressed headers */ 444 | if (magic == LZFSE_COMPRESSEDV2_BLOCK_MAGIC) { 445 | const lzfse_compressed_block_header_v2 *header2; 446 | int decodeStatus; 447 | /* Check we have the fixed part of the structure */ 448 | if (s->src + offsetof(lzfse_compressed_block_header_v2, 449 | freq) > 450 | s->src_end) 451 | return LZFSE_STATUS_SRC_EMPTY; /* SRC truncated */ 452 | 453 | /* Get size, and check we have the entire structure */ 454 | header2 = (const lzfse_compressed_block_header_v2 *) 455 | s->src; /* not aligned, OK */ 456 | header_size = lzfse_decode_v2_header_size(header2); 457 | if (s->src + header_size > s->src_end) 458 | return LZFSE_STATUS_SRC_EMPTY; /* SRC truncated */ 459 | decodeStatus = lzfse_decode_v1(&header1, header2); 460 | if (decodeStatus != 0) 461 | return LZFSE_STATUS_ERROR; /* failed */ 462 | } else { 463 | if (s->src + sizeof(lzfse_compressed_block_header_v1) > 464 | s->src_end) 465 | return LZFSE_STATUS_SRC_EMPTY; /* SRC truncated */ 466 | memcpy(&header1, s->src, 467 | sizeof(lzfse_compressed_block_header_v1)); 468 | header_size = sizeof(lzfse_compressed_block_header_v1); 469 | } 470 | 471 | /* 472 | * We require the header + entire encoded block 473 | * to be present in SRC during the entire block 474 | * decoding. This can be relaxed somehow, if it 475 | * becomes a limiting factor, at the price of a 476 | * more complex state maintenance. For DST, we 477 | * can't easily require space for the entire 478 | * decoded block, because it may expand to 479 | * something very very large. 480 | */ 481 | if (s->src + header_size + header1.n_literal_payload_bytes + 482 | header1.n_lmd_payload_bytes > 483 | s->src_end) 484 | return LZFSE_STATUS_SRC_EMPTY; /* need all encoded block */ 485 | 486 | /* Sanity checks */ 487 | if (lzfse_check_block_header_v1(&header1) != 0) 488 | return LZFSE_STATUS_ERROR; 489 | 490 | /* Skip header */ 491 | s->src += header_size; 492 | 493 | /* Setup state for compressed V1 block from header */ 494 | bs = &(s->compressed_lzfse_block_state); 495 | bs->n_lmd_payload_bytes = header1.n_lmd_payload_bytes; 496 | bs->n_matches = header1.n_matches; 497 | fse_init_decoder_table(LZFSE_ENCODE_LITERAL_STATES, 498 | LZFSE_ENCODE_LITERAL_SYMBOLS, 499 | header1.literal_freq, bs->literal_decoder); 500 | fse_init_value_decoder_table(LZFSE_ENCODE_L_STATES, 501 | LZFSE_ENCODE_L_SYMBOLS, header1.l_freq, 502 | l_extra_bits, l_base_value, 503 | bs->l_decoder); 504 | fse_init_value_decoder_table(LZFSE_ENCODE_M_STATES, 505 | LZFSE_ENCODE_M_SYMBOLS, header1.m_freq, 506 | m_extra_bits, m_base_value, 507 | bs->m_decoder); 508 | fse_init_value_decoder_table(LZFSE_ENCODE_D_STATES, 509 | LZFSE_ENCODE_D_SYMBOLS, header1.d_freq, 510 | d_extra_bits, d_base_value, 511 | bs->d_decoder); 512 | 513 | /* Decode literals */ 514 | { 515 | fse_in_stream in; 516 | const uint8_t *buf_start = s->src_begin; 517 | const uint8_t *buf; 518 | fse_state state0; 519 | fse_state state1; 520 | fse_state state2; 521 | fse_state state3; 522 | uint32_t i; 523 | 524 | s->src += 525 | header1.n_literal_payload_bytes; /* skip literal payload */ 526 | buf = s->src; /* read bits backwards from the end */ 527 | if (fse_in_init(&in, header1.literal_bits, &buf, 528 | buf_start) != 0) 529 | return LZFSE_STATUS_ERROR; 530 | 531 | state0 = header1.literal_state[0]; 532 | state1 = header1.literal_state[1]; 533 | state2 = header1.literal_state[2]; 534 | state3 = header1.literal_state[3]; 535 | 536 | for (i = 0; i < header1.n_literals; 537 | i += 4 /* n_literals is multiple of 4 */) { 538 | #if FSE_IOSTREAM_64 539 | if (fse_in_flush(&in, &buf, buf_start) != 0) 540 | return LZFSE_STATUS_ERROR; /* [57, 64] bits */ 541 | bs->literals[i + 0] = 542 | fse_decode(&state0, bs->literal_decoder, 543 | &in); /* 10b max */ 544 | bs->literals[i + 1] = 545 | fse_decode(&state1, bs->literal_decoder, 546 | &in); /* 10b max */ 547 | bs->literals[i + 2] = 548 | fse_decode(&state2, bs->literal_decoder, 549 | &in); /* 10b max */ 550 | bs->literals[i + 3] = 551 | fse_decode(&state3, bs->literal_decoder, 552 | &in); /* 10b max */ 553 | #else 554 | if (fse_in_flush(&in, &buf, buf_start) != 0) 555 | return LZFSE_STATUS_ERROR; /* [25, 23] bits */ 556 | bs->literals[i + 0] = 557 | fse_decode(&state0, bs->literal_decoder, 558 | &in); /* 10b max */ 559 | bs->literals[i + 1] = 560 | fse_decode(&state1, bs->literal_decoder, 561 | &in); /* 10b max */ 562 | if (fse_in_flush(&in, &buf, buf_start) != 0) 563 | return LZFSE_STATUS_ERROR; /* [25, 23] bits */ 564 | bs->literals[i + 2] = 565 | fse_decode(&state2, bs->literal_decoder, 566 | &in); /* 10b max */ 567 | bs->literals[i + 3] = 568 | fse_decode(&state3, bs->literal_decoder, 569 | &in); /* 10b max */ 570 | #endif 571 | } 572 | 573 | bs->current_literal = bs->literals; 574 | } /* literals */ 575 | 576 | /* 577 | * SRC is not incremented to skip the LMD 578 | * payload, since we need it during block 579 | * decode. We will increment SRC at the end of 580 | * the block only after this point. 581 | */ 582 | 583 | /* 584 | * Initialize the L,M,D decode stream, do not 585 | * start decoding matches yet, and store decoder 586 | * state 587 | */ 588 | { 589 | fse_in_stream in; 590 | /* read bits backwards from the end */ 591 | const uint8_t *buf = s->src + header1.n_lmd_payload_bytes; 592 | 593 | if (fse_in_init(&in, header1.lmd_bits, &buf, s->src) != 0) 594 | return LZFSE_STATUS_ERROR; 595 | 596 | bs->l_state = header1.l_state; 597 | bs->m_state = header1.m_state; 598 | bs->d_state = header1.d_state; 599 | bs->lmd_in_buf = (uint32_t)(buf - s->src); 600 | bs->l_value = bs->m_value = 0; 601 | /* 602 | * Initialize D to an illegal value so 603 | * we can't erroneously use an 604 | * uninitialized "previous" value. 605 | */ 606 | bs->d_value = -1; 607 | bs->lmd_in_stream = in; 608 | } 609 | 610 | s->block_magic = magic; 611 | break; 612 | } 613 | 614 | /* Here we have an invalid magic number */ 615 | return LZFSE_STATUS_ERROR; 616 | } /* LZFSE_NO_BLOCK_MAGIC */ 617 | 618 | case LZFSE_UNCOMPRESSED_BLOCK_MAGIC: { 619 | uncompressed_block_decoder_state *bs = &(s->uncompressed_block_state); 620 | 621 | /* 622 | * Compute the size (in bytes) of the data that we will 623 | * actually copy. This size is 624 | * minimum(bs->n_raw_bytes, space in src, space in dst). 625 | */ 626 | 627 | uint32_t copy_size = bs->n_raw_bytes; /* bytes left to copy */ 628 | size_t src_space, dst_space; 629 | 630 | if (copy_size == 0) { 631 | s->block_magic = 0; 632 | break; 633 | } /* end of block */ 634 | 635 | if (s->src_end <= s->src) 636 | return LZFSE_STATUS_SRC_EMPTY; /* need more SRC data */ 637 | src_space = s->src_end - s->src; 638 | if (copy_size > src_space) 639 | copy_size = (uint32_t)src_space; /* limit to SRC data (> 0) */ 640 | 641 | if (s->dst_end <= s->dst) 642 | return LZFSE_STATUS_DST_FULL; /* need more DST capacity */ 643 | dst_space = s->dst_end - s->dst; 644 | if (copy_size > dst_space) 645 | copy_size = (uint32_t)dst_space; /* limit to DST capacity (> 0) */ 646 | 647 | /* 648 | * Now that we know that the copy size is bounded to the 649 | * source and dest buffers, go ahead and copy the data. 650 | * We always have copy_size > 0 here 651 | */ 652 | memcpy(s->dst, s->src, copy_size); 653 | s->src += copy_size; 654 | s->dst += copy_size; 655 | bs->n_raw_bytes -= copy_size; 656 | 657 | break; 658 | } /* LZFSE_UNCOMPRESSED_BLOCK_MAGIC */ 659 | 660 | case LZFSE_COMPRESSEDV1_BLOCK_MAGIC: 661 | case LZFSE_COMPRESSEDV2_BLOCK_MAGIC: { 662 | int status; 663 | lzfse_compressed_block_decoder_state *bs = 664 | &(s->compressed_lzfse_block_state); 665 | /* Require the entire LMD payload to be in SRC */ 666 | if (s->src_end <= s->src || 667 | bs->n_lmd_payload_bytes > (size_t)(s->src_end - s->src)) 668 | return LZFSE_STATUS_SRC_EMPTY; 669 | 670 | status = lzfse_decode_lmd(s); 671 | if (status != LZFSE_STATUS_OK) 672 | return status; 673 | 674 | s->block_magic = LZFSE_NO_BLOCK_MAGIC; 675 | s->src += bs->n_lmd_payload_bytes; // to next block 676 | break; 677 | } /* LZFSE_COMPRESSEDV1_BLOCK_MAGIC || LZFSE_COMPRESSEDV2_BLOCK_MAGIC */ 678 | 679 | case LZFSE_COMPRESSEDLZVN_BLOCK_MAGIC: { 680 | lzvn_compressed_block_decoder_state *bs = &(s->compressed_lzvn_block_state); 681 | lzvn_decoder_state dstate; 682 | size_t src_used, dst_used; 683 | 684 | if (bs->n_payload_bytes > 0 && s->src_end <= s->src) 685 | return LZFSE_STATUS_SRC_EMPTY; /* need more SRC data */ 686 | 687 | /* Init LZVN decoder state */ 688 | memset(&dstate, 0x00, sizeof(dstate)); 689 | dstate.src = s->src; 690 | dstate.src_end = s->src_end; 691 | if (dstate.src_end - s->src > bs->n_payload_bytes) 692 | dstate.src_end = 693 | s->src + bs->n_payload_bytes; /* limit to payload bytes */ 694 | dstate.dst_begin = s->dst_begin; 695 | dstate.dst = s->dst; 696 | dstate.dst_end = s->dst_end; 697 | if (dstate.dst_end - s->dst > bs->n_raw_bytes) 698 | dstate.dst_end = s->dst + bs->n_raw_bytes; /* limit to raw bytes */ 699 | dstate.d_prev = bs->d_prev; 700 | dstate.end_of_stream = 0; 701 | 702 | /* Run LZVN decoder */ 703 | lzvn_decode(&dstate); 704 | 705 | /* Update our state */ 706 | src_used = dstate.src - s->src; 707 | dst_used = dstate.dst - s->dst; 708 | if (src_used > bs->n_payload_bytes || dst_used > bs->n_raw_bytes) 709 | return LZFSE_STATUS_ERROR; /* sanity check */ 710 | s->src = dstate.src; 711 | s->dst = dstate.dst; 712 | bs->n_payload_bytes -= (uint32_t)src_used; 713 | bs->n_raw_bytes -= (uint32_t)dst_used; 714 | bs->d_prev = (uint32_t)dstate.d_prev; 715 | 716 | /* Test end of block */ 717 | if (bs->n_payload_bytes == 0 && bs->n_raw_bytes == 0 && 718 | dstate.end_of_stream) { 719 | s->block_magic = 0; 720 | break; 721 | } /* block done */ 722 | 723 | /* Check for invalid state */ 724 | if (bs->n_payload_bytes == 0 || bs->n_raw_bytes == 0 || 725 | dstate.end_of_stream) 726 | return LZFSE_STATUS_ERROR; 727 | 728 | /* 729 | * Here, block is not done and state is valid, so we 730 | * need more space in dst. 731 | */ 732 | return LZFSE_STATUS_DST_FULL; 733 | } 734 | 735 | default: 736 | return LZFSE_STATUS_ERROR; /* invalid magic */ 737 | 738 | } /* switch magic */ 739 | 740 | } /* block loop */ 741 | 742 | return LZFSE_STATUS_OK; 743 | } 744 | -------------------------------------------------------------------------------- /lzfse/lzfse_fse.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* 3 | * Copyright (c) 2015-2016, Apple Inc. All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | * 7 | * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | * 9 | * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer 10 | * in the documentation and/or other materials provided with the distribution. 11 | * 12 | * 3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived 13 | * from this software without specific prior written permission. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 16 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 17 | * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 19 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 20 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 21 | */ 22 | 23 | #include "lzfse_internal.h" 24 | 25 | /* 26 | * Initialize decoder table T[NSTATES]. 27 | * NSTATES = sum FREQ[i] is the number of states (a power of 2) 28 | * NSYMBOLS is the number of symbols. 29 | * FREQ[NSYMBOLS] is a normalized histogram of symbol frequencies, with FREQ[i] 30 | * >= 0. 31 | * Some symbols may have a 0 frequency. In that case, they should not be 32 | * present in the data. 33 | */ 34 | int fse_init_decoder_table(int nstates, int nsymbols, const uint16_t *__restrict freq, 35 | int32_t *__restrict t) 36 | { 37 | int n_clz = __builtin_clz(nstates); 38 | int sum_of_freq = 0; 39 | int i, j0, j; 40 | 41 | for (i = 0; i < nsymbols; i++) { 42 | int f = (int)freq[i]; 43 | int k; 44 | 45 | if (f == 0) 46 | continue; /* skip this symbol, no occurrences */ 47 | 48 | sum_of_freq += f; 49 | 50 | if (sum_of_freq > nstates) 51 | return -1; 52 | 53 | k = __builtin_clz(f) - n_clz; /* shift needed to ensure N <= (F<> k) - f; 55 | 56 | /* Initialize all states S reached by this symbol: OFFSET <= S < OFFSET + F */ 57 | for (j = 0; j < f; j++) { 58 | fse_decoder_entry e; 59 | 60 | e.symbol = (uint8_t)i; 61 | if (j < j0) { 62 | e.k = (int8_t)k; 63 | e.delta = (int16_t)(((f + j) << k) - nstates); 64 | } else { 65 | e.k = (int8_t)(k - 1); 66 | e.delta = (int16_t)((j - j0) << (k - 1)); 67 | } 68 | 69 | memcpy(t, &e, sizeof(e)); 70 | t++; 71 | } 72 | } 73 | 74 | return 0; /* OK */ 75 | } 76 | 77 | /* 78 | * Initialize value decoder table T[NSTATES]. 79 | * NSTATES = sum FREQ[i] is the number of states (a power of 2) 80 | * NSYMBOLS is the number of symbols. 81 | * FREQ[NSYMBOLS] is a normalized histogram of symbol frequencies, with FREQ[i] 82 | * >= 0. 83 | * SYMBOL_VBITS[NSYMBOLS] and SYMBOLS_VBASE[NSYMBOLS] are the number of value 84 | * bits to read and the base value for each symbol. 85 | * Some symbols may have a 0 frequency. In that case, they should not be 86 | * present in the data. 87 | */ 88 | void fse_init_value_decoder_table(int nstates, int nsymbols, const uint16_t *__restrict freq, 89 | const uint8_t *__restrict symbol_vbits, 90 | const int32_t *__restrict symbol_vbase, 91 | struct fse_value_decoder_entry *__restrict t) 92 | { 93 | int n_clz = __builtin_clz(nstates); 94 | int i; 95 | 96 | for (i = 0; i < nsymbols; i++) { 97 | struct fse_value_decoder_entry ei = { 0 }; 98 | int f = (int)freq[i]; 99 | int k, j0, j; 100 | 101 | if (f == 0) 102 | continue; /* skip this symbol, no occurrences */ 103 | 104 | k = __builtin_clz(f) - n_clz; /* shift needed to ensure N <= (F<> k) - f; 106 | 107 | ei.value_bits = symbol_vbits[i]; 108 | ei.vbase = symbol_vbase[i]; 109 | 110 | /* Initialize all states S reached by this symbol: OFFSET <= S < OFFSET + F */ 111 | for (j = 0; j < f; j++) { 112 | struct fse_value_decoder_entry e = ei; 113 | 114 | if (j < j0) { 115 | e.total_bits = (uint8_t)k + e.value_bits; 116 | e.delta = (int16_t)(((f + j) << k) - nstates); 117 | } else { 118 | e.total_bits = (uint8_t)(k - 1) + e.value_bits; 119 | e.delta = (int16_t)((j - j0) << (k - 1)); 120 | } 121 | 122 | memcpy(t, &e, 8); 123 | t++; 124 | } 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /lzfse/lzfse_fse.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: BSD-3-Clause */ 2 | /* 3 | * Copyright (c) 2015-2016, Apple Inc. All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | * 7 | * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | * 9 | * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer 10 | * in the documentation and/or other materials provided with the distribution. 11 | * 12 | * 3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived 13 | * from this software without specific prior written permission. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 16 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 17 | * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 19 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 20 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 21 | */ 22 | 23 | /* 24 | * Finite state entropy coding (FSE) 25 | * This is an implementation of the tANS algorithm described by Jarek Duda, 26 | * we use the more descriptive name "Finite State Entropy". 27 | */ 28 | 29 | #ifndef LZFSE_FSE_H 30 | #define LZFSE_FSE_H 31 | 32 | #include 33 | #include 34 | 35 | /* 36 | * Select between 32/64-bit I/O streams for FSE. Note that the FSE stream 37 | * size need not match the word size of the machine, but in practice you 38 | * want to use 64b streams on 64b systems for better performance. 39 | */ 40 | #if defined(_M_AMD64) || defined(__x86_64__) || defined(__arm64__) 41 | #define FSE_IOSTREAM_64 1 42 | #else 43 | #define FSE_IOSTREAM_64 0 44 | #endif 45 | 46 | /* MARK: - Bit utils */ 47 | 48 | /*! @abstract Signed type used to represent bit count. */ 49 | typedef int32_t fse_bit_count; 50 | 51 | /*! @abstract Unsigned type used to represent FSE state. */ 52 | typedef uint16_t fse_state; 53 | 54 | /* Mask the NBITS lsb of X. 0 <= NBITS < 64 */ 55 | static inline uint64_t fse_mask_lsb64(uint64_t x, fse_bit_count nbits) 56 | { 57 | static const uint64_t mtable[65] = { 58 | 0x0000000000000000LLU, 0x0000000000000001LLU, 0x0000000000000003LLU, 59 | 0x0000000000000007LLU, 0x000000000000000fLLU, 0x000000000000001fLLU, 60 | 0x000000000000003fLLU, 0x000000000000007fLLU, 0x00000000000000ffLLU, 61 | 0x00000000000001ffLLU, 0x00000000000003ffLLU, 0x00000000000007ffLLU, 62 | 0x0000000000000fffLLU, 0x0000000000001fffLLU, 0x0000000000003fffLLU, 63 | 0x0000000000007fffLLU, 0x000000000000ffffLLU, 0x000000000001ffffLLU, 64 | 0x000000000003ffffLLU, 0x000000000007ffffLLU, 0x00000000000fffffLLU, 65 | 0x00000000001fffffLLU, 0x00000000003fffffLLU, 0x00000000007fffffLLU, 66 | 0x0000000000ffffffLLU, 0x0000000001ffffffLLU, 0x0000000003ffffffLLU, 67 | 0x0000000007ffffffLLU, 0x000000000fffffffLLU, 0x000000001fffffffLLU, 68 | 0x000000003fffffffLLU, 0x000000007fffffffLLU, 0x00000000ffffffffLLU, 69 | 0x00000001ffffffffLLU, 0x00000003ffffffffLLU, 0x00000007ffffffffLLU, 70 | 0x0000000fffffffffLLU, 0x0000001fffffffffLLU, 0x0000003fffffffffLLU, 71 | 0x0000007fffffffffLLU, 0x000000ffffffffffLLU, 0x000001ffffffffffLLU, 72 | 0x000003ffffffffffLLU, 0x000007ffffffffffLLU, 0x00000fffffffffffLLU, 73 | 0x00001fffffffffffLLU, 0x00003fffffffffffLLU, 0x00007fffffffffffLLU, 74 | 0x0000ffffffffffffLLU, 0x0001ffffffffffffLLU, 0x0003ffffffffffffLLU, 75 | 0x0007ffffffffffffLLU, 0x000fffffffffffffLLU, 0x001fffffffffffffLLU, 76 | 0x003fffffffffffffLLU, 0x007fffffffffffffLLU, 0x00ffffffffffffffLLU, 77 | 0x01ffffffffffffffLLU, 0x03ffffffffffffffLLU, 0x07ffffffffffffffLLU, 78 | 0x0fffffffffffffffLLU, 0x1fffffffffffffffLLU, 0x3fffffffffffffffLLU, 79 | 0x7fffffffffffffffLLU, 0xffffffffffffffffLLU, 80 | }; 81 | return x & mtable[nbits]; 82 | } 83 | 84 | /* Mask the NBITS lsb of X. 0 <= NBITS < 32 */ 85 | static inline uint32_t fse_mask_lsb32(uint32_t x, fse_bit_count nbits) 86 | { 87 | static const uint32_t mtable[33] = { 88 | 0x0000000000000000U, 0x0000000000000001U, 0x0000000000000003U, 0x0000000000000007U, 89 | 0x000000000000000fU, 0x000000000000001fU, 0x000000000000003fU, 0x000000000000007fU, 90 | 0x00000000000000ffU, 0x00000000000001ffU, 0x00000000000003ffU, 0x00000000000007ffU, 91 | 0x0000000000000fffU, 0x0000000000001fffU, 0x0000000000003fffU, 0x0000000000007fffU, 92 | 0x000000000000ffffU, 0x000000000001ffffU, 0x000000000003ffffU, 0x000000000007ffffU, 93 | 0x00000000000fffffU, 0x00000000001fffffU, 0x00000000003fffffU, 0x00000000007fffffU, 94 | 0x0000000000ffffffU, 0x0000000001ffffffU, 0x0000000003ffffffU, 0x0000000007ffffffU, 95 | 0x000000000fffffffU, 0x000000001fffffffU, 0x000000003fffffffU, 0x000000007fffffffU, 96 | 0x00000000ffffffffU, 97 | }; 98 | return x & mtable[nbits]; 99 | } 100 | 101 | /*! @abstract Select \c nbits at index \c start from \c x. 102 | * 0 <= start <= start+nbits <= 64 103 | */ 104 | static __always_inline uint64_t fse_extract_bits64(uint64_t x, fse_bit_count start, 105 | fse_bit_count nbits) 106 | { 107 | /* If START and NBITS are constants, map to bit-field extraction instructions */ 108 | if (__builtin_constant_p(start) && __builtin_constant_p(nbits)) 109 | return (x >> start) & ((1LLU << nbits) - 1LLU); 110 | 111 | /* Otherwise, shift and mask */ 112 | return fse_mask_lsb64(x >> start, nbits); 113 | } 114 | 115 | /*! @abstract Select \c nbits at index \c start from \c x. 116 | * 0 <= start <= start+nbits <= 32 117 | */ 118 | static __always_inline uint32_t fse_extract_bits32(uint32_t x, fse_bit_count start, 119 | fse_bit_count nbits) 120 | { 121 | /* If START and NBITS are constants, map to bit-field extraction instructions */ 122 | if (__builtin_constant_p(start) && __builtin_constant_p(nbits)) 123 | return (x >> start) & ((1U << nbits) - 1U); 124 | 125 | /* Otherwise, shift and mask */ 126 | return fse_mask_lsb32(x >> start, nbits); 127 | } 128 | 129 | /* MARK: - Bit stream */ 130 | 131 | /* 132 | * I/O streams 133 | * The streams can be shared between several FSE encoders/decoders, which is why 134 | * they are not in the state struct 135 | */ 136 | 137 | /*! @abstract Output stream, 64-bit accum. */ 138 | typedef struct { 139 | uint64_t accum; /* Output bits */ 140 | fse_bit_count accum_nbits; /* Number of valid bits in ACCUM, other bits are 0 */ 141 | } fse_out_stream64; 142 | 143 | /*! @abstract Output stream, 32-bit accum. */ 144 | typedef struct { 145 | uint32_t accum; /* Output bits */ 146 | fse_bit_count accum_nbits; /* Number of valid bits in ACCUM, other bits are 0 */ 147 | } fse_out_stream32; 148 | 149 | /*! @abstract Object representing an input stream. */ 150 | typedef struct { 151 | uint64_t accum; /* Input bits */ 152 | fse_bit_count accum_nbits; /* Number of valid bits in ACCUM, other bits are 0 */ 153 | } fse_in_stream64; 154 | 155 | /*! @abstract Object representing an input stream. */ 156 | typedef struct { 157 | uint32_t accum; /* Input bits */ 158 | fse_bit_count accum_nbits; /* Number of valid bits in ACCUM, other bits are 0 */ 159 | } fse_in_stream32; 160 | 161 | /*! @abstract Initialize an output stream object. */ 162 | static __always_inline void fse_out_init64(fse_out_stream64 *s) 163 | { 164 | s->accum = 0; 165 | s->accum_nbits = 0; 166 | } 167 | 168 | /*! @abstract Initialize an output stream object. */ 169 | static __always_inline void fse_out_init32(fse_out_stream32 *s) 170 | { 171 | s->accum = 0; 172 | s->accum_nbits = 0; 173 | } 174 | 175 | /*! @abstract Write full bytes from the accumulator to output buffer, ensuring 176 | * accum_nbits is in [0, 7]. 177 | * We assume we can write 8 bytes to the output buffer \c (*pbuf[0..7]) in all 178 | * cases. 179 | * @note *pbuf is incremented by the number of written bytes. 180 | */ 181 | static __always_inline void fse_out_flush64(fse_out_stream64 *s, uint8_t **pbuf) 182 | { 183 | fse_bit_count nbits = s->accum_nbits & -8; /* number of bits written, multiple of 8 */ 184 | 185 | /* Write 8 bytes of current accumulator */ 186 | memcpy(*pbuf, &(s->accum), 8); 187 | *pbuf += (nbits >> 3); /* bytes */ 188 | 189 | /* Update state */ 190 | s->accum >>= nbits; /* remove nbits */ 191 | s->accum_nbits -= nbits; 192 | } 193 | 194 | /*! @abstract Write full bytes from the accumulator to output buffer, ensuring 195 | * accum_nbits is in [0, 7]. 196 | * We assume we can write 4 bytes to the output buffer \c (*pbuf[0..3]) in all 197 | * cases. 198 | * @note *pbuf is incremented by the number of written bytes. 199 | */ 200 | static __always_inline void fse_out_flush32(fse_out_stream32 *s, uint8_t **pbuf) 201 | { 202 | fse_bit_count nbits = s->accum_nbits & -8; /* number of bits written, multiple of 8 */ 203 | 204 | /* Write 4 bytes of current accumulator */ 205 | memcpy(*pbuf, &(s->accum), 4); 206 | *pbuf += (nbits >> 3); /* bytes */ 207 | 208 | /* Update state */ 209 | s->accum >>= nbits; /* remove nbits */ 210 | s->accum_nbits -= nbits; 211 | } 212 | 213 | /*! @abstract Write the last bytes from the accumulator to output buffer, 214 | * ensuring accum_nbits is in [-7, 0]. Bits are padded with 0 if needed. 215 | * We assume we can write 8 bytes to the output buffer \c (*pbuf[0..7]) in all 216 | * cases. 217 | * @note *pbuf is incremented by the number of written bytes. 218 | */ 219 | static __always_inline void fse_out_finish64(fse_out_stream64 *s, uint8_t **pbuf) 220 | { 221 | fse_bit_count nbits = (s->accum_nbits + 7) & -8; /* number of bits written, multiple of 8 */ 222 | 223 | /* Write 8 bytes of current accumulator */ 224 | memcpy(*pbuf, &(s->accum), 8); 225 | *pbuf += (nbits >> 3); /* bytes */ 226 | 227 | /* Update state */ 228 | s->accum = 0; /* remove nbits */ 229 | s->accum_nbits -= nbits; 230 | } 231 | 232 | /*! @abstract Write the last bytes from the accumulator to output buffer, 233 | * ensuring accum_nbits is in [-7, 0]. Bits are padded with 0 if needed. 234 | * We assume we can write 4 bytes to the output buffer \c (*pbuf[0..3]) in all 235 | * cases. 236 | * @note *pbuf is incremented by the number of written bytes. 237 | */ 238 | static __always_inline void fse_out_finish32(fse_out_stream32 *s, uint8_t **pbuf) 239 | { 240 | fse_bit_count nbits = (s->accum_nbits + 7) & -8; /* number of bits written, multiple of 8 */ 241 | 242 | /* Write 8 bytes of current accumulator */ 243 | memcpy(*pbuf, &(s->accum), 4); 244 | *pbuf += (nbits >> 3); /* bytes */ 245 | 246 | /* Update state */ 247 | s->accum = 0; /* remove nbits */ 248 | s->accum_nbits -= nbits; 249 | } 250 | 251 | /*! @abstract Accumulate \c n bits \c b to output stream \c s. We \b must have: 252 | * 0 <= b < 2^n, and N + s->accum_nbits <= 64. 253 | * @note The caller must ensure out_flush is called \b before the accumulator 254 | * overflows to more than 64 bits. 255 | */ 256 | static __always_inline void fse_out_push64(fse_out_stream64 *s, fse_bit_count n, uint64_t b) 257 | { 258 | s->accum |= b << s->accum_nbits; 259 | s->accum_nbits += n; 260 | } 261 | 262 | /*! @abstract Accumulate \c n bits \c b to output stream \c s. We \b must have: 263 | * 0 <= n < 2^n, and n + s->accum_nbits <= 32. 264 | * @note The caller must ensure out_flush is called \b before the accumulator 265 | * overflows to more than 32 bits. 266 | */ 267 | static __always_inline void fse_out_push32(fse_out_stream32 *s, fse_bit_count n, uint32_t b) 268 | { 269 | s->accum |= b << s->accum_nbits; 270 | s->accum_nbits += n; 271 | } 272 | 273 | #define DEBUG_CHECK_INPUT_STREAM_PARAMETERS 274 | 275 | /*! @abstract Initialize the fse input stream so that accum holds between 56 276 | * and 63 bits. We never want to have 64 bits in the stream, because that allows 277 | * us to avoid a special case in the fse_in_pull function (eliminating an 278 | * unpredictable branch), while not requiring any additional fse_flush 279 | * operations. This is why we have the special case for n == 0 (in which case 280 | * we want to load only 7 bytes instead of 8). 281 | */ 282 | static __always_inline int fse_in_checked_init64(fse_in_stream64 *s, fse_bit_count n, 283 | const uint8_t **pbuf, const uint8_t *buf_start) 284 | { 285 | if (n) { 286 | if (*pbuf < buf_start + 8) 287 | return -1; /* out of range */ 288 | *pbuf -= 8; 289 | memcpy(&(s->accum), *pbuf, 8); 290 | s->accum_nbits = n + 64; 291 | } else { 292 | if (*pbuf < buf_start + 7) 293 | return -1; /* out of range */ 294 | *pbuf -= 7; 295 | memcpy(&(s->accum), *pbuf, 7); 296 | s->accum &= 0xffffffffffffff; 297 | s->accum_nbits = n + 56; 298 | } 299 | 300 | if ((s->accum_nbits < 56 || s->accum_nbits >= 64) || ((s->accum >> s->accum_nbits) != 0)) { 301 | /* 302 | * the incoming input is wrong (encoder should have zeroed the 303 | * upper bits) 304 | */ 305 | return -1; 306 | } 307 | 308 | return 0; /* OK */ 309 | } 310 | 311 | /*! @abstract Identical to previous function, but for 32-bit operation 312 | * (resulting bit count is between 24 and 31 bits). 313 | */ 314 | static __always_inline int fse_in_checked_init32(fse_in_stream32 *s, fse_bit_count n, 315 | const uint8_t **pbuf, const uint8_t *buf_start) 316 | { 317 | if (n) { 318 | if (*pbuf < buf_start + 4) 319 | return -1; /* out of range */ 320 | *pbuf -= 4; 321 | memcpy(&(s->accum), *pbuf, 4); 322 | s->accum_nbits = n + 32; 323 | } else { 324 | if (*pbuf < buf_start + 3) 325 | return -1; /* out of range */ 326 | *pbuf -= 3; 327 | memcpy(&(s->accum), *pbuf, 3); 328 | s->accum &= 0xffffff; 329 | s->accum_nbits = n + 24; 330 | } 331 | 332 | if ((s->accum_nbits < 24 || s->accum_nbits >= 32) || ((s->accum >> s->accum_nbits) != 0)) { 333 | /* 334 | * the incoming input is wrong (encoder should have zeroed the 335 | * upper bits) 336 | */ 337 | return -1; 338 | } 339 | 340 | return 0; /* OK */ 341 | } 342 | 343 | /*! @abstract Read in new bytes from buffer to ensure that we have a full 344 | * complement of bits in the stream object (again, between 56 and 63 bits). 345 | * checking the new value of \c *pbuf remains >= \c buf_start. 346 | * @return 0 if OK. 347 | * @return -1 on failure. 348 | */ 349 | static __always_inline int fse_in_checked_flush64(fse_in_stream64 *s, const uint8_t **pbuf, 350 | const uint8_t *buf_start) 351 | { 352 | /* Get number of bits to add to bring us into the desired range. */ 353 | fse_bit_count nbits = (63 - s->accum_nbits) & -8; 354 | /* Convert bits to bytes and decrement buffer address, then load new data. */ 355 | const uint8_t *buf = (*pbuf) - (nbits >> 3); 356 | uint64_t incoming; 357 | 358 | if (buf < buf_start) 359 | return -1; /* out of range */ 360 | *pbuf = buf; 361 | memcpy(&incoming, buf, 8); 362 | /* Update the state object and verify its validity (in DEBUG). */ 363 | s->accum = (s->accum << nbits) | fse_mask_lsb64(incoming, nbits); 364 | s->accum_nbits += nbits; 365 | DEBUG_CHECK_INPUT_STREAM_PARAMETERS 366 | return 0; // OK 367 | } 368 | 369 | /*! @abstract Identical to previous function (but again, we're only filling 370 | * a 32-bit field with between 24 and 31 bits). 371 | */ 372 | static __always_inline int fse_in_checked_flush32(fse_in_stream32 *s, const uint8_t **pbuf, 373 | const uint8_t *buf_start) 374 | { 375 | /* Get number of bits to add to bring us into the desired range. */ 376 | fse_bit_count nbits = (31 - s->accum_nbits) & -8; 377 | 378 | if (nbits > 0) { 379 | /* Convert bits to bytes and decrement buffer address, then load new data. */ 380 | const uint8_t *buf = (*pbuf) - (nbits >> 3); 381 | uint32_t incoming; 382 | 383 | if (buf < buf_start) 384 | return -1; /* out of range */ 385 | 386 | *pbuf = buf; 387 | 388 | incoming = *((uint32_t *)buf); 389 | 390 | /* Update the state object and verify its validity (in DEBUG). */ 391 | s->accum = (s->accum << nbits) | fse_mask_lsb32(incoming, nbits); 392 | s->accum_nbits += nbits; 393 | } 394 | DEBUG_CHECK_INPUT_STREAM_PARAMETERS 395 | return 0; /* OK */ 396 | } 397 | 398 | /*! @abstract Pull n bits out of the fse stream object. */ 399 | static __always_inline uint64_t fse_in_pull64(fse_in_stream64 *s, fse_bit_count n) 400 | { 401 | uint64_t result; 402 | 403 | s->accum_nbits -= n; 404 | result = s->accum >> s->accum_nbits; 405 | s->accum = fse_mask_lsb64(s->accum, s->accum_nbits); 406 | return result; 407 | } 408 | 409 | /*! @abstract Pull n bits out of the fse stream object. */ 410 | static __always_inline uint32_t fse_in_pull32(fse_in_stream32 *s, fse_bit_count n) 411 | { 412 | uint32_t result; 413 | 414 | s->accum_nbits -= n; 415 | result = s->accum >> s->accum_nbits; 416 | s->accum = fse_mask_lsb32(s->accum, s->accum_nbits); 417 | return result; 418 | } 419 | 420 | /* MARK: - Encode/Decode */ 421 | 422 | /* Map to 32/64-bit implementations and types for I/O */ 423 | #if FSE_IOSTREAM_64 424 | 425 | typedef uint64_t fse_bits; 426 | typedef fse_out_stream64 fse_out_stream; 427 | typedef fse_in_stream64 fse_in_stream; 428 | #define fse_mask_lsb fse_mask_lsb64 429 | #define fse_extract_bits fse_extract_bits64 430 | #define fse_out_init fse_out_init64 431 | #define fse_out_flush fse_out_flush64 432 | #define fse_out_finish fse_out_finish64 433 | #define fse_out_push fse_out_push64 434 | #define fse_in_init fse_in_checked_init64 435 | #define fse_in_checked_init fse_in_checked_init64 436 | #define fse_in_flush fse_in_checked_flush64 437 | #define fse_in_checked_flush fse_in_checked_flush64 438 | #define fse_in_flush2(_unused, _parameters, _unused2) 0 /* nothing */ 439 | #define fse_in_checked_flush2(_unused, _parameters) /* nothing */ 440 | #define fse_in_pull fse_in_pull64 441 | 442 | #else 443 | 444 | typedef uint32_t fse_bits; 445 | typedef fse_out_stream32 fse_out_stream; 446 | typedef fse_in_stream32 fse_in_stream; 447 | #define fse_mask_lsb fse_mask_lsb32 448 | #define fse_extract_bits fse_extract_bits32 449 | #define fse_out_init fse_out_init32 450 | #define fse_out_flush fse_out_flush32 451 | #define fse_out_finish fse_out_finish32 452 | #define fse_out_push fse_out_push32 453 | #define fse_in_init fse_in_checked_init32 454 | #define fse_in_checked_init fse_in_checked_init32 455 | #define fse_in_flush fse_in_checked_flush32 456 | #define fse_in_checked_flush fse_in_checked_flush32 457 | #define fse_in_flush2 fse_in_checked_flush32 458 | #define fse_in_checked_flush2 fse_in_checked_flush32 459 | #define fse_in_pull fse_in_pull32 460 | 461 | #endif 462 | 463 | /*! @abstract Entry for one state in the decoder table (32b). */ 464 | typedef struct { /* DO NOT REORDER THE FIELDS */ 465 | int8_t k; /* Number of bits to read */ 466 | uint8_t symbol; /* Emitted symbol */ 467 | int16_t delta; /* Signed increment used to compute next state (+bias) */ 468 | } fse_decoder_entry; 469 | 470 | /*! @abstract Entry for one state in the value decoder table (64b). */ 471 | struct fse_value_decoder_entry { /* DO NOT REORDER THE FIELDS */ 472 | uint8_t total_bits; /* state bits + extra value bits = shift for next decode */ 473 | uint8_t value_bits; /* extra value bits */ 474 | int16_t delta; /* state base (delta) */ 475 | int32_t vbase; /* value base */ 476 | }; 477 | 478 | /*! @abstract Decode and return symbol using the decoder table, and update 479 | * \c *pstate, \c in. 480 | * @note The caller must ensure we have enough bits available in the input 481 | * stream accumulator. 482 | */ 483 | static __always_inline uint8_t fse_decode(fse_state *__restrict pstate, 484 | const int32_t *__restrict decoder_table, 485 | fse_in_stream *__restrict in) 486 | { 487 | int32_t e = decoder_table[*pstate]; 488 | 489 | /* Update state from K bits of input + DELTA */ 490 | *pstate = (fse_state)(e >> 16) + (fse_state)fse_in_pull(in, e & 0xff); 491 | 492 | /* Return the symbol for this state */ 493 | return fse_extract_bits(e, 8, 8); /* symbol */ 494 | } 495 | 496 | /*! @abstract Decode and return value using the decoder table, and update \c 497 | * *pstate, \c in. 498 | * \c value_decoder_table[nstates] 499 | * @note The caller must ensure we have enough bits available in the input 500 | * stream accumulator. 501 | */ 502 | static __always_inline int32_t fse_value_decode(fse_state *__restrict pstate, 503 | const struct fse_value_decoder_entry *value_decoder_table, 504 | fse_in_stream *__restrict in) 505 | { 506 | struct fse_value_decoder_entry entry = value_decoder_table[*pstate]; 507 | uint32_t state_and_value_bits = (uint32_t)fse_in_pull(in, entry.total_bits); 508 | *pstate = (fse_state)(entry.delta + (state_and_value_bits >> entry.value_bits)); 509 | return (int32_t)(entry.vbase + fse_mask_lsb(state_and_value_bits, entry.value_bits)); 510 | } 511 | 512 | /* MARK: - Tables */ 513 | 514 | /* 515 | * IMPORTANT: To properly decode an FSE encoded stream, both encoder/decoder 516 | * tables shall be initialized with the same parameters, including the 517 | * FREQ[NSYMBOL] array. 518 | */ 519 | 520 | /*! @abstract Sanity check on frequency table, verify sum of \c freq 521 | * is <= \c number_of_states. 522 | */ 523 | static __always_inline int fse_check_freq(const uint16_t *freq_table, const size_t table_size, 524 | const size_t number_of_states) 525 | { 526 | size_t sum_of_freq = 0; 527 | int i; 528 | 529 | for (i = 0; i < table_size; i++) 530 | sum_of_freq += freq_table[i]; 531 | return (sum_of_freq > number_of_states) ? -1 : 0; 532 | } 533 | 534 | /*! @abstract Initialize decoder table \c t[nstates]. 535 | * 536 | * @param nstates 537 | * sum \c freq[i]; the number of states (a power of 2). 538 | * 539 | * @param nsymbols 540 | * the number of symbols. 541 | * 542 | * @param feq[nsymbols] 543 | * a normalized histogram of symbol frequencies, with \c freq[i] >= 0. 544 | * Some symbols may have a 0 frequency. In that case they should not be 545 | * present in the data. 546 | * 547 | * @return 0 if OK. 548 | * @return -1 on failure. 549 | */ 550 | int fse_init_decoder_table(int nstates, int nsymbols, const uint16_t *__restrict freq, 551 | int32_t *__restrict t); 552 | 553 | /*! @abstract Initialize value decoder table \c t[nstates]. 554 | * 555 | * @param nstates 556 | * sum \cfreq[i]; the number of states (a power of 2). 557 | * 558 | * @param nsymbols 559 | * the number of symbols. 560 | * 561 | * @param freq[nsymbols] 562 | * a normalized histogram of symbol frequencies, with \c freq[i] >= 0. 563 | * \c symbol_vbits[nsymbols] and \c symbol_vbase[nsymbols] are the number of 564 | * value bits to read and the base value for each symbol. 565 | * Some symbols may have a 0 frequency. In that case they should not be 566 | * present in the data. 567 | */ 568 | void fse_init_value_decoder_table(int nstates, int nsymbols, const uint16_t *__restrict freq, 569 | const uint8_t *__restrict symbol_vbits, 570 | const int32_t *__restrict symbol_vbase, 571 | struct fse_value_decoder_entry *__restrict t); 572 | 573 | #endif /* LZFSE_FSE_H */ 574 | -------------------------------------------------------------------------------- /lzfse/lzfse_internal.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: BSD-3-Clause */ 2 | /* 3 | * Copyright (c) 2015-2016, Apple Inc. All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | * 7 | * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | * 9 | * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer 10 | * in the documentation and/or other materials provided with the distribution. 11 | * 12 | * 3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived 13 | * from this software without specific prior written permission. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 16 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 17 | * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 19 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 20 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 21 | */ 22 | 23 | #ifndef LZFSE_INTERNAL_H 24 | #define LZFSE_INTERNAL_H 25 | 26 | #include "lzfse_fse.h" 27 | #include 28 | #include 29 | 30 | /* 31 | * Throughout LZFSE we refer to "L", "M" and "D"; these will always appear as 32 | * a triplet, and represent a "usual" LZ-style literal and match pair. "L" 33 | * is the number of literal bytes, "M" is the number of match bytes, and "D" 34 | * is the match "distance"; the distance in bytes between the current pointer 35 | * and the start of the match. 36 | */ 37 | #define LZFSE_ENCODE_HASH_VALUES (1 << LZFSE_ENCODE_HASH_BITS) 38 | #define LZFSE_ENCODE_L_SYMBOLS 20 39 | #define LZFSE_ENCODE_M_SYMBOLS 20 40 | #define LZFSE_ENCODE_D_SYMBOLS 64 41 | #define LZFSE_ENCODE_LITERAL_SYMBOLS 256 42 | #define LZFSE_ENCODE_L_STATES 64 43 | #define LZFSE_ENCODE_M_STATES 64 44 | #define LZFSE_ENCODE_D_STATES 256 45 | #define LZFSE_ENCODE_LITERAL_STATES 1024 46 | #define LZFSE_MATCHES_PER_BLOCK 10000 47 | #define LZFSE_LITERALS_PER_BLOCK (4 * LZFSE_MATCHES_PER_BLOCK) 48 | #define LZFSE_DECODE_LITERALS_PER_BLOCK (4 * LZFSE_DECODE_MATCHES_PER_BLOCK) 49 | 50 | /* 51 | * LZFSE internal status. These values are used by internal LZFSE routines 52 | * as return codes. There should not be any good reason to change their 53 | * values; it is plausible that additional codes might be added in the 54 | * future. 55 | */ 56 | #define LZFSE_STATUS_OK 0 57 | #define LZFSE_STATUS_SRC_EMPTY -1 58 | #define LZFSE_STATUS_DST_FULL -2 59 | #define LZFSE_STATUS_ERROR -3 60 | 61 | /* 62 | * Type representing an offset between elements in a buffer. On 64-bit 63 | * systems, this is stored in a 64-bit container to avoid extra sign- 64 | * extension operations in addressing arithmetic, but the value is always 65 | * representable as a 32-bit signed value in LZFSE's usage. 66 | */ 67 | #if defined(_M_AMD64) || defined(__x86_64__) || defined(__arm64__) 68 | typedef int64_t lzfse_offset; 69 | #else 70 | typedef int32_t lzfse_offset; 71 | #endif 72 | 73 | typedef uint64_t uintmax_t; 74 | 75 | /*! @abstract Decoder state object for lzfse compressed blocks. */ 76 | typedef struct { 77 | /* Number of matches remaining in the block. */ 78 | uint32_t n_matches; 79 | /* Number of bytes used to encode L, M, D triplets for the block. */ 80 | uint32_t n_lmd_payload_bytes; 81 | /* Pointer to the next literal to emit. */ 82 | const uint8_t *current_literal; 83 | /* 84 | * L, M, D triplet for the match currently being emitted. This is used 85 | * only if we need to restart after reaching the end of the destination 86 | * buffer in the middle of a literal or match. 87 | */ 88 | int32_t l_value, m_value, d_value; 89 | /* FSE stream object. */ 90 | fse_in_stream lmd_in_stream; 91 | /* 92 | * Offset of L,M,D encoding in the input buffer. Because we read through 93 | * an FSE stream *backwards* while decoding, this is decremented as we 94 | * move through a block. 95 | */ 96 | uint32_t lmd_in_buf; 97 | /* The current state of the L, M, and D FSE decoders. */ 98 | uint16_t l_state, m_state, d_state; 99 | /* 100 | * Internal FSE decoder tables for the current block. These have 101 | * alignment forced to 8 bytes to guarantee that a single state's 102 | * entry cannot span two cachelines. 103 | */ 104 | struct fse_value_decoder_entry l_decoder[LZFSE_ENCODE_L_STATES] __aligned(8); 105 | struct fse_value_decoder_entry m_decoder[LZFSE_ENCODE_M_STATES] __aligned(8); 106 | struct fse_value_decoder_entry d_decoder[LZFSE_ENCODE_D_STATES] __aligned(8); 107 | int32_t literal_decoder[LZFSE_ENCODE_LITERAL_STATES]; 108 | /* 109 | * The literal stream for the block, plus padding to allow for faster 110 | * copy operations. 111 | */ 112 | uint8_t literals[LZFSE_LITERALS_PER_BLOCK + 64]; 113 | } lzfse_compressed_block_decoder_state; 114 | 115 | /* Decoder state object for uncompressed blocks. */ 116 | typedef struct { 117 | uint32_t n_raw_bytes; 118 | } uncompressed_block_decoder_state; 119 | 120 | /*! @abstract Decoder state object for lzvn-compressed blocks. */ 121 | typedef struct { 122 | uint32_t n_raw_bytes; 123 | uint32_t n_payload_bytes; 124 | uint32_t d_prev; 125 | } lzvn_compressed_block_decoder_state; 126 | 127 | /*! @abstract Decoder state object. */ 128 | typedef struct { 129 | /* 130 | * Pointer to next byte to read from source buffer (this is advanced as 131 | * we decode; src_begin describe the buffer and do not change). 132 | */ 133 | const uint8_t *src; 134 | /* Pointer to first byte of source buffer. */ 135 | const uint8_t *src_begin; 136 | /* Pointer to one byte past the end of the source buffer. */ 137 | const uint8_t *src_end; 138 | /* 139 | * Pointer to the next byte to write to destination buffer (this is 140 | * advanced as we decode; dst_begin and dst_end describe the buffer and 141 | * do not change). 142 | */ 143 | uint8_t *dst; 144 | /* Pointer to first byte of destination buffer. */ 145 | uint8_t *dst_begin; 146 | /* Pointer to one byte past the end of the destination buffer. */ 147 | uint8_t *dst_end; 148 | /* 1 if we have reached the end of the stream, 0 otherwise. */ 149 | int end_of_stream; 150 | /* 151 | * magic number of the current block if we are within a block, 152 | * LZFSE_NO_BLOCK_MAGIC otherwise. 153 | */ 154 | uint32_t block_magic; 155 | lzfse_compressed_block_decoder_state compressed_lzfse_block_state; 156 | lzvn_compressed_block_decoder_state compressed_lzvn_block_state; 157 | uncompressed_block_decoder_state uncompressed_block_state; 158 | } lzfse_decoder_state; 159 | 160 | /* MARK: - Block header objects */ 161 | 162 | #define LZFSE_NO_BLOCK_MAGIC 0x00000000 /* 0 (invalid) */ 163 | #define LZFSE_ENDOFSTREAM_BLOCK_MAGIC 0x24787662 /* bvx$ (end of stream) */ 164 | #define LZFSE_UNCOMPRESSED_BLOCK_MAGIC 0x2d787662 /* bvx- (raw data) */ 165 | #define LZFSE_COMPRESSEDV1_BLOCK_MAGIC 0x31787662 /* bvx1 (lzfse compressed, uncompressed tables) */ 166 | #define LZFSE_COMPRESSEDV2_BLOCK_MAGIC 0x32787662 /* bvx2 (lzfse compressed, compressed tables) */ 167 | #define LZFSE_COMPRESSEDLZVN_BLOCK_MAGIC 0x6e787662 /* bvxn (lzvn compressed) */ 168 | 169 | /*! @abstract Uncompressed block header in encoder stream. */ 170 | typedef struct { 171 | /* Magic number, always LZFSE_UNCOMPRESSED_BLOCK_MAGIC. */ 172 | uint32_t magic; 173 | /* Number of raw bytes in block. */ 174 | uint32_t n_raw_bytes; 175 | } uncompressed_block_header; 176 | 177 | /*! @abstract Compressed block header with uncompressed tables. */ 178 | typedef struct { 179 | /* Magic number, always LZFSE_COMPRESSEDV1_BLOCK_MAGIC. */ 180 | uint32_t magic; 181 | /* Number of decoded (output) bytes in block. */ 182 | uint32_t n_raw_bytes; 183 | /* Number of encoded (source) bytes in block. */ 184 | uint32_t n_payload_bytes; 185 | /* Number of literal bytes output by block (*not* the number of literals). */ 186 | uint32_t n_literals; 187 | /* Number of matches in block (which is also the number of literals). */ 188 | uint32_t n_matches; 189 | /* Number of bytes used to encode literals. */ 190 | uint32_t n_literal_payload_bytes; 191 | /* Number of bytes used to encode matches. */ 192 | uint32_t n_lmd_payload_bytes; 193 | 194 | /* 195 | * Final encoder states for the block, which will be the initial states 196 | * for the decoder: 197 | */ 198 | /* Final accum_nbits for literals stream. */ 199 | int32_t literal_bits; 200 | /* 201 | * There are four interleaved streams of literals, so there are four 202 | * final states. 203 | */ 204 | uint16_t literal_state[4]; 205 | /* accum_nbits for the l, m, d stream. */ 206 | int32_t lmd_bits; 207 | /* Final L (literal length) state. */ 208 | uint16_t l_state; 209 | /* Final M (match length) state. */ 210 | uint16_t m_state; 211 | /* Final D (match distance) state. */ 212 | uint16_t d_state; 213 | 214 | /* 215 | * Normalized frequency tables for each stream. Sum of values in each 216 | * array is the number of states. 217 | */ 218 | uint16_t l_freq[LZFSE_ENCODE_L_SYMBOLS]; 219 | uint16_t m_freq[LZFSE_ENCODE_M_SYMBOLS]; 220 | uint16_t d_freq[LZFSE_ENCODE_D_SYMBOLS]; 221 | uint16_t literal_freq[LZFSE_ENCODE_LITERAL_SYMBOLS]; 222 | } lzfse_compressed_block_header_v1; 223 | 224 | /*! @abstract Compressed block header with compressed tables. Note that because 225 | * freq[] is compressed, the structure-as-stored-in-the-stream is *truncated*; 226 | * we only store the used bytes of freq[]. This means that some extra care must 227 | * be taken when reading one of these headers from the stream. 228 | */ 229 | typedef struct { 230 | /* Magic number, always LZFSE_COMPRESSEDV2_BLOCK_MAGIC. */ 231 | uint32_t magic; 232 | /* Number of decoded (output) bytes in block. */ 233 | uint32_t n_raw_bytes; 234 | /* 235 | * The fields n_payload_bytes ... d_state from the 236 | * lzfse_compressed_block_header_v1 object are packed into three 64-bit 237 | * fields in the compressed header, as follows: 238 | * 239 | * offset bits value 240 | * 0 20 n_literals 241 | * 20 20 n_literal_payload_bytes 242 | * 40 20 n_matches 243 | * 60 3 literal_bits 244 | * 63 1 --- unused --- 245 | * 246 | * 0 10 literal_state[0] 247 | * 10 10 literal_state[1] 248 | * 20 10 literal_state[2] 249 | * 30 10 literal_state[3] 250 | * 40 20 n_lmd_payload_bytes 251 | * 60 3 lmd_bits 252 | * 63 1 --- unused --- 253 | * 254 | * 0 32 header_size (total header size in bytes; this does not 255 | * correspond to a field in the uncompressed header version, 256 | * but is required; we wouldn't know the size of the 257 | * compresssed header otherwise. 258 | * 32 10 l_state 259 | * 42 10 m_state 260 | * 52 10 d_state 261 | * 62 2 --- unused --- 262 | */ 263 | uint64_t packed_fields[3]; 264 | /* 265 | * Variable size freq tables, using a Huffman-style fixed encoding. 266 | * Size allocated here is an upper bound (all values stored on 16 bits). 267 | */ 268 | uint8_t freq[2 * (LZFSE_ENCODE_L_SYMBOLS + LZFSE_ENCODE_M_SYMBOLS + LZFSE_ENCODE_D_SYMBOLS + 269 | LZFSE_ENCODE_LITERAL_SYMBOLS)]; 270 | } __packed __aligned(1) lzfse_compressed_block_header_v2; 271 | 272 | /*! @abstract LZVN compressed block header. */ 273 | typedef struct { 274 | /* Magic number, always LZFSE_COMPRESSEDLZVN_BLOCK_MAGIC. */ 275 | uint32_t magic; 276 | /* Number of decoded (output) bytes. */ 277 | uint32_t n_raw_bytes; 278 | /* Number of encoded (source) bytes. */ 279 | uint32_t n_payload_bytes; 280 | } lzvn_compressed_block_header; 281 | 282 | int lzfse_decode(lzfse_decoder_state *s); 283 | size_t lzvn_decode_scratch_size(void); 284 | size_t lzvn_decode_buffer(void *__restrict dst, size_t dst_size, const void *__restrict src, 285 | size_t src_size); 286 | 287 | /*! @abstract Signed offset in buffers, stored on either 32 or 64 bits. */ 288 | #if defined(_M_AMD64) || defined(__x86_64__) || defined(__arm64__) 289 | typedef int64_t lzvn_offset; 290 | #else 291 | typedef int32_t lzvn_offset; 292 | #endif 293 | 294 | /* MARK: - LZFSE utility functions */ 295 | 296 | /*! @abstract Load bytes from memory location SRC. */ 297 | static __always_inline uint16_t load2(const void *ptr) 298 | { 299 | uint16_t data; 300 | 301 | memcpy(&data, ptr, sizeof(data)); 302 | return data; 303 | } 304 | 305 | static __always_inline uint32_t load4(const void *ptr) 306 | { 307 | uint32_t data; 308 | 309 | memcpy(&data, ptr, sizeof(data)); 310 | return data; 311 | } 312 | 313 | static __always_inline uint64_t load8(const void *ptr) 314 | { 315 | uint64_t data; 316 | 317 | memcpy(&data, ptr, sizeof(data)); 318 | return data; 319 | } 320 | 321 | /*! @abstract Store bytes to memory location DST. */ 322 | static __always_inline void store2(void *ptr, uint16_t data) 323 | { 324 | memcpy(ptr, &data, sizeof(data)); 325 | } 326 | 327 | static __always_inline void store4(void *ptr, uint32_t data) 328 | { 329 | memcpy(ptr, &data, sizeof(data)); 330 | } 331 | 332 | static __always_inline void store8(void *ptr, uint64_t data) 333 | { 334 | memcpy(ptr, &data, sizeof(data)); 335 | } 336 | 337 | /*! @abstract Load+store bytes from locations SRC to DST. Not intended for use 338 | * with overlapping buffers. Note that for LZ-style compression, you need 339 | * copies to behave like naive memcpy( ) implementations do, splatting the 340 | * leading sequence if the buffers overlap. This copy does not do that, so 341 | * should not be used with overlapping buffers. 342 | */ 343 | static __always_inline void copy8(void *dst, const void *src) 344 | { 345 | store8(dst, load8(src)); 346 | } 347 | static __always_inline void copy16(void *dst, const void *src) 348 | { 349 | uint64_t m0 = load8(src); 350 | uint64_t m1 = load8((const unsigned char *)src + 8); 351 | 352 | store8(dst, m0); 353 | store8((unsigned char *)dst + 8, m1); 354 | } 355 | 356 | /* 357 | * =============================================================== 358 | * Bitfield Operations 359 | */ 360 | 361 | /*! @abstract Extracts \p width bits from \p container, starting with \p lsb; if 362 | * we view \p container as a bit array, we extract \c container[lsb:lsb+width]. 363 | */ 364 | static __always_inline uintmax_t extract(uintmax_t container, unsigned int lsb, unsigned int width) 365 | { 366 | static const size_t container_width = sizeof(container) * 8; 367 | 368 | if (width == container_width) 369 | return container; 370 | return (container >> lsb) & (((uintmax_t)1 << width) - 1); 371 | } 372 | 373 | /*! @abstract Inserts \p width bits from \p data into \p container, starting with \p lsb. 374 | * Viewed as bit arrays, the operations is: 375 | * @code 376 | * container[:lsb] is unchanged 377 | * container[lsb:lsb+width] <-- data[0:width] 378 | * container[lsb+width:] is unchanged 379 | * @endcode 380 | */ 381 | static __always_inline uintmax_t insert(uintmax_t container, uintmax_t data, unsigned int lsb, 382 | unsigned int width) 383 | { 384 | static const size_t container_width = sizeof(container) * 8; 385 | uintmax_t mask; 386 | 387 | if (width == container_width) 388 | return container; 389 | mask = ((uintmax_t)1 << width) - 1; 390 | return (container & ~(mask << lsb)) | (data & mask) << lsb; 391 | } 392 | 393 | /*! @abstract Perform sanity checks on the values of lzfse_compressed_block_header_v1. 394 | * Test that the field values are in the allowed limits, verify that the 395 | * frequency tables sum to value less than total number of states. 396 | * @return 0 if all tests passed. 397 | * @return negative error code with 1 bit set for each failed test. 398 | */ 399 | static __always_inline int 400 | lzfse_check_block_header_v1(const lzfse_compressed_block_header_v1 *header) 401 | { 402 | int tests_results = 0; 403 | uint16_t literal_state[4]; 404 | int res; 405 | 406 | tests_results = tests_results | 407 | ((header->magic == LZFSE_COMPRESSEDV1_BLOCK_MAGIC) ? 0 : (1 << 0)); 408 | tests_results = tests_results | 409 | ((header->n_literals <= LZFSE_LITERALS_PER_BLOCK) ? 0 : (1 << 1)); 410 | tests_results = tests_results | 411 | ((header->n_matches <= LZFSE_MATCHES_PER_BLOCK) ? 0 : (1 << 2)); 412 | 413 | memcpy(literal_state, header->literal_state, sizeof(uint16_t) * 4); 414 | 415 | tests_results = tests_results | 416 | ((literal_state[0] < LZFSE_ENCODE_LITERAL_STATES) ? 0 : (1 << 3)); 417 | tests_results = tests_results | 418 | ((literal_state[1] < LZFSE_ENCODE_LITERAL_STATES) ? 0 : (1 << 4)); 419 | tests_results = tests_results | 420 | ((literal_state[2] < LZFSE_ENCODE_LITERAL_STATES) ? 0 : (1 << 5)); 421 | tests_results = tests_results | 422 | ((literal_state[3] < LZFSE_ENCODE_LITERAL_STATES) ? 0 : (1 << 6)); 423 | 424 | tests_results = tests_results | ((header->l_state < LZFSE_ENCODE_L_STATES) ? 0 : (1 << 7)); 425 | tests_results = tests_results | ((header->m_state < LZFSE_ENCODE_M_STATES) ? 0 : (1 << 8)); 426 | tests_results = tests_results | ((header->d_state < LZFSE_ENCODE_D_STATES) ? 0 : (1 << 9)); 427 | 428 | res = fse_check_freq(header->l_freq, LZFSE_ENCODE_L_SYMBOLS, LZFSE_ENCODE_L_STATES); 429 | tests_results = tests_results | ((res == 0) ? 0 : (1 << 10)); 430 | res = fse_check_freq(header->m_freq, LZFSE_ENCODE_M_SYMBOLS, LZFSE_ENCODE_M_STATES); 431 | tests_results = tests_results | ((res == 0) ? 0 : (1 << 11)); 432 | res = fse_check_freq(header->d_freq, LZFSE_ENCODE_D_SYMBOLS, LZFSE_ENCODE_D_STATES); 433 | tests_results = tests_results | ((res == 0) ? 0 : (1 << 12)); 434 | res = fse_check_freq(header->literal_freq, LZFSE_ENCODE_LITERAL_SYMBOLS, 435 | LZFSE_ENCODE_LITERAL_STATES); 436 | tests_results = tests_results | ((res == 0) ? 0 : (1 << 13)); 437 | 438 | if (tests_results) { 439 | /* each 1 bit is a test that failed (except for the sign bit) */ 440 | return tests_results | 0x80000000; 441 | } 442 | 443 | return 0; /* OK */ 444 | } 445 | 446 | /*! @abstract The L, M, D data streams are all encoded as a "base" value, which is 447 | * FSE-encoded, and an "extra bits" value, which is the difference between 448 | * value and base, and is simply represented as a raw bit value (because it 449 | * is the low-order bits of a larger number, not much entropy can be 450 | * extracted from these bits by more complex encoding schemes). The following 451 | * tables represent the number of low-order bits to encode separately and the 452 | * base values for each of L, M, and D. 453 | * 454 | * @note The inverse tables for mapping the other way are significantly larger. 455 | * Those tables have been split out to lzfse_encode_tables.h in order to keep 456 | * this file relatively small. 457 | */ 458 | static const uint8_t l_extra_bits[LZFSE_ENCODE_L_SYMBOLS] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459 | 0, 0, 0, 0, 0, 0, 2, 3, 5, 8 }; 460 | static const int32_t l_base_value[LZFSE_ENCODE_L_SYMBOLS] = { 461 | 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 20, 28, 60 462 | }; 463 | static const uint8_t m_extra_bits[LZFSE_ENCODE_M_SYMBOLS] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464 | 0, 0, 0, 0, 0, 0, 3, 5, 8, 11 }; 465 | static const int32_t m_base_value[LZFSE_ENCODE_M_SYMBOLS] = { 0, 1, 2, 3, 4, 5, 6, 466 | 7, 8, 9, 10, 11, 12, 13, 467 | 14, 15, 16, 24, 56, 312 }; 468 | static const uint8_t d_extra_bits[LZFSE_ENCODE_D_SYMBOLS] = { 469 | 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 470 | 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 471 | 11, 11, 11, 11, 12, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15 472 | }; 473 | static const int32_t d_base_value[LZFSE_ENCODE_D_SYMBOLS] = { 474 | 0, 1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 475 | 24, 28, 36, 44, 52, 60, 76, 92, 108, 124, 156, 476 | 188, 220, 252, 316, 380, 444, 508, 636, 764, 892, 1020, 477 | 1276, 1532, 1788, 2044, 2556, 3068, 3580, 4092, 5116, 6140, 7164, 478 | 8188, 10236, 12284, 14332, 16380, 20476, 24572, 28668, 32764, 40956, 49148, 479 | 57340, 65532, 81916, 98300, 114684, 131068, 163836, 196604, 229372 480 | }; 481 | 482 | #endif /* LZFSE_INTERNAL_H */ 483 | -------------------------------------------------------------------------------- /lzfse/lzvn_decode_base.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* 3 | * Copyright (c) 2015-2016, Apple Inc. All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | * 7 | * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | * 9 | * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer 10 | * in the documentation and/or other materials provided with the distribution. 11 | * 12 | * 3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived 13 | * from this software without specific prior written permission. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 16 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 17 | * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 19 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 20 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 21 | */ 22 | 23 | /* LZVN low-level decoder */ 24 | 25 | #include 26 | #include 27 | #include "lzvn_decode_base.h" 28 | 29 | /* Older kernel versions will still get an objtool warning here */ 30 | #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0) 31 | #define __annotate_jump_table 32 | #endif 33 | 34 | /* 35 | * Both the source and destination buffers are represented by a pointer and 36 | * a length; they are *always* updated in concert using this macro; however 37 | * many bytes the pointer is advanced, the length is decremented by the same 38 | * amount. Thus, pointer + length always points to the byte one past the end 39 | * of the buffer. 40 | */ 41 | #define PTR_LEN_INC(_pointer, _length, _increment) (_pointer += _increment, _length -= _increment) 42 | 43 | /* 44 | * Update state with current positions and distance, corresponding to the 45 | * beginning of an instruction in both streams 46 | */ 47 | #define UPDATE_GOOD (state->src = src_ptr, state->dst = dst_ptr, state->d_prev = D) 48 | 49 | void lzvn_decode(lzvn_decoder_state *state) 50 | { 51 | /* Jump table for all instructions */ 52 | static const void *opc_tbl[256] __annotate_jump_table = { 53 | &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&eos, &&lrg_d, &&sml_d, 54 | &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&nop, &&lrg_d, &&sml_d, &&sml_d, 55 | &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&nop, &&lrg_d, &&sml_d, &&sml_d, &&sml_d, 56 | &&sml_d, &&sml_d, &&sml_d, &&udef, &&lrg_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, 57 | &&sml_d, &&sml_d, &&udef, &&lrg_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, 58 | &&sml_d, &&udef, &&lrg_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, 59 | &&udef, &&lrg_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&udef, 60 | &&lrg_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d, 61 | &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d, &&sml_d, 62 | &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d, &&sml_d, &&sml_d, 63 | &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d, &&sml_d, &&sml_d, &&sml_d, 64 | &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, 65 | &&sml_d, &&sml_d, &&pre_d, &&lrg_d, &&udef, &&udef, &&udef, &&udef, &&udef, 66 | &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, 67 | &&udef, &&udef, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, 68 | &&lrg_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d, 69 | &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d, &&sml_d, 70 | &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d, &&med_d, &&med_d, 71 | &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, 72 | &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, 73 | &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, 74 | &&med_d, &&med_d, &&med_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, 75 | &&pre_d, &&lrg_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, 76 | &&lrg_d, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, 77 | &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&lrg_l, 78 | &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, 79 | &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&lrg_m, &&sml_m, &&sml_m, 80 | &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, 81 | &&sml_m, &&sml_m, &&sml_m, &&sml_m 82 | }; 83 | size_t src_len = state->src_end - state->src; 84 | size_t dst_len = state->dst_end - state->dst; 85 | const unsigned char *src_ptr = state->src; 86 | unsigned char *dst_ptr = state->dst; 87 | size_t D = state->d_prev; 88 | size_t M; 89 | size_t L; 90 | size_t opc_len; 91 | unsigned char opc; 92 | uint16_t opc23; 93 | 94 | if (src_len == 0 || dst_len == 0) 95 | return; /* empty buffer */ 96 | 97 | /* Do we have a partially expanded match saved in state? */ 98 | if (state->L != 0 || state->M != 0) { 99 | L = state->L; 100 | M = state->M; 101 | D = state->D; 102 | opc_len = 0; /* we already skipped the op */ 103 | state->L = state->M = state->D = 0; 104 | if (M == 0) 105 | goto copy_literal; 106 | if (L == 0) 107 | goto copy_match; 108 | goto copy_literal_and_match; 109 | } 110 | 111 | opc = src_ptr[0]; 112 | 113 | goto *opc_tbl[opc]; 114 | /* 115 | * =============================================================== 116 | * These four opcodes (sml_d, med_d, lrg_d, and pre_d) encode both a 117 | * literal and a match. The bulk of their implementations are shared; 118 | * each label here only does the work of setting the opcode length (not 119 | * including any literal bytes), and extracting the literal length, match 120 | * length, and match distance (except in pre_d). They then jump into the 121 | * shared implementation to actually output the literal and match bytes. 122 | * 123 | * No error checking happens in the first stage, except for ensuring that 124 | * the source has enough length to represent the full opcode before 125 | * reading past the first byte. 126 | */ 127 | sml_d: 128 | UPDATE_GOOD; 129 | /* 130 | * "small distance": This opcode has the structure LLMMMDDD DDDDDDDD 131 | * LITERAL where the length of literal (0-3 bytes) is encoded by the 132 | * high 2 bits of the first byte. We first extract the literal length so 133 | * we know how long the opcode is, then check that the source can hold 134 | * both this opcode and at least one byte of the next (because any valid 135 | * input stream must be terminated with an eos token). 136 | */ 137 | opc_len = 2; 138 | L = (size_t)extract(opc, 6, 2); 139 | M = (size_t)extract(opc, 3, 3) + 3; 140 | /* 141 | * We need to ensure that the source buffer is long enough that we can 142 | * safely read this entire opcode, the literal that follows, and the 143 | * first byte of the next opcode. Once we satisfy this requirement, we 144 | * can safely unpack the match distance. A check similar to this one is 145 | * present in all the opcode implementations. 146 | */ 147 | if (src_len <= opc_len + L) 148 | return; /* source truncated */ 149 | D = (size_t)extract(opc, 0, 3) << 8 | src_ptr[1]; 150 | goto copy_literal_and_match; 151 | 152 | med_d: 153 | UPDATE_GOOD; 154 | /* 155 | * "medium distance": This is a minor variant of the "small distance" 156 | * encoding, where we will now use two extra bytes instead of one to 157 | * encode the restof the match length and distance. This allows an extra 158 | * two bits for the match length, and an extra three bits for the match 159 | * distance. The full structure of the opcode is 160 | * 101LLMMM DDDDDDMM DDDDDDDD LITERAL. 161 | */ 162 | opc_len = 3; 163 | L = (size_t)extract(opc, 3, 2); 164 | if (src_len <= opc_len + L) 165 | return; /* source truncated */ 166 | opc23 = load2(&src_ptr[1]); 167 | M = (size_t)((extract(opc, 0, 3) << 2 | extract(opc23, 0, 2)) + 3); 168 | D = (size_t)extract(opc23, 2, 14); 169 | goto copy_literal_and_match; 170 | 171 | lrg_d: 172 | UPDATE_GOOD; 173 | /* 174 | * "large distance": This is another variant of the "small distance" 175 | * encoding, where we will now use two extra bytes to encode the match 176 | * distance, which allows distances up to 65535 to be represented. The 177 | * full structure of the opcode is LLMMM111 DDDDDDDD DDDDDDDD LITERAL. 178 | */ 179 | opc_len = 3; 180 | L = (size_t)extract(opc, 6, 2); 181 | M = (size_t)extract(opc, 3, 3) + 3; 182 | if (src_len <= opc_len + L) 183 | return; /* source truncated */ 184 | D = load2(&src_ptr[1]); 185 | goto copy_literal_and_match; 186 | 187 | pre_d: 188 | UPDATE_GOOD; 189 | /* 190 | * "previous distance": This opcode has the structure LLMMM110, where 191 | * the length of the literal (0-3 bytes) is encoded by the high 2 bits 192 | * of the first byte. We first extract the literal length so we know how 193 | * long the opcode is, then check that the source can hold both this 194 | * opcode and at least one byte of the next (because any valid input 195 | * stream must be terminated with an eos token). 196 | */ 197 | opc_len = 1; 198 | L = (size_t)extract(opc, 6, 2); 199 | M = (size_t)extract(opc, 3, 3) + 3; 200 | if (src_len <= opc_len + L) 201 | return; /* source truncated */ 202 | goto copy_literal_and_match; 203 | 204 | copy_literal_and_match: 205 | /* 206 | * Common implementation of writing data for opcodes that have both a 207 | * literal and a match. We begin by advancing the source pointer past 208 | * the opcode, so that it points at the first literal byte (if L 209 | * is non-zero; otherwise it points at the next opcode). 210 | */ 211 | PTR_LEN_INC(src_ptr, src_len, opc_len); 212 | /* Now we copy the literal from the source pointer to the destination */ 213 | if (__builtin_expect(dst_len >= 4 && src_len >= 4, 1)) { 214 | /* 215 | * The literal is 0-3 bytes; if we are not near the end of the 216 | * buffer, we can safely just do a 4 byte copy (which is 217 | * guaranteed to cover the complete literal, and may include 218 | * some other bytes as well). 219 | */ 220 | store4(dst_ptr, load4(src_ptr)); 221 | } else if (L <= dst_len) { 222 | /* 223 | * We are too close to the end of either the input or output 224 | * stream to be able to safely use a four-byte copy, but we will 225 | * not exhaust either stream (we already know that the source 226 | * will not be exhausted from checks in the individual opcode 227 | * implementations, and we just tested that dst_len > L). Thus, 228 | * we need to do a byte-by-byte copy of the literal. This is 229 | * slow, but it can only ever happen near the very end of a 230 | * buffer, so it is not an important case to optimize. 231 | */ 232 | size_t i; 233 | 234 | for (i = 0; i < L; ++i) 235 | dst_ptr[i] = src_ptr[i]; 236 | } else { 237 | /* Destination truncated: fill DST, and store partial match */ 238 | 239 | /* Copy partial literal */ 240 | size_t i; 241 | 242 | for (i = 0; i < dst_len; ++i) 243 | dst_ptr[i] = src_ptr[i]; 244 | /* Save state */ 245 | state->src = src_ptr + dst_len; 246 | state->dst = dst_ptr + dst_len; 247 | state->L = L - dst_len; 248 | state->M = M; 249 | state->D = D; 250 | return; /* destination truncated */ 251 | } 252 | /* 253 | * Having completed the copy of the literal, we advance both the source 254 | * and destination pointers by the number of literal bytes. 255 | */ 256 | PTR_LEN_INC(dst_ptr, dst_len, L); 257 | PTR_LEN_INC(src_ptr, src_len, L); 258 | /* 259 | * Check if the match distance is valid; matches must not reference 260 | * bytes that preceed the start of the output buffer, nor can the match 261 | * distance be zero. 262 | */ 263 | if (D > dst_ptr - state->dst_begin || D == 0) 264 | goto invalid_match_distance; 265 | copy_match: 266 | /* 267 | * Now we copy the match from dst_ptr - D to dst_ptr. It is important to 268 | * keep in mind that we may have D < M, in which case the source and 269 | * destination windows overlap in the copy. The semantics of the match 270 | * copy are *not* those of memmove( ); if the buffers overlap it needs 271 | * to behave as though we were copying byte-by-byte in increasing 272 | * address order. If, for example, D is 1, the copy operation is 273 | * equivalent to: 274 | * 275 | * memset(dst_ptr, dst_ptr[-1], M); 276 | * 277 | * i.e. it splats the previous byte. This means that we need to be very 278 | * careful about using wide loads or stores to perform the copy 279 | * operation. 280 | */ 281 | if (__builtin_expect(dst_len >= M + 7 && D >= 8, 1)) { 282 | /* 283 | * We are not near the end of the buffer, and the match distance 284 | * is at least eight. Thus, we can safely loop using eight byte 285 | * copies. The last of these may slop over the intended end of 286 | * the match, but this is OK because we know we have a safety 287 | * bound away from the end of the destination buffer. 288 | */ 289 | size_t i; 290 | 291 | for (i = 0; i < M; i += 8) 292 | store8(&dst_ptr[i], load8(&dst_ptr[i - D])); 293 | } else if (M <= dst_len) { 294 | /* 295 | * Either the match distance is too small, or we are too close 296 | * to the end of the buffer to safely use eight byte copies. 297 | * Fall back on a simple byte-by-byte implementation. 298 | */ 299 | size_t i; 300 | 301 | for (i = 0; i < M; ++i) 302 | dst_ptr[i] = dst_ptr[i - D]; 303 | } else { 304 | /* Destination truncated: fill DST, and store partial match */ 305 | 306 | /* Copy partial match */ 307 | size_t i; 308 | 309 | for (i = 0; i < dst_len; ++i) 310 | dst_ptr[i] = dst_ptr[i - D]; 311 | /* Save state */ 312 | state->src = src_ptr; 313 | state->dst = dst_ptr + dst_len; 314 | state->L = 0; 315 | state->M = M - dst_len; 316 | state->D = D; 317 | return; /* destination truncated */ 318 | } 319 | /* 320 | * Update the destination pointer and length to account for the bytes 321 | * written by the match, then load the next opcode byte and branch to 322 | * the appropriate implementation. 323 | */ 324 | PTR_LEN_INC(dst_ptr, dst_len, M); 325 | opc = src_ptr[0]; 326 | goto *opc_tbl[opc]; 327 | 328 | /* 329 | * =============================================================== 330 | * Opcodes representing only a match (no literal). 331 | * These two opcodes (lrg_m and sml_m) encode only a match. The match 332 | * distance is carried over from the previous opcode, so all they need 333 | * to encode is the match length. We are able to reuse the match copy 334 | * sequence from the literal and match opcodes to perform the actual 335 | * copy implementation. 336 | */ 337 | sml_m: 338 | UPDATE_GOOD; 339 | /* 340 | * "small match": This opcode has no literal, and uses the previous 341 | * match distance (i.e. it encodes only the match length), in a single 342 | * byte as 1111MMMM. 343 | */ 344 | opc_len = 1; 345 | if (src_len <= opc_len) 346 | return; /* source truncated */ 347 | M = (size_t)extract(opc, 0, 4); 348 | PTR_LEN_INC(src_ptr, src_len, opc_len); 349 | goto copy_match; 350 | 351 | lrg_m: 352 | UPDATE_GOOD; 353 | /* 354 | * "large match": This opcode has no literal, and uses the previous 355 | * match distance (i.e. it encodes only the match length). It is encoded 356 | * in two bytes as 11110000 MMMMMMMM. Because matches smaller than 16 357 | * bytes can be represented by sml_m, there is an implicit bias of 16 on 358 | * the match length; the representable values are [16,271]. 359 | */ 360 | opc_len = 2; 361 | if (src_len <= opc_len) 362 | return; /* source truncated */ 363 | M = src_ptr[1] + 16; 364 | PTR_LEN_INC(src_ptr, src_len, opc_len); 365 | goto copy_match; 366 | 367 | /* 368 | * =============================================================== 369 | * Opcodes representing only a literal (no match). 370 | * These two opcodes (lrg_l and sml_l) encode only a literal. There is no 371 | * match length or match distance to worry about (but we need to *not* 372 | * touch D, as it must be preserved between opcodes). 373 | */ 374 | sml_l: 375 | UPDATE_GOOD; 376 | /* 377 | * "small literal": This opcode has no match, and encodes only a literal 378 | * of length up to 15 bytes. The format is 1110LLLL LITERAL. 379 | */ 380 | opc_len = 1; 381 | L = (size_t)extract(opc, 0, 4); 382 | goto copy_literal; 383 | 384 | lrg_l: 385 | UPDATE_GOOD; 386 | /* 387 | * "large literal": This opcode has no match, and uses the previous 388 | * match distance (i.e. it encodes only the match length). It is encoded 389 | * in two bytes as 11100000 LLLLLLLL LITERAL. Because literals smaller 390 | * than 16 bytes can be represented by sml_l, there is an implicit bias 391 | * of 16 on the literal length; the representable values are [16,271]. 392 | */ 393 | opc_len = 2; 394 | if (src_len <= 2) 395 | return; /* source truncated */ 396 | L = src_ptr[1] + 16; 397 | goto copy_literal; 398 | 399 | copy_literal: 400 | /* 401 | * Check that the source buffer is large enough to hold the complete 402 | * literal and at least the first byte of the next opcode. If so, 403 | * advance the source pointer to point to the first byte of the literal 404 | * and adjust the source length accordingly. 405 | */ 406 | if (src_len <= opc_len + L) 407 | return; /* source truncated */ 408 | PTR_LEN_INC(src_ptr, src_len, opc_len); 409 | /* Now we copy the literal from the source pointer to the destination */ 410 | if (dst_len >= L + 7 && src_len >= L + 7) { 411 | /* 412 | * We are not near the end of the source or destination buffers; 413 | * thus we can safely copy the literal using wide copies, 414 | * without worrying about reading or writing past the end of 415 | * either buffer. 416 | */ 417 | size_t i; 418 | 419 | for (i = 0; i < L; i += 8) 420 | store8(&dst_ptr[i], load8(&src_ptr[i])); 421 | } else if (L <= dst_len) { 422 | /* 423 | * We are too close to the end of either the input or output 424 | * stream to be able to safely use an eight-byte copy. Instead 425 | * we copy the literal byte-by-byte. 426 | */ 427 | size_t i; 428 | 429 | for (i = 0; i < L; ++i) 430 | dst_ptr[i] = src_ptr[i]; 431 | } else { 432 | /* Destination truncated: fill DST, and store partial match */ 433 | 434 | /* Copy partial literal */ 435 | size_t i; 436 | 437 | for (i = 0; i < dst_len; ++i) 438 | dst_ptr[i] = src_ptr[i]; 439 | /* Save state */ 440 | state->src = src_ptr + dst_len; 441 | state->dst = dst_ptr + dst_len; 442 | state->L = L - dst_len; 443 | state->M = 0; 444 | state->D = D; 445 | return; /* destination truncated */ 446 | } 447 | /* 448 | * Having completed the copy of the literal, we advance both the source 449 | * and destination pointers by the number of literal bytes. 450 | */ 451 | PTR_LEN_INC(dst_ptr, dst_len, L); 452 | PTR_LEN_INC(src_ptr, src_len, L); 453 | /* Load the first byte of the next opcode, and jump to its implementation */ 454 | opc = src_ptr[0]; 455 | goto *opc_tbl[opc]; 456 | 457 | /* 458 | * =============================================================== 459 | * Other opcodes 460 | */ 461 | nop: 462 | UPDATE_GOOD; 463 | opc_len = 1; 464 | if (src_len <= opc_len) 465 | return; /* source truncated */ 466 | PTR_LEN_INC(src_ptr, src_len, opc_len); 467 | opc = src_ptr[0]; 468 | goto *opc_tbl[opc]; 469 | 470 | eos: 471 | opc_len = 8; 472 | if (src_len < opc_len) 473 | return; /* source truncated (here we don't need an extra byte for next op code) */ 474 | PTR_LEN_INC(src_ptr, src_len, opc_len); 475 | state->end_of_stream = 1; 476 | UPDATE_GOOD; 477 | return; /* end-of-stream */ 478 | 479 | /* 480 | * =============================================================== 481 | * Return on error 482 | */ 483 | udef: 484 | invalid_match_distance: 485 | 486 | return; /* we already updated state */ 487 | } 488 | -------------------------------------------------------------------------------- /lzfse/lzvn_decode_base.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: BSD-3-Clause */ 2 | /* 3 | * Copyright (c) 2015-2016, Apple Inc. All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | * 7 | * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | * 9 | * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer 10 | * in the documentation and/or other materials provided with the distribution. 11 | * 12 | * 3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived 13 | * from this software without specific prior written permission. 14 | * 15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 16 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 17 | * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 19 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 20 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 21 | */ 22 | 23 | /* 24 | * LZVN low-level decoder (v2) 25 | * Functions in the low-level API should switch to these at some point. 26 | * Apr 2014 27 | */ 28 | 29 | #ifndef LZVN_DECODE_BASE_H 30 | #define LZVN_DECODE_BASE_H 31 | 32 | #include "lzfse_internal.h" 33 | 34 | /*! @abstract Base decoder state. */ 35 | typedef struct { 36 | /* Decoder I/O */ 37 | 38 | /* Next byte to read in source buffer */ 39 | const unsigned char *src; 40 | /* Next byte after source buffer */ 41 | const unsigned char *src_end; 42 | 43 | /* Next byte to write in destination buffer (by decoder) */ 44 | unsigned char *dst; 45 | /* Valid range for destination buffer is [dst_begin, dst_end - 1] */ 46 | unsigned char *dst_begin; 47 | unsigned char *dst_end; 48 | /* Next byte to read in destination buffer (modified by caller) */ 49 | unsigned char *dst_current; 50 | 51 | /* Decoder state */ 52 | 53 | /* 54 | * Partially expanded match, or 0,0,0. 55 | * In that case, src points to the next literal to copy, or the next 56 | * op-code if L==0. 57 | */ 58 | size_t L, M, D; 59 | 60 | /* Distance for last emitted match, or 0 */ 61 | lzvn_offset d_prev; 62 | 63 | /* Did we decode end-of-stream? */ 64 | int end_of_stream; 65 | 66 | } lzvn_decoder_state; 67 | 68 | /*! @abstract Decode source to destination. 69 | * Updates \p state (src,dst,d_prev). 70 | */ 71 | void lzvn_decode(lzvn_decoder_state *state); 72 | 73 | #endif /* LZVN_DECODE_BASE_H */ 74 | -------------------------------------------------------------------------------- /message.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright (C) 2018 Ernesto A. Fernández 4 | */ 5 | 6 | #include 7 | #include "apfs.h" 8 | 9 | void apfs_msg(struct super_block *sb, const char *prefix, const char *func, int line, const char *fmt, ...) 10 | { 11 | char *sb_id = NULL; 12 | struct va_format vaf; 13 | va_list args; 14 | 15 | va_start(args, fmt); 16 | 17 | vaf.fmt = fmt; 18 | vaf.va = &args; 19 | 20 | /* The superblock is not available to all callers */ 21 | sb_id = sb ? sb->s_id : "?"; 22 | 23 | if (func) 24 | printk("%sAPFS (%s): %pV (%s:%d)\n", prefix, sb_id, &vaf, func, line); 25 | else 26 | printk("%sAPFS (%s): %pV\n", prefix, sb_id, &vaf); 27 | 28 | va_end(args); 29 | } 30 | -------------------------------------------------------------------------------- /namei.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright (C) 2018 Ernesto A. Fernández 4 | */ 5 | 6 | #include 7 | #include "apfs.h" 8 | #include "unicode.h" 9 | 10 | static struct dentry *apfs_lookup(struct inode *dir, struct dentry *dentry, 11 | unsigned int flags) 12 | { 13 | struct inode *inode = NULL; 14 | u64 ino = 0; 15 | int err; 16 | 17 | if (dentry->d_name.len > APFS_NAME_LEN) 18 | return ERR_PTR(-ENAMETOOLONG); 19 | 20 | err = apfs_inode_by_name(dir, &dentry->d_name, &ino); 21 | if (err && err != -ENODATA) { 22 | apfs_err(dir->i_sb, "inode lookup by name failed"); 23 | return ERR_PTR(err); 24 | } 25 | 26 | if (!err) { 27 | inode = apfs_iget(dir->i_sb, ino); 28 | if (IS_ERR(inode)) { 29 | apfs_err(dir->i_sb, "iget failed"); 30 | return ERR_CAST(inode); 31 | } 32 | } 33 | 34 | return d_splice_alias(inode, dentry); 35 | } 36 | 37 | #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 12, 0) 38 | static int apfs_symlink(struct inode *dir, struct dentry *dentry, 39 | const char *symname) 40 | #elif LINUX_VERSION_CODE < KERNEL_VERSION(6, 3, 0) && !RHEL_VERSION_GE(9, 6) 41 | static int apfs_symlink(struct user_namespace *mnt_userns, struct inode *dir, 42 | struct dentry *dentry, const char *symname) 43 | #else 44 | static int apfs_symlink(struct mnt_idmap *idmap, struct inode *dir, 45 | struct dentry *dentry, const char *symname) 46 | #endif 47 | { 48 | /* Symlink permissions don't mean anything and their value is fixed */ 49 | return apfs_mkany(dir, dentry, S_IFLNK | 0x1ed, 0 /* rdev */, symname); 50 | } 51 | 52 | const struct inode_operations apfs_dir_inode_operations = { 53 | .create = apfs_create, 54 | .lookup = apfs_lookup, 55 | .link = apfs_link, 56 | .unlink = apfs_unlink, 57 | .symlink = apfs_symlink, 58 | .mkdir = apfs_mkdir, 59 | .rmdir = apfs_rmdir, 60 | .mknod = apfs_mknod, 61 | .rename = apfs_rename, 62 | .getattr = apfs_getattr, 63 | .listxattr = apfs_listxattr, 64 | .setattr = apfs_setattr, 65 | .update_time = apfs_update_time, 66 | #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 13, 0) 67 | .fileattr_get = apfs_fileattr_get, 68 | .fileattr_set = apfs_fileattr_set, 69 | #endif 70 | }; 71 | 72 | const struct inode_operations apfs_special_inode_operations = { 73 | .getattr = apfs_getattr, 74 | .listxattr = apfs_listxattr, 75 | .setattr = apfs_setattr, 76 | .update_time = apfs_update_time, 77 | }; 78 | 79 | static int apfs_dentry_hash(const struct dentry *dir, struct qstr *child) 80 | { 81 | struct apfs_unicursor cursor; 82 | unsigned long hash; 83 | bool case_fold = apfs_is_case_insensitive(dir->d_sb); 84 | 85 | if (!apfs_is_normalization_insensitive(dir->d_sb)) 86 | return 0; 87 | 88 | apfs_init_unicursor(&cursor, child->name, child->len); 89 | hash = init_name_hash(dir); 90 | 91 | while (1) { 92 | int i; 93 | unicode_t utf32; 94 | 95 | utf32 = apfs_normalize_next(&cursor, case_fold); 96 | if (!utf32) 97 | break; 98 | 99 | /* Hash the unicode character one byte at a time */ 100 | for (i = 0; i < 4; ++i) { 101 | hash = partial_name_hash((u8)utf32, hash); 102 | utf32 = utf32 >> 8; 103 | } 104 | } 105 | child->hash = end_name_hash(hash); 106 | 107 | /* TODO: return error instead of truncating invalid UTF-8? */ 108 | return 0; 109 | } 110 | 111 | static int apfs_dentry_compare(const struct dentry *dentry, unsigned int len, 112 | const char *str, const struct qstr *name) 113 | { 114 | return apfs_filename_cmp(dentry->d_sb, name->name, name->len, str, len); 115 | } 116 | 117 | #if LINUX_VERSION_CODE < KERNEL_VERSION(6, 14, 0) 118 | static int apfs_dentry_revalidate(struct dentry *dentry, unsigned int flags) 119 | #else 120 | static int apfs_dentry_revalidate(struct inode *dir, const struct qstr *name, struct dentry *dentry, unsigned int flags) 121 | #endif 122 | { 123 | struct super_block *sb = dentry->d_sb; 124 | 125 | if (flags & LOOKUP_RCU) 126 | return -ECHILD; 127 | 128 | /* 129 | * If we want to create a link with a name that normalizes to the same 130 | * as an existing negative dentry, then we first need to invalidate the 131 | * dentry; otherwise it would keep the existing name. 132 | */ 133 | if (d_really_is_positive(dentry)) 134 | return 1; 135 | if (!apfs_is_case_insensitive(sb) && !apfs_is_normalization_insensitive(sb)) 136 | return 1; 137 | if (flags & (LOOKUP_CREATE | LOOKUP_RENAME_TARGET)) 138 | return 0; 139 | return 1; 140 | } 141 | 142 | const struct dentry_operations apfs_dentry_operations = { 143 | .d_revalidate = apfs_dentry_revalidate, 144 | .d_hash = apfs_dentry_hash, 145 | .d_compare = apfs_dentry_compare, 146 | }; 147 | -------------------------------------------------------------------------------- /object.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Checksum routines for an APFS object 4 | */ 5 | 6 | #include 7 | #include 8 | #include "apfs.h" 9 | 10 | /* 11 | * Note that this is not a generic implementation of fletcher64, as it assumes 12 | * a message length that doesn't overflow sum1 and sum2. This constraint is ok 13 | * for apfs, though, since the block size is limited to 2^16. For a more 14 | * generic optimized implementation, see Nakassis (1988). 15 | */ 16 | static u64 apfs_fletcher64(void *addr, size_t len) 17 | { 18 | __le32 *buff = addr; 19 | u64 sum1 = 0; 20 | u64 sum2 = 0; 21 | u64 c1, c2; 22 | int i, count_32; 23 | 24 | count_32 = len >> 2; 25 | for (i = 0; i < count_32; i++) { 26 | sum1 += le32_to_cpu(buff[i]); 27 | sum2 += sum1; 28 | } 29 | 30 | c1 = sum1 + sum2; 31 | c1 = 0xFFFFFFFF - do_div(c1, 0xFFFFFFFF); 32 | c2 = sum1 + c1; 33 | c2 = 0xFFFFFFFF - do_div(c2, 0xFFFFFFFF); 34 | 35 | return (c2 << 32) | c1; 36 | } 37 | 38 | int apfs_obj_verify_csum(struct super_block *sb, struct buffer_head *bh) 39 | { 40 | /* The checksum may be stale until the transaction is committed */ 41 | if (buffer_trans(bh)) 42 | return 1; 43 | return apfs_multiblock_verify_csum(bh->b_data, sb->s_blocksize); 44 | } 45 | 46 | /** 47 | * apfs_multiblock_verify_csum - Verify an object's checksum 48 | * @object: the object to verify 49 | * @size: size of the object in bytes (may be multiple blocks) 50 | * 51 | * Returns 1 on success, 0 on failure. 52 | */ 53 | int apfs_multiblock_verify_csum(char *object, u32 size) 54 | { 55 | struct apfs_obj_phys *obj = (struct apfs_obj_phys *)object; 56 | u64 actual_csum, header_csum; 57 | 58 | header_csum = le64_to_cpu(obj->o_cksum); 59 | actual_csum = apfs_fletcher64(object + APFS_MAX_CKSUM_SIZE, size - APFS_MAX_CKSUM_SIZE); 60 | return header_csum == actual_csum; 61 | } 62 | 63 | /** 64 | * apfs_obj_set_csum - Set the fletcher checksum in an object header 65 | * @sb: superblock structure 66 | * @obj: the object header 67 | * 68 | * The object must have a length of a single block. 69 | */ 70 | void apfs_obj_set_csum(struct super_block *sb, struct apfs_obj_phys *obj) 71 | { 72 | apfs_multiblock_set_csum((char *)obj, sb->s_blocksize); 73 | } 74 | 75 | /** 76 | * apfs_multiblock_set_csum - Set an object's checksum 77 | * @object: the object to checksum 78 | * @size: size of the object in bytes (may be multiple blocks) 79 | */ 80 | void apfs_multiblock_set_csum(char *object, u32 size) 81 | { 82 | struct apfs_obj_phys *obj = (struct apfs_obj_phys *)object; 83 | u64 cksum; 84 | 85 | cksum = apfs_fletcher64(object + APFS_MAX_CKSUM_SIZE, size - APFS_MAX_CKSUM_SIZE); 86 | obj->o_cksum = cpu_to_le64(cksum); 87 | } 88 | 89 | /** 90 | * apfs_create_cpm_block - Create a new checkpoint-mapping block 91 | * @sb: filesystem superblock 92 | * @bno: block number to use 93 | * @bh_p: on return, the buffer head for the block 94 | * 95 | * Returns 0 on success or a negative error code in case of failure. 96 | */ 97 | int apfs_create_cpm_block(struct super_block *sb, u64 bno, struct buffer_head **bh_p) 98 | { 99 | struct apfs_nxsb_info *nxi = APFS_NXI(sb); 100 | struct apfs_checkpoint_map_phys *cpm = NULL; 101 | struct buffer_head *bh = NULL; 102 | int err; 103 | 104 | bh = apfs_getblk(sb, bno); 105 | if (!bh) { 106 | apfs_err(sb, "failed to map cpm block"); 107 | return -EIO; 108 | } 109 | err = apfs_transaction_join(sb, bh); 110 | if (err) { 111 | brelse(bh); 112 | return err; 113 | } 114 | set_buffer_csum(bh); 115 | 116 | cpm = (void *)bh->b_data; 117 | memset(cpm, 0, sb->s_blocksize); 118 | cpm->cpm_o.o_oid = cpu_to_le64(bno); 119 | cpm->cpm_o.o_xid = cpu_to_le64(nxi->nx_xid); 120 | cpm->cpm_o.o_type = cpu_to_le32(APFS_OBJ_PHYSICAL | APFS_OBJECT_TYPE_CHECKPOINT_MAP); 121 | cpm->cpm_o.o_subtype = cpu_to_le32(APFS_OBJECT_TYPE_INVALID); 122 | 123 | /* For now: the caller will have to update these fields */ 124 | cpm->cpm_flags = cpu_to_le32(APFS_CHECKPOINT_MAP_LAST); 125 | cpm->cpm_count = 0; 126 | 127 | *bh_p = bh; 128 | return 0; 129 | } 130 | 131 | /** 132 | * apfs_create_cpoint_map - Create a checkpoint mapping for an object 133 | * @sb: filesystem superblock 134 | * @cpm: checkpoint mapping block to use 135 | * @obj: header for the ephemeral object 136 | * @bno: block number for the ephemeral object 137 | * @size: size of the ephemeral object in bytes 138 | * 139 | * Returns 0 on success or a negative error code in case of failure, which may 140 | * be -ENOSPC if @cpm is full. 141 | */ 142 | int apfs_create_cpoint_map(struct super_block *sb, struct apfs_checkpoint_map_phys *cpm, struct apfs_obj_phys *obj, u64 bno, u32 size) 143 | { 144 | struct apfs_checkpoint_mapping *map = NULL; 145 | u32 cpm_count; 146 | 147 | apfs_assert_in_transaction(sb, &cpm->cpm_o); 148 | 149 | cpm_count = le32_to_cpu(cpm->cpm_count); 150 | if (cpm_count >= apfs_max_maps_per_block(sb)) 151 | return -ENOSPC; 152 | map = &cpm->cpm_map[cpm_count]; 153 | le32_add_cpu(&cpm->cpm_count, 1); 154 | 155 | map->cpm_type = obj->o_type; 156 | map->cpm_subtype = obj->o_subtype; 157 | map->cpm_size = cpu_to_le32(size); 158 | map->cpm_pad = 0; 159 | map->cpm_fs_oid = 0; 160 | map->cpm_oid = obj->o_oid; 161 | map->cpm_paddr = cpu_to_le64(bno); 162 | return 0; 163 | } 164 | 165 | /** 166 | * apfs_index_in_data_area - Get position of block in current checkpoint's data 167 | * @sb: superblock structure 168 | * @bno: block number 169 | */ 170 | u32 apfs_index_in_data_area(struct super_block *sb, u64 bno) 171 | { 172 | struct apfs_nx_superblock *raw_sb = APFS_NXI(sb)->nx_raw; 173 | u64 data_base = le64_to_cpu(raw_sb->nx_xp_data_base); 174 | u32 data_index = le32_to_cpu(raw_sb->nx_xp_data_index); 175 | u32 data_blks = le32_to_cpu(raw_sb->nx_xp_data_blocks); 176 | u64 tmp; 177 | 178 | tmp = bno - data_base + data_blks - data_index; 179 | return do_div(tmp, data_blks); 180 | } 181 | 182 | /** 183 | * apfs_data_index_to_bno - Convert index in data area to block number 184 | * @sb: superblock structure 185 | * @index: index of the block in the current checkpoint's data area 186 | */ 187 | u64 apfs_data_index_to_bno(struct super_block *sb, u32 index) 188 | { 189 | struct apfs_nx_superblock *raw_sb = APFS_NXI(sb)->nx_raw; 190 | u64 data_base = le64_to_cpu(raw_sb->nx_xp_data_base); 191 | u32 data_index = le32_to_cpu(raw_sb->nx_xp_data_index); 192 | u32 data_blks = le32_to_cpu(raw_sb->nx_xp_data_blocks); 193 | 194 | return data_base + (index + data_index) % data_blks; 195 | } 196 | 197 | /** 198 | * apfs_ephemeral_object_lookup - Find an ephemeral object info in memory 199 | * @sb: superblock structure 200 | * @oid: ephemeral object id 201 | * 202 | * Returns a pointer to the object info on success, or an error pointer in case 203 | * of failure. 204 | */ 205 | struct apfs_ephemeral_object_info *apfs_ephemeral_object_lookup(struct super_block *sb, u64 oid) 206 | { 207 | struct apfs_nxsb_info *nxi = APFS_NXI(sb); 208 | struct apfs_ephemeral_object_info *list = NULL; 209 | int i; 210 | 211 | list = nxi->nx_eph_list; 212 | for (i = 0; i < nxi->nx_eph_count; ++i) { 213 | if (list[i].oid == oid) 214 | return &list[i]; 215 | } 216 | apfs_err(sb, "no mapping for oid 0x%llx", oid); 217 | return ERR_PTR(-EFSCORRUPTED); 218 | } 219 | 220 | /** 221 | * apfs_read_object_block - Map a non-ephemeral object block 222 | * @sb: superblock structure 223 | * @bno: block number for the object 224 | * @write: request write access? 225 | * @preserve: preserve the old block? 226 | * 227 | * On success returns the mapped buffer head for the object, which may now be 228 | * in a new location if write access was requested. Returns an error pointer 229 | * in case of failure. 230 | */ 231 | struct buffer_head *apfs_read_object_block(struct super_block *sb, u64 bno, bool write, bool preserve) 232 | { 233 | struct apfs_nxsb_info *nxi = APFS_NXI(sb); 234 | struct apfs_superblock *vsb_raw = NULL; 235 | struct buffer_head *bh, *new_bh; 236 | struct apfs_obj_phys *obj; 237 | u32 type; 238 | u64 new_bno; 239 | int err; 240 | 241 | ASSERT(write || !preserve); 242 | 243 | bh = apfs_sb_bread(sb, bno); 244 | if (!bh) { 245 | apfs_err(sb, "failed to read object block 0x%llx", bno); 246 | return ERR_PTR(-EIO); 247 | } 248 | 249 | obj = (struct apfs_obj_phys *)bh->b_data; 250 | type = le32_to_cpu(obj->o_type); 251 | ASSERT(!(type & APFS_OBJ_EPHEMERAL)); 252 | if (nxi->nx_flags & APFS_CHECK_NODES && !apfs_obj_verify_csum(sb, bh)) { 253 | apfs_err(sb, "bad checksum for object in block 0x%llx", bno); 254 | err = -EFSBADCRC; 255 | goto fail; 256 | } 257 | 258 | if (!write) 259 | return bh; 260 | ASSERT(!(sb->s_flags & SB_RDONLY)); 261 | 262 | /* Is the object already part of the current transaction? */ 263 | if (obj->o_xid == cpu_to_le64(nxi->nx_xid)) 264 | return bh; 265 | 266 | err = apfs_spaceman_allocate_block(sb, &new_bno, true /* backwards */); 267 | if (err) { 268 | apfs_err(sb, "block allocation failed"); 269 | goto fail; 270 | } 271 | new_bh = apfs_getblk(sb, new_bno); 272 | if (!new_bh) { 273 | apfs_err(sb, "failed to map block for CoW (0x%llx)", new_bno); 274 | err = -EIO; 275 | goto fail; 276 | } 277 | memcpy(new_bh->b_data, bh->b_data, sb->s_blocksize); 278 | 279 | /* 280 | * Don't free the old copy of the object if it's part of a snapshot. 281 | * Also increase the allocation count, except for the volume superblock 282 | * which is never counted there. 283 | */ 284 | if (!preserve) { 285 | err = apfs_free_queue_insert(sb, bh->b_blocknr, 1); 286 | if (err) 287 | apfs_err(sb, "free queue insertion failed for 0x%llx", (unsigned long long)bh->b_blocknr); 288 | } else if ((type & APFS_OBJECT_TYPE_MASK) != APFS_OBJECT_TYPE_FS) { 289 | vsb_raw = APFS_SB(sb)->s_vsb_raw; 290 | apfs_assert_in_transaction(sb, &vsb_raw->apfs_o); 291 | le64_add_cpu(&vsb_raw->apfs_fs_alloc_count, 1); 292 | le64_add_cpu(&vsb_raw->apfs_total_blocks_alloced, 1); 293 | } 294 | 295 | brelse(bh); 296 | bh = new_bh; 297 | new_bh = NULL; 298 | if (err) 299 | goto fail; 300 | obj = (struct apfs_obj_phys *)bh->b_data; 301 | 302 | if (type & APFS_OBJ_PHYSICAL) 303 | obj->o_oid = cpu_to_le64(new_bno); 304 | obj->o_xid = cpu_to_le64(nxi->nx_xid); 305 | err = apfs_transaction_join(sb, bh); 306 | if (err) 307 | goto fail; 308 | 309 | set_buffer_csum(bh); 310 | return bh; 311 | 312 | fail: 313 | brelse(bh); 314 | return ERR_PTR(err); 315 | } 316 | -------------------------------------------------------------------------------- /snapshot.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright (C) 2022 Ernesto A. Fernández 4 | */ 5 | 6 | #include 7 | #include 8 | #include "apfs.h" 9 | 10 | /** 11 | * apfs_create_superblock_snapshot - Take a snapshot of the volume superblock 12 | * @sb: superblock structure 13 | * @bno: on return, the block number for the new superblock copy 14 | * 15 | * Returns 0 on success or a negative error code in case of failure. 16 | */ 17 | static int apfs_create_superblock_snapshot(struct super_block *sb, u64 *bno) 18 | { 19 | struct apfs_sb_info *sbi = APFS_SB(sb); 20 | struct apfs_superblock *vsb_raw = sbi->s_vsb_raw; 21 | struct buffer_head *curr_bh = NULL; 22 | struct buffer_head *snap_bh = NULL; 23 | struct apfs_superblock *snap_raw = NULL; 24 | int err; 25 | 26 | err = apfs_spaceman_allocate_block(sb, bno, true /* backwards */); 27 | if (err) { 28 | apfs_err(sb, "block allocation failed"); 29 | goto fail; 30 | } 31 | 32 | snap_bh = apfs_getblk(sb, *bno); 33 | if (!snap_bh) { 34 | apfs_err(sb, "failed to map block for volume snap (0x%llx)", *bno); 35 | err = -EIO; 36 | goto fail; 37 | } 38 | apfs_assert_in_transaction(sb, &vsb_raw->apfs_o); 39 | 40 | curr_bh = sbi->s_vobject.o_bh; 41 | memcpy(snap_bh->b_data, curr_bh->b_data, sb->s_blocksize); 42 | curr_bh = NULL; 43 | 44 | err = apfs_transaction_join(sb, snap_bh); 45 | if (err) 46 | goto fail; 47 | set_buffer_csum(snap_bh); 48 | 49 | snap_raw = (struct apfs_superblock *)snap_bh->b_data; 50 | /* Volume superblocks in snapshots are physical objects */ 51 | snap_raw->apfs_o.o_oid = cpu_to_le64p(bno); 52 | snap_raw->apfs_o.o_type = cpu_to_le32(APFS_OBJ_PHYSICAL | APFS_OBJECT_TYPE_FS); 53 | /* The omap is shared with the current volume */ 54 | snap_raw->apfs_omap_oid = 0; 55 | /* The extent reference tree is given by the snapshot metadata */ 56 | snap_raw->apfs_extentref_tree_oid = 0; 57 | /* No snapshots inside snapshots */ 58 | snap_raw->apfs_snap_meta_tree_oid = 0; 59 | 60 | err = 0; 61 | fail: 62 | snap_raw = NULL; 63 | brelse(snap_bh); 64 | snap_bh = NULL; 65 | return err; 66 | } 67 | 68 | static int apfs_create_snap_metadata_rec(struct inode *mntpoint, struct apfs_node *snap_root, const char *name, int name_len, u64 sblock_oid) 69 | { 70 | struct super_block *sb = mntpoint->i_sb; 71 | struct apfs_sb_info *sbi = APFS_SB(sb); 72 | struct apfs_superblock *vsb_raw = sbi->s_vsb_raw; 73 | struct apfs_query *query = NULL; 74 | struct apfs_snap_metadata_key raw_key; 75 | struct apfs_snap_metadata_val *raw_val = NULL; 76 | int val_len; 77 | struct timespec64 now; 78 | u64 xid = APFS_NXI(sb)->nx_xid; 79 | int err; 80 | 81 | query = apfs_alloc_query(snap_root, NULL /* parent */); 82 | if (!query) { 83 | err = -ENOMEM; 84 | goto fail; 85 | } 86 | apfs_init_snap_metadata_key(xid, &query->key); 87 | query->flags |= APFS_QUERY_SNAP_META | APFS_QUERY_EXACT; 88 | 89 | err = apfs_btree_query(sb, &query); 90 | if (err == 0) { 91 | apfs_err(sb, "record exists for xid 0x%llx", xid); 92 | err = -EFSCORRUPTED; 93 | goto fail; 94 | } 95 | if (err != -ENODATA) { 96 | apfs_err(sb, "query failed for xid 0x%llx", xid); 97 | goto fail; 98 | } 99 | 100 | apfs_key_set_hdr(APFS_TYPE_SNAP_METADATA, xid, &raw_key); 101 | 102 | val_len = sizeof(*raw_val) + name_len + 1; 103 | raw_val = kzalloc(val_len, GFP_KERNEL); 104 | if (!raw_val) { 105 | err = -ENOMEM; 106 | goto fail; 107 | } 108 | raw_val->extentref_tree_oid = vsb_raw->apfs_extentref_tree_oid; 109 | raw_val->sblock_oid = cpu_to_le64(sblock_oid); 110 | now = current_time(mntpoint); 111 | raw_val->create_time = cpu_to_le64(timespec64_to_ns(&now)); 112 | raw_val->change_time = raw_val->create_time; 113 | raw_val->extentref_tree_type = vsb_raw->apfs_extentref_tree_type; 114 | raw_val->flags = 0; 115 | raw_val->name_len = cpu_to_le16(name_len + 1); /* Count the null byte */ 116 | strscpy(raw_val->name, name, name_len + 1); 117 | 118 | apfs_assert_in_transaction(sb, &vsb_raw->apfs_o); 119 | raw_val->inum = vsb_raw->apfs_next_obj_id; 120 | le64_add_cpu(&vsb_raw->apfs_next_obj_id, 1); 121 | 122 | err = apfs_btree_insert(query, &raw_key, sizeof(raw_key), raw_val, val_len); 123 | if (err) 124 | apfs_err(sb, "insertion failed for xid 0x%llx", xid); 125 | fail: 126 | kfree(raw_val); 127 | raw_val = NULL; 128 | apfs_free_query(query); 129 | query = NULL; 130 | return err; 131 | } 132 | 133 | static int apfs_create_snap_name_rec(struct apfs_node *snap_root, const char *name, int name_len) 134 | { 135 | struct super_block *sb = snap_root->object.sb; 136 | struct apfs_query *query = NULL; 137 | struct apfs_snap_name_key *raw_key = NULL; 138 | struct apfs_snap_name_val raw_val; 139 | int key_len; 140 | int err; 141 | 142 | query = apfs_alloc_query(snap_root, NULL /* parent */); 143 | if (!query) { 144 | err = -ENOMEM; 145 | goto fail; 146 | } 147 | apfs_init_snap_name_key(name, &query->key); 148 | query->flags |= APFS_QUERY_SNAP_META | APFS_QUERY_EXACT; 149 | 150 | err = apfs_btree_query(sb, &query); 151 | if (err == 0) { 152 | /* This should never happen here, the caller has checked */ 153 | apfs_alert(sb, "snapshot name record already exists"); 154 | err = -EFSCORRUPTED; 155 | goto fail; 156 | } 157 | if (err != -ENODATA) { 158 | apfs_err(sb, "query failed (%s)", name); 159 | goto fail; 160 | } 161 | 162 | key_len = sizeof(*raw_key) + name_len + 1; 163 | raw_key = kzalloc(key_len, GFP_KERNEL); 164 | if (!raw_key) { 165 | err = -ENOMEM; 166 | goto fail; 167 | } 168 | apfs_key_set_hdr(APFS_TYPE_SNAP_NAME, APFS_SNAP_NAME_OBJ_ID, raw_key); 169 | raw_key->name_len = cpu_to_le16(name_len + 1); /* Count the null byte */ 170 | strscpy(raw_key->name, name, name_len + 1); 171 | 172 | raw_val.snap_xid = cpu_to_le64(APFS_NXI(sb)->nx_xid); 173 | 174 | err = apfs_btree_insert(query, raw_key, key_len, &raw_val, sizeof(raw_val)); 175 | if (err) 176 | apfs_err(sb, "insertion failed (%s)", name); 177 | fail: 178 | kfree(raw_key); 179 | raw_key = NULL; 180 | apfs_free_query(query); 181 | query = NULL; 182 | return err; 183 | } 184 | 185 | static int apfs_create_snap_meta_records(struct inode *mntpoint, const char *name, int name_len, u64 sblock_oid) 186 | { 187 | struct super_block *sb = mntpoint->i_sb; 188 | struct apfs_superblock *vsb_raw = APFS_SB(sb)->s_vsb_raw; 189 | struct apfs_node *snap_root = NULL; 190 | int err; 191 | 192 | snap_root = apfs_read_node(sb, le64_to_cpu(vsb_raw->apfs_snap_meta_tree_oid), APFS_OBJ_PHYSICAL, true /* write */); 193 | if (IS_ERR(snap_root)) { 194 | apfs_err(sb, "failed to read snap meta root 0x%llx", le64_to_cpu(vsb_raw->apfs_snap_meta_tree_oid)); 195 | return PTR_ERR(snap_root); 196 | } 197 | apfs_assert_in_transaction(sb, &vsb_raw->apfs_o); 198 | vsb_raw->apfs_snap_meta_tree_oid = cpu_to_le64(snap_root->object.oid); 199 | 200 | err = apfs_create_snap_metadata_rec(mntpoint, snap_root, name, name_len, sblock_oid); 201 | if (err) { 202 | apfs_err(sb, "meta rec creation failed"); 203 | goto fail; 204 | } 205 | err = apfs_create_snap_name_rec(snap_root, name, name_len); 206 | if (err) 207 | apfs_err(sb, "name rec creation failed"); 208 | 209 | fail: 210 | apfs_node_free(snap_root); 211 | return err; 212 | } 213 | 214 | static int apfs_create_new_extentref_tree(struct super_block *sb) 215 | { 216 | struct apfs_superblock *vsb_raw = APFS_SB(sb)->s_vsb_raw; 217 | u64 oid; 218 | int err; 219 | 220 | err = apfs_make_empty_btree_root(sb, APFS_OBJECT_TYPE_BLOCKREFTREE, &oid); 221 | if (err) { 222 | apfs_err(sb, "failed to make empty root"); 223 | return err; 224 | } 225 | 226 | apfs_assert_in_transaction(sb, &vsb_raw->apfs_o); 227 | vsb_raw->apfs_extentref_tree_oid = cpu_to_le64(oid); 228 | return 0; 229 | } 230 | 231 | /** 232 | * apfs_update_omap_snap_tree - Add the current xid to the omap's snapshot tree 233 | * @sb: filesystem superblock 234 | * @oid_p: pointer to the on-disk block number for the root node 235 | * 236 | * Returns 0 on success, or a negative error code in case of failure. 237 | */ 238 | static int apfs_update_omap_snap_tree(struct super_block *sb, __le64 *oid_p) 239 | { 240 | struct apfs_nxsb_info *nxi = APFS_NXI(sb); 241 | struct apfs_node *root = NULL; 242 | u64 oid = le64_to_cpup(oid_p); 243 | struct apfs_query *query = NULL; 244 | __le64 raw_key; 245 | struct apfs_omap_snapshot raw_val = {0}; 246 | int err; 247 | 248 | /* An empty snapshots tree may not even have a root yet */ 249 | if (!oid) { 250 | err = apfs_make_empty_btree_root(sb, APFS_OBJECT_TYPE_OMAP_SNAPSHOT, &oid); 251 | if (err) { 252 | apfs_err(sb, "failed to make empty root"); 253 | return err; 254 | } 255 | } 256 | 257 | root = apfs_read_node(sb, oid, APFS_OBJ_PHYSICAL, true /* write */); 258 | if (IS_ERR(root)) { 259 | apfs_err(sb, "failed to read omap snap root 0x%llx", oid); 260 | return PTR_ERR(root); 261 | } 262 | oid = 0; 263 | 264 | query = apfs_alloc_query(root, NULL /* parent */); 265 | if (!query) { 266 | err = -ENOMEM; 267 | goto fail; 268 | } 269 | apfs_init_omap_snap_key(nxi->nx_xid, &query->key); 270 | query->flags = APFS_QUERY_OMAP_SNAP | APFS_QUERY_EXACT; 271 | 272 | err = apfs_btree_query(sb, &query); 273 | if (err == 0) { 274 | apfs_err(sb, "record exists for xid 0x%llx", nxi->nx_xid); 275 | err = -EFSCORRUPTED; 276 | goto fail; 277 | } 278 | if (err != -ENODATA) { 279 | apfs_err(sb, "query failed for xid 0x%llx", nxi->nx_xid); 280 | goto fail; 281 | } 282 | 283 | raw_key = cpu_to_le64(nxi->nx_xid); 284 | err = apfs_btree_insert(query, &raw_key, sizeof(raw_key), &raw_val, sizeof(raw_val)); 285 | if (err) 286 | apfs_err(sb, "insertion failed for xid 0x%llx", nxi->nx_xid); 287 | *oid_p = cpu_to_le64(root->object.block_nr); 288 | 289 | fail: 290 | apfs_free_query(query); 291 | query = NULL; 292 | apfs_node_free(root); 293 | root = NULL; 294 | return err; 295 | } 296 | 297 | /** 298 | * apfs_update_omap_snapshots - Add the current xid to the omap's snapshots 299 | * @sb: filesystem superblock 300 | * 301 | * Returns 0 on success, or a negative error code in case of failure. 302 | */ 303 | static int apfs_update_omap_snapshots(struct super_block *sb) 304 | { 305 | struct apfs_superblock *vsb_raw = APFS_SB(sb)->s_vsb_raw; 306 | struct buffer_head *bh = NULL; 307 | struct apfs_omap_phys *omap = NULL; 308 | u64 omap_blk; 309 | u64 xid; 310 | int err; 311 | 312 | xid = APFS_NXI(sb)->nx_xid; 313 | 314 | omap_blk = le64_to_cpu(vsb_raw->apfs_omap_oid); 315 | bh = apfs_read_object_block(sb, omap_blk, true /* write */, false /* preserve */); 316 | if (IS_ERR(bh)) { 317 | apfs_err(sb, "CoW failed for bno 0x%llx", omap_blk); 318 | return PTR_ERR(bh); 319 | } 320 | omap = (struct apfs_omap_phys *)bh->b_data; 321 | 322 | apfs_assert_in_transaction(sb, &omap->om_o); 323 | le32_add_cpu(&omap->om_snap_count, 1); 324 | omap->om_most_recent_snap = cpu_to_le64(xid); 325 | err = apfs_update_omap_snap_tree(sb, &omap->om_snapshot_tree_oid); 326 | if (err) 327 | apfs_err(sb, "omap snap tree update failed"); 328 | 329 | omap = NULL; 330 | brelse(bh); 331 | bh = NULL; 332 | return err; 333 | } 334 | 335 | /** 336 | * apfs_snapshot_name_check - Check if a snapshot with the given name exists 337 | * @sb: filesystem superblock 338 | * @name: name to check 339 | * @name_len: length of @name 340 | * @eexist: on return, whether the name exists or not 341 | * 342 | * Returns 0 on success, or a negative error code in case of failure. 343 | */ 344 | static int apfs_snapshot_name_check(struct super_block *sb, const char *name, int name_len, bool *eexist) 345 | { 346 | struct apfs_sb_info *sbi = APFS_SB(sb); 347 | struct apfs_superblock *vsb_raw = sbi->s_vsb_raw; 348 | struct apfs_node *snap_root = NULL; 349 | struct apfs_query *query = NULL; 350 | int err; 351 | 352 | snap_root = apfs_read_node(sb, le64_to_cpu(vsb_raw->apfs_snap_meta_tree_oid), APFS_OBJ_PHYSICAL, false /* write */); 353 | if (IS_ERR(snap_root)) { 354 | apfs_err(sb, "failed to read snap meta root 0x%llx", le64_to_cpu(vsb_raw->apfs_snap_meta_tree_oid)); 355 | return PTR_ERR(snap_root); 356 | } 357 | vsb_raw = NULL; 358 | 359 | query = apfs_alloc_query(snap_root, NULL /* parent */); 360 | if (!query) { 361 | err = -ENOMEM; 362 | goto out; 363 | } 364 | apfs_init_snap_name_key(name, &query->key); 365 | query->flags |= APFS_QUERY_SNAP_META | APFS_QUERY_EXACT; 366 | 367 | err = apfs_btree_query(sb, &query); 368 | if (err == 0) { 369 | *eexist = true; 370 | } else if (err == -ENODATA) { 371 | *eexist = false; 372 | err = 0; 373 | } else { 374 | apfs_err(sb, "query failed (%s)", name); 375 | } 376 | 377 | out: 378 | apfs_free_query(query); 379 | query = NULL; 380 | apfs_node_free(snap_root); 381 | snap_root = NULL; 382 | return err; 383 | } 384 | 385 | /** 386 | * apfs_do_ioc_takesnapshot - Actual work for apfs_ioc_take_snapshot() 387 | * @mntpoint: inode of the mount point to snapshot 388 | * @name: label for the snapshot 389 | * 390 | * Returns 0 on success, or a negative error code in case of failure. 391 | */ 392 | static int apfs_do_ioc_take_snapshot(struct inode *mntpoint, const char *name, int name_len) 393 | { 394 | struct super_block *sb = mntpoint->i_sb; 395 | struct apfs_sb_info *sbi = APFS_SB(sb); 396 | struct apfs_superblock *vsb_raw = NULL; 397 | struct apfs_omap *omap = sbi->s_omap; 398 | u64 sblock_oid; 399 | bool eexist = false; 400 | int err; 401 | 402 | err = apfs_transaction_start(sb, APFS_TRANS_REG); 403 | if (err) 404 | return err; 405 | 406 | /* 407 | * This check can fail in normal operation, so run it before making any 408 | * changes and exit the transaction cleanly if necessary. The name 409 | * lookup will have to be repeated later, but it's ok because I don't 410 | * expect snapshot creation to be a bottleneck for anyone. 411 | */ 412 | err = apfs_snapshot_name_check(sb, name, name_len, &eexist); 413 | if (err) { 414 | apfs_err(sb, "snapshot name existence check failed"); 415 | goto fail; 416 | } 417 | if (eexist) { 418 | err = apfs_transaction_commit(sb); 419 | if (err) 420 | goto fail; 421 | apfs_warn(sb, "snapshot name already in use (%s)", name); 422 | return -EEXIST; 423 | } 424 | 425 | /* 426 | * Flush the extent caches to the extenref tree before it gets moved to 427 | * the snapshot. It seems safer in general to avoid big unpredictable 428 | * changes to the layout after the snapshot is set up. 429 | */ 430 | err = apfs_transaction_flush_all_inodes(sb); 431 | if (err) { 432 | apfs_err(sb, "failed to flush all inodes"); 433 | goto fail; 434 | } 435 | 436 | err = apfs_create_superblock_snapshot(sb, &sblock_oid); 437 | if (err) { 438 | apfs_err(sb, "failed to snapshot superblock"); 439 | goto fail; 440 | } 441 | 442 | err = apfs_create_snap_meta_records(mntpoint, name, name_len, sblock_oid); 443 | if (err) { 444 | apfs_err(sb, "failed to create snap meta records"); 445 | goto fail; 446 | } 447 | 448 | err = apfs_create_new_extentref_tree(sb); 449 | if (err) { 450 | apfs_err(sb, "failed to create new extref tree"); 451 | goto fail; 452 | } 453 | 454 | err = apfs_update_omap_snapshots(sb); 455 | if (err) { 456 | apfs_err(sb, "failed to update omap snapshots"); 457 | goto fail; 458 | } 459 | 460 | /* 461 | * The official reference allows old implementations to ignore extended 462 | * snapshot metadata, so I don't see any reason why we can't do the 463 | * same for now. 464 | */ 465 | 466 | vsb_raw = sbi->s_vsb_raw; 467 | apfs_assert_in_transaction(sb, &vsb_raw->apfs_o); 468 | le64_add_cpu(&vsb_raw->apfs_num_snapshots, 1); 469 | 470 | omap->omap_latest_snap = APFS_NXI(sb)->nx_xid; 471 | 472 | sbi->s_nxi->nx_transaction.t_state |= APFS_NX_TRANS_FORCE_COMMIT; 473 | err = apfs_transaction_commit(sb); 474 | if (err) 475 | goto fail; 476 | return 0; 477 | 478 | fail: 479 | apfs_transaction_abort(sb); 480 | return err; 481 | } 482 | 483 | /** 484 | * apfs_ioc_take_snapshot - Ioctl handler for APFS_IOC_CREATE_SNAPSHOT 485 | * @file: affected file 486 | * @arg: ioctl argument 487 | * 488 | * Returns 0 on success, or a negative error code in case of failure. 489 | */ 490 | int apfs_ioc_take_snapshot(struct file *file, void __user *user_arg) 491 | { 492 | struct inode *inode = file_inode(file); 493 | struct super_block *sb = inode->i_sb; 494 | struct apfs_ioctl_snap_name *arg = NULL; 495 | size_t name_len; 496 | int err; 497 | 498 | if (apfs_ino(inode) != APFS_ROOT_DIR_INO_NUM) { 499 | apfs_info(sb, "snapshot must be requested on mountpoint"); 500 | return -ENOTTY; 501 | } 502 | 503 | #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 12, 0) 504 | if (!inode_owner_or_capable(inode)) 505 | #elif LINUX_VERSION_CODE < KERNEL_VERSION(6, 3, 0) && !RHEL_VERSION_GE(9, 6) 506 | if (!inode_owner_or_capable(&init_user_ns, inode)) 507 | #else 508 | if (!inode_owner_or_capable(&nop_mnt_idmap, inode)) 509 | #endif 510 | return -EPERM; 511 | 512 | err = mnt_want_write_file(file); 513 | if (err) 514 | return err; 515 | 516 | arg = kzalloc(sizeof(*arg), GFP_KERNEL); 517 | if (!arg) { 518 | err = -ENOMEM; 519 | goto fail; 520 | } 521 | 522 | if (copy_from_user(arg, user_arg, sizeof(*arg))) { 523 | err = -EFAULT; 524 | goto fail; 525 | } 526 | 527 | name_len = strnlen(arg->name, sizeof(arg->name)); 528 | if (name_len == sizeof(arg->name)) { 529 | apfs_warn(sb, "snapshot name is too long (%d)", (int)name_len); 530 | err = -EINVAL; 531 | goto fail; 532 | } 533 | 534 | err = apfs_do_ioc_take_snapshot(inode, arg->name, name_len); 535 | fail: 536 | kfree(arg); 537 | arg = NULL; 538 | mnt_drop_write_file(file); 539 | return err; 540 | } 541 | 542 | static int apfs_snap_xid_from_query(struct apfs_query *query, u64 *xid) 543 | { 544 | char *raw = query->node->object.data; 545 | __le64 *val = NULL; 546 | 547 | if (query->len != sizeof(*val)) { 548 | apfs_err(query->node->object.sb, "bad value length (%d)", query->len); 549 | return -EFSCORRUPTED; 550 | } 551 | val = (__le64 *)(raw + query->off); 552 | 553 | *xid = le64_to_cpup(val); 554 | return 0; 555 | } 556 | 557 | static int apfs_snapshot_name_to_xid(struct apfs_node *snap_root, const char *name, u64 *xid) 558 | { 559 | struct super_block *sb = snap_root->object.sb; 560 | struct apfs_query *query = NULL; 561 | int err; 562 | 563 | query = apfs_alloc_query(snap_root, NULL /* parent */); 564 | if (!query) 565 | return -ENOMEM; 566 | apfs_init_snap_name_key(name, &query->key); 567 | query->flags |= APFS_QUERY_SNAP_META | APFS_QUERY_EXACT; 568 | 569 | err = apfs_btree_query(sb, &query); 570 | if (err) { 571 | if (err != -ENODATA) 572 | apfs_err(sb, "query failed (%s)", name); 573 | goto fail; 574 | } 575 | 576 | err = apfs_snap_xid_from_query(query, xid); 577 | if (err) 578 | apfs_err(sb, "bad snap name record (%s)", name); 579 | fail: 580 | apfs_free_query(query); 581 | query = NULL; 582 | return err; 583 | } 584 | 585 | static int apfs_snap_sblock_from_query(struct apfs_query *query, u64 *sblock_oid) 586 | { 587 | struct super_block *sb = query->node->object.sb; 588 | char *raw = query->node->object.data; 589 | struct apfs_snap_metadata_val *val = NULL; 590 | 591 | if (query->len < sizeof(*val)) { 592 | apfs_err(sb, "bad value length (%d)", query->len); 593 | return -EFSCORRUPTED; 594 | } 595 | val = (struct apfs_snap_metadata_val *)(raw + query->off); 596 | 597 | /* I don't know anything about these snapshots, can they be mounted? */ 598 | if (le32_to_cpu(val->flags) & APFS_SNAP_META_PENDING_DATALESS) { 599 | apfs_warn(sb, "the requested snapshot is dataless"); 600 | return -EOPNOTSUPP; 601 | } 602 | 603 | *sblock_oid = le64_to_cpu(val->sblock_oid); 604 | return 0; 605 | } 606 | 607 | static int apfs_snapshot_xid_to_sblock(struct apfs_node *snap_root, u64 xid, u64 *sblock_oid) 608 | { 609 | struct super_block *sb = snap_root->object.sb; 610 | struct apfs_query *query = NULL; 611 | int err; 612 | 613 | query = apfs_alloc_query(snap_root, NULL /* parent */); 614 | if (!query) 615 | return -ENOMEM; 616 | apfs_init_snap_metadata_key(xid, &query->key); 617 | query->flags |= APFS_QUERY_SNAP_META | APFS_QUERY_EXACT; 618 | 619 | err = apfs_btree_query(sb, &query); 620 | if (err) { 621 | apfs_err(sb, "query failed for xid 0x%llx", xid); 622 | goto fail; 623 | } 624 | 625 | err = apfs_snap_sblock_from_query(query, sblock_oid); 626 | if (err) 627 | apfs_err(sb, "bad snap meta record for xid 0x%llx", xid); 628 | fail: 629 | apfs_free_query(query); 630 | query = NULL; 631 | return err; 632 | } 633 | 634 | /** 635 | * apfs_switch_to_snapshot - Start working with the snapshot volume superblock 636 | * @sb: superblock structure 637 | * 638 | * Maps the volume superblock from the snapshot specified in the mount options. 639 | * Returns 0 on success or a negative error code in case of failure. 640 | */ 641 | int apfs_switch_to_snapshot(struct super_block *sb) 642 | { 643 | struct apfs_sb_info *sbi = APFS_SB(sb); 644 | struct apfs_nxsb_info *nxi = APFS_NXI(sb); 645 | struct apfs_superblock *vsb_raw = sbi->s_vsb_raw; 646 | struct apfs_node *snap_root = NULL; 647 | const char *name = NULL; 648 | u64 sblock_oid = 0; 649 | u64 xid = 0; 650 | int err; 651 | 652 | ASSERT(sb->s_flags & SB_RDONLY); 653 | 654 | name = sbi->s_snap_name; 655 | if (strlen(name) > APFS_SNAP_MAX_NAMELEN) { 656 | apfs_warn(sb, "snapshot name is too long"); 657 | return -EINVAL; 658 | } 659 | 660 | snap_root = apfs_read_node(sb, le64_to_cpu(vsb_raw->apfs_snap_meta_tree_oid), APFS_OBJ_PHYSICAL, false /* write */); 661 | if (IS_ERR(snap_root)) { 662 | apfs_err(sb, "failed to read snap meta root 0x%llx", le64_to_cpu(vsb_raw->apfs_snap_meta_tree_oid)); 663 | return PTR_ERR(snap_root); 664 | } 665 | vsb_raw = NULL; 666 | 667 | err = apfs_snapshot_name_to_xid(snap_root, name, &xid); 668 | if (err) { 669 | if (err == -ENODATA) 670 | apfs_info(sb, "no snapshot under that name (%s)", name); 671 | goto fail; 672 | } 673 | sbi->s_snap_xid = xid; 674 | 675 | err = apfs_snapshot_xid_to_sblock(snap_root, xid, &sblock_oid); 676 | if (err) 677 | goto fail; 678 | 679 | apfs_unmap_volume_super(sb); 680 | err = apfs_map_volume_super_bno(sb, sblock_oid, nxi->nx_flags & APFS_CHECK_NODES); 681 | if (err) 682 | apfs_err(sb, "failed to map volume block 0x%llx", sblock_oid); 683 | 684 | fail: 685 | apfs_node_free(snap_root); 686 | return err; 687 | } 688 | -------------------------------------------------------------------------------- /symlink.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright (C) 2018 Ernesto A. Fernández 4 | */ 5 | 6 | #include 7 | #include 8 | #include "apfs.h" 9 | 10 | /** 11 | * apfs_get_link - Follow a symbolic link 12 | * @dentry: dentry for the link 13 | * @inode: inode for the link 14 | * @done: delayed call to free the returned buffer after use 15 | * 16 | * Returns a pointer to a buffer containing the target path, or an appropriate 17 | * error pointer in case of failure. 18 | */ 19 | static const char *apfs_get_link(struct dentry *dentry, struct inode *inode, 20 | struct delayed_call *done) 21 | { 22 | struct super_block *sb = inode->i_sb; 23 | struct apfs_nxsb_info *nxi = APFS_NXI(sb); 24 | char *target = NULL; 25 | int err; 26 | int size; 27 | 28 | down_read(&nxi->nx_big_sem); 29 | 30 | if (!dentry) { 31 | err = -ECHILD; 32 | goto fail; 33 | } 34 | 35 | size = __apfs_xattr_get(inode, APFS_XATTR_NAME_SYMLINK, 36 | NULL /* buffer */, 0 /* size */); 37 | if (size < 0) { /* TODO: return a better error code */ 38 | apfs_err(sb, "symlink size read failed"); 39 | err = size; 40 | goto fail; 41 | } 42 | 43 | target = kmalloc(size, GFP_KERNEL); 44 | if (!target) { 45 | err = -ENOMEM; 46 | goto fail; 47 | } 48 | 49 | size = __apfs_xattr_get(inode, APFS_XATTR_NAME_SYMLINK, target, size); 50 | if (size < 0) { 51 | apfs_err(sb, "symlink read failed"); 52 | err = size; 53 | goto fail; 54 | } 55 | if (size == 0 || *(target + size - 1) != 0) { 56 | /* Target path must be NULL-terminated */ 57 | apfs_err(sb, "bad link target in inode 0x%llx", apfs_ino(inode)); 58 | err = -EFSCORRUPTED; 59 | goto fail; 60 | } 61 | 62 | up_read(&nxi->nx_big_sem); 63 | set_delayed_call(done, kfree_link, target); 64 | return target; 65 | 66 | fail: 67 | kfree(target); 68 | up_read(&nxi->nx_big_sem); 69 | return ERR_PTR(err); 70 | } 71 | 72 | const struct inode_operations apfs_symlink_inode_operations = { 73 | .get_link = apfs_get_link, 74 | .getattr = apfs_getattr, 75 | .listxattr = apfs_listxattr, 76 | .update_time = apfs_update_time, 77 | #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) /* Now this is the default */ 78 | .readlink = generic_readlink, 79 | #endif 80 | }; 81 | -------------------------------------------------------------------------------- /unicode.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * Copyright (C) 2018 Ernesto A. Fernández 4 | */ 5 | 6 | #ifndef _APFS_UNICODE_H 7 | #define _APFS_UNICODE_H 8 | 9 | #include 10 | 11 | /* 12 | * This structure helps apfs_normalize_next() to retrieve one normalized 13 | * (and case-folded) UTF-32 character at a time from a UTF-8 string. 14 | */ 15 | struct apfs_unicursor { 16 | const char *utf8curr; /* Start of UTF-8 to decompose and reorder */ 17 | unsigned int total_len; /* Length of the whole UTF-8 string */ 18 | int length; /* Length of normalization until next starter */ 19 | int last_pos; /* Offset in substring of last char returned */ 20 | u8 last_ccc; /* CCC of the last character returned */ 21 | }; 22 | 23 | extern void apfs_init_unicursor(struct apfs_unicursor *cursor, const char *utf8str, unsigned int total_len); 24 | extern unicode_t apfs_normalize_next(struct apfs_unicursor *cursor, 25 | bool case_fold); 26 | 27 | #endif /* _APFS_UNICODE_H */ 28 | -------------------------------------------------------------------------------- /xfield.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0-only 2 | /* 3 | * Copyright (C) 2019 Ernesto A. Fernández 4 | */ 5 | 6 | #include 7 | #include 8 | #include "apfs.h" 9 | 10 | /** 11 | * apfs_find_xfield - Find an extended field value in an inode or dentry record 12 | * @xfields: pointer to the on-disk xfield collection for the record 13 | * @len: length of the collection 14 | * @xtype: type of the xfield to retrieve 15 | * @xval: on return, a pointer to the wanted on-disk xfield value 16 | * 17 | * Returns the length of @xval on success, or 0 if no matching xfield was found; 18 | * the caller must check that the expected structures fit before casting @xval. 19 | */ 20 | int apfs_find_xfield(u8 *xfields, int len, u8 xtype, char **xval) 21 | { 22 | struct apfs_xf_blob *blob; 23 | struct apfs_x_field *xfield; 24 | int count; 25 | int rest = len; 26 | int i; 27 | 28 | if (!len) 29 | return 0; /* No xfield data */ 30 | 31 | rest -= sizeof(*blob); 32 | if (rest < 0) 33 | return 0; /* Corruption */ 34 | blob = (struct apfs_xf_blob *)xfields; 35 | 36 | count = le16_to_cpu(blob->xf_num_exts); 37 | rest -= count * sizeof(*xfield); 38 | if (rest < 0) 39 | return 0; /* Corruption */ 40 | xfield = (struct apfs_x_field *)blob->xf_data; 41 | 42 | for (i = 0; i < count; ++i) { 43 | int xlen; 44 | 45 | /* Attribute length is padded to a multiple of 8 */ 46 | xlen = round_up(le16_to_cpu(xfield[i].x_size), 8); 47 | if (xlen > rest) 48 | return 0; /* Corruption */ 49 | 50 | if (xfield[i].x_type == xtype) { 51 | *xval = (char *)xfields + len - rest; 52 | return xlen; 53 | } 54 | rest -= xlen; 55 | } 56 | return 0; 57 | } 58 | 59 | /** 60 | * apfs_init_xfields - Set an empty collection of xfields in a buffer 61 | * @buffer: buffer to hold the xfields 62 | * @buflen: length of the buffer; should be enough to fit an xfield blob 63 | * 64 | * Returns 0 on success, or -1 if the buffer isn't long enough. 65 | */ 66 | int apfs_init_xfields(u8 *buffer, int buflen) 67 | { 68 | struct apfs_xf_blob *blob; 69 | 70 | if (buflen < sizeof(*blob)) 71 | return -1; 72 | blob = (struct apfs_xf_blob *)buffer; 73 | 74 | blob->xf_num_exts = 0; 75 | blob->xf_used_data = 0; 76 | return 0; 77 | } 78 | 79 | /** 80 | * apfs_insert_xfield - Add a new xfield to an in-memory collection 81 | * @buffer: buffer holding the collection of xfields 82 | * @buflen: length of the buffer; should be enough to fit the new xfield 83 | * @xkey: metadata for the new xfield 84 | * @xval: value for the new xfield 85 | * 86 | * Returns the new length of the collection, or 0 if it the allocation would 87 | * overflow @buffer. 88 | */ 89 | int apfs_insert_xfield(u8 *buffer, int buflen, const struct apfs_x_field *xkey, 90 | const void *xval) 91 | { 92 | struct apfs_xf_blob *blob; 93 | struct apfs_x_field *curr_xkey; 94 | void *curr_xval; 95 | int count; 96 | int rest = buflen; 97 | u16 used_data; 98 | int xlen, padded_xlen; 99 | int meta_len, total_len; 100 | int i; 101 | 102 | xlen = le16_to_cpu(xkey->x_size); 103 | padded_xlen = round_up(xlen, 8); 104 | 105 | if (!buflen) 106 | return 0; 107 | 108 | rest -= sizeof(*blob); 109 | if (rest < 0) 110 | return 0; 111 | blob = (struct apfs_xf_blob *)buffer; 112 | used_data = le16_to_cpu(blob->xf_used_data); 113 | 114 | count = le16_to_cpu(blob->xf_num_exts); 115 | rest -= count * sizeof(*curr_xkey); 116 | if (rest < 0) 117 | return 0; 118 | meta_len = buflen - rest; 119 | curr_xkey = (struct apfs_x_field *)blob->xf_data; 120 | 121 | for (i = 0; i < count; ++i, ++curr_xkey) { 122 | int curr_xlen; 123 | 124 | /* Attribute length is padded to a multiple of 8 */ 125 | curr_xlen = round_up(le16_to_cpu(curr_xkey->x_size), 8); 126 | if (curr_xlen > rest) 127 | return 0; 128 | if (curr_xkey->x_type != xkey->x_type) { 129 | rest -= curr_xlen; 130 | continue; 131 | } 132 | 133 | /* The xfield already exists, so just resize it and set it */ 134 | memcpy(curr_xkey, xkey, sizeof(*curr_xkey)); 135 | if (padded_xlen > rest) 136 | return 0; 137 | curr_xval = buffer + buflen - rest; 138 | rest -= max(padded_xlen, curr_xlen); 139 | memmove(curr_xval + padded_xlen, curr_xval + curr_xlen, rest); 140 | memcpy(curr_xval, xval, xlen); 141 | memset(curr_xval + xlen, 0, padded_xlen - xlen); 142 | used_data += padded_xlen - curr_xlen; 143 | 144 | goto done; 145 | } 146 | 147 | /* Create a metadata entry for the new xfield */ 148 | rest -= sizeof(*curr_xkey); 149 | if (rest < 0) 150 | return 0; 151 | meta_len += sizeof(*curr_xkey); 152 | memmove(curr_xkey + 1, curr_xkey, buflen - meta_len); 153 | memcpy(curr_xkey, xkey, sizeof(*curr_xkey)); 154 | ++count; 155 | 156 | /* Now set the xfield value */ 157 | if (padded_xlen > rest) 158 | return 0; 159 | curr_xval = buffer + buflen - rest; 160 | memcpy(curr_xval, xval, xlen); 161 | memset(curr_xval + xlen, 0, padded_xlen - xlen); 162 | used_data += padded_xlen; 163 | 164 | done: 165 | total_len = used_data + meta_len; 166 | if (total_len > buflen) 167 | return 0; 168 | blob->xf_num_exts = cpu_to_le16(count); 169 | blob->xf_used_data = cpu_to_le16(used_data); 170 | return total_len; 171 | } 172 | --------------------------------------------------------------------------------