├── .gitignore ├── Cargo.toml ├── LICENSE.txt ├── NOTES.mkdn ├── examples ├── kvtool.rs └── llsh.rs ├── proposal.mkdn ├── src ├── lib.rs └── low_level.rs └── testimg /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "lethe" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | byteorder = {version = "1.4.3", default-features = false} 10 | crc = { version = "3.0.0", default-features = false } 11 | fnv = {version = "1.0.7", default-features = false} 12 | num-derive = "0.3.3" 13 | num-traits = {version = "0.2.15", default-features = false} 14 | zerocopy = {version = "0.6.1", default-features = false} 15 | 16 | [dev-dependencies] 17 | pretty-hex = "0.3.0" 18 | clap = {version = "3", features = ["derive"]} 19 | rustyline = "10.0.0" 20 | anyhow = "1.0.65" 21 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Mozilla Public License Version 2.0 2 | ================================== 3 | 4 | 1. Definitions 5 | -------------- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | -------------------------------- 88 | 89 | 2.1. Grants 90 | 91 | Each Contributor hereby grants You a world-wide, royalty-free, 92 | non-exclusive license: 93 | 94 | (a) under intellectual property rights (other than patent or trademark) 95 | Licensable by such Contributor to use, reproduce, make available, 96 | modify, display, perform, distribute, and otherwise exploit its 97 | Contributions, either on an unmodified basis, with Modifications, or 98 | as part of a Larger Work; and 99 | 100 | (b) under Patent Claims of such Contributor to make, use, sell, offer 101 | for sale, have made, import, and otherwise transfer either its 102 | Contributions or its Contributor Version. 103 | 104 | 2.2. Effective Date 105 | 106 | The licenses granted in Section 2.1 with respect to any Contribution 107 | become effective for each Contribution on the date the Contributor first 108 | distributes such Contribution. 109 | 110 | 2.3. Limitations on Grant Scope 111 | 112 | The licenses granted in this Section 2 are the only rights granted under 113 | this License. No additional rights or licenses will be implied from the 114 | distribution or licensing of Covered Software under this License. 115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 116 | Contributor: 117 | 118 | (a) for any code that a Contributor has removed from Covered Software; 119 | or 120 | 121 | (b) for infringements caused by: (i) Your and any other third party's 122 | modifications of Covered Software, or (ii) the combination of its 123 | Contributions with other software (except as part of its Contributor 124 | Version); or 125 | 126 | (c) under Patent Claims infringed by Covered Software in the absence of 127 | its Contributions. 128 | 129 | This License does not grant any rights in the trademarks, service marks, 130 | or logos of any Contributor (except as may be necessary to comply with 131 | the notice requirements in Section 3.4). 132 | 133 | 2.4. Subsequent Licenses 134 | 135 | No Contributor makes additional grants as a result of Your choice to 136 | distribute the Covered Software under a subsequent version of this 137 | License (see Section 10.2) or under the terms of a Secondary License (if 138 | permitted under the terms of Section 3.3). 139 | 140 | 2.5. Representation 141 | 142 | Each Contributor represents that the Contributor believes its 143 | Contributions are its original creation(s) or it has sufficient rights 144 | to grant the rights to its Contributions conveyed by this License. 145 | 146 | 2.6. Fair Use 147 | 148 | This License is not intended to limit any rights You have under 149 | applicable copyright doctrines of fair use, fair dealing, or other 150 | equivalents. 151 | 152 | 2.7. Conditions 153 | 154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 155 | in Section 2.1. 156 | 157 | 3. Responsibilities 158 | ------------------- 159 | 160 | 3.1. Distribution of Source Form 161 | 162 | All distribution of Covered Software in Source Code Form, including any 163 | Modifications that You create or to which You contribute, must be under 164 | the terms of this License. You must inform recipients that the Source 165 | Code Form of the Covered Software is governed by the terms of this 166 | License, and how they can obtain a copy of this License. You may not 167 | attempt to alter or restrict the recipients' rights in the Source Code 168 | Form. 169 | 170 | 3.2. Distribution of Executable Form 171 | 172 | If You distribute Covered Software in Executable Form then: 173 | 174 | (a) such Covered Software must also be made available in Source Code 175 | Form, as described in Section 3.1, and You must inform recipients of 176 | the Executable Form how they can obtain a copy of such Source Code 177 | Form by reasonable means in a timely manner, at a charge no more 178 | than the cost of distribution to the recipient; and 179 | 180 | (b) You may distribute such Executable Form under the terms of this 181 | License, or sublicense it under different terms, provided that the 182 | license for the Executable Form does not attempt to limit or alter 183 | the recipients' rights in the Source Code Form under this License. 184 | 185 | 3.3. Distribution of a Larger Work 186 | 187 | You may create and distribute a Larger Work under terms of Your choice, 188 | provided that You also comply with the requirements of this License for 189 | the Covered Software. If the Larger Work is a combination of Covered 190 | Software with a work governed by one or more Secondary Licenses, and the 191 | Covered Software is not Incompatible With Secondary Licenses, this 192 | License permits You to additionally distribute such Covered Software 193 | under the terms of such Secondary License(s), so that the recipient of 194 | the Larger Work may, at their option, further distribute the Covered 195 | Software under the terms of either this License or such Secondary 196 | License(s). 197 | 198 | 3.4. Notices 199 | 200 | You may not remove or alter the substance of any license notices 201 | (including copyright notices, patent notices, disclaimers of warranty, 202 | or limitations of liability) contained within the Source Code Form of 203 | the Covered Software, except that You may alter any license notices to 204 | the extent required to remedy known factual inaccuracies. 205 | 206 | 3.5. Application of Additional Terms 207 | 208 | You may choose to offer, and to charge a fee for, warranty, support, 209 | indemnity or liability obligations to one or more recipients of Covered 210 | Software. However, You may do so only on Your own behalf, and not on 211 | behalf of any Contributor. You must make it absolutely clear that any 212 | such warranty, support, indemnity, or liability obligation is offered by 213 | You alone, and You hereby agree to indemnify every Contributor for any 214 | liability incurred by such Contributor as a result of warranty, support, 215 | indemnity or liability terms You offer. You may include additional 216 | disclaimers of warranty and limitations of liability specific to any 217 | jurisdiction. 218 | 219 | 4. Inability to Comply Due to Statute or Regulation 220 | --------------------------------------------------- 221 | 222 | If it is impossible for You to comply with any of the terms of this 223 | License with respect to some or all of the Covered Software due to 224 | statute, judicial order, or regulation then You must: (a) comply with 225 | the terms of this License to the maximum extent possible; and (b) 226 | describe the limitations and the code they affect. Such description must 227 | be placed in a text file included with all distributions of the Covered 228 | Software under this License. Except to the extent prohibited by statute 229 | or regulation, such description must be sufficiently detailed for a 230 | recipient of ordinary skill to be able to understand it. 231 | 232 | 5. Termination 233 | -------------- 234 | 235 | 5.1. The rights granted under this License will terminate automatically 236 | if You fail to comply with any of its terms. However, if You become 237 | compliant, then the rights granted under this License from a particular 238 | Contributor are reinstated (a) provisionally, unless and until such 239 | Contributor explicitly and finally terminates Your grants, and (b) on an 240 | ongoing basis, if such Contributor fails to notify You of the 241 | non-compliance by some reasonable means prior to 60 days after You have 242 | come back into compliance. Moreover, Your grants from a particular 243 | Contributor are reinstated on an ongoing basis if such Contributor 244 | notifies You of the non-compliance by some reasonable means, this is the 245 | first time You have received notice of non-compliance with this License 246 | from such Contributor, and You become compliant prior to 30 days after 247 | Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, 251 | counter-claims, and cross-claims) alleging that a Contributor Version 252 | directly or indirectly infringes any patent, then the rights granted to 253 | You by any and all Contributors for the Covered Software under Section 254 | 2.1 of this License shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 257 | end user license agreements (excluding distributors and resellers) which 258 | have been validly granted by You or Your distributors under this License 259 | prior to termination shall survive termination. 260 | 261 | ************************************************************************ 262 | * * 263 | * 6. Disclaimer of Warranty * 264 | * ------------------------- * 265 | * * 266 | * Covered Software is provided under this License on an "as is" * 267 | * basis, without warranty of any kind, either expressed, implied, or * 268 | * statutory, including, without limitation, warranties that the * 269 | * Covered Software is free of defects, merchantable, fit for a * 270 | * particular purpose or non-infringing. The entire risk as to the * 271 | * quality and performance of the Covered Software is with You. * 272 | * Should any Covered Software prove defective in any respect, You * 273 | * (not any Contributor) assume the cost of any necessary servicing, * 274 | * repair, or correction. This disclaimer of warranty constitutes an * 275 | * essential part of this License. No use of any Covered Software is * 276 | * authorized under this License except under this disclaimer. * 277 | * * 278 | ************************************************************************ 279 | 280 | ************************************************************************ 281 | * * 282 | * 7. Limitation of Liability * 283 | * -------------------------- * 284 | * * 285 | * Under no circumstances and under no legal theory, whether tort * 286 | * (including negligence), contract, or otherwise, shall any * 287 | * Contributor, or anyone who distributes Covered Software as * 288 | * permitted above, be liable to You for any direct, indirect, * 289 | * special, incidental, or consequential damages of any character * 290 | * including, without limitation, damages for lost profits, loss of * 291 | * goodwill, work stoppage, computer failure or malfunction, or any * 292 | * and all other commercial damages or losses, even if such party * 293 | * shall have been informed of the possibility of such damages. This * 294 | * limitation of liability shall not apply to liability for death or * 295 | * personal injury resulting from such party's negligence to the * 296 | * extent applicable law prohibits such limitation. Some * 297 | * jurisdictions do not allow the exclusion or limitation of * 298 | * incidental or consequential damages, so this exclusion and * 299 | * limitation may not apply to You. * 300 | * * 301 | ************************************************************************ 302 | 303 | 8. Litigation 304 | ------------- 305 | 306 | Any litigation relating to this License may be brought only in the 307 | courts of a jurisdiction where the defendant maintains its principal 308 | place of business and such litigation shall be governed by laws of that 309 | jurisdiction, without reference to its conflict-of-law provisions. 310 | Nothing in this Section shall prevent a party's ability to bring 311 | cross-claims or counter-claims. 312 | 313 | 9. Miscellaneous 314 | ---------------- 315 | 316 | This License represents the complete agreement concerning the subject 317 | matter hereof. If any provision of this License is held to be 318 | unenforceable, such provision shall be reformed only to the extent 319 | necessary to make it enforceable. Any law or regulation which provides 320 | that the language of a contract shall be construed against the drafter 321 | shall not be used to construe this License against a Contributor. 322 | 323 | 10. Versions of the License 324 | --------------------------- 325 | 326 | 10.1. New Versions 327 | 328 | Mozilla Foundation is the license steward. Except as provided in Section 329 | 10.3, no one other than the license steward has the right to modify or 330 | publish new versions of this License. Each version will be given a 331 | distinguishing version number. 332 | 333 | 10.2. Effect of New Versions 334 | 335 | You may distribute the Covered Software under the terms of the version 336 | of the License under which You originally received the Covered Software, 337 | or under the terms of any subsequent version published by the license 338 | steward. 339 | 340 | 10.3. Modified Versions 341 | 342 | If you create software not governed by this License, and you want to 343 | create a new license for such software, you may create and use a 344 | modified version of this License if you rename the license and remove 345 | any references to the name of the license steward (except to note that 346 | such modified license differs from this License). 347 | 348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 349 | Licenses 350 | 351 | If You choose to distribute Source Code Form that is Incompatible With 352 | Secondary Licenses under the terms of this version of the License, the 353 | notice described in Exhibit B of this License must be attached. 354 | 355 | Exhibit A - Source Code Form License Notice 356 | ------------------------------------------- 357 | 358 | This Source Code Form is subject to the terms of the Mozilla Public 359 | License, v. 2.0. If a copy of the MPL was not distributed with this 360 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 361 | 362 | If it is not possible or desirable to put the notice in a particular 363 | file, then You may include the notice in a location (such as a LICENSE 364 | file in a relevant directory) where a recipient would be likely to look 365 | for such a notice. 366 | 367 | You may add additional accurate notices of copyright ownership. 368 | 369 | Exhibit B - "Incompatible With Secondary Licenses" Notice 370 | --------------------------------------------------------- 371 | 372 | This Source Code Form is "Incompatible With Secondary Licenses", as 373 | defined by the Mozilla Public License, v. 2.0. 374 | -------------------------------------------------------------------------------- /NOTES.mkdn: -------------------------------------------------------------------------------- 1 | Unusual requirements: 2 | 3 | - The LPC55 has one sector size for both program and erase. It does not appear 4 | to be willing to program a smaller unit than 512 B. On the up side, the erase 5 | sectors are 512 B. 6 | 7 | - The STM32H7 has giant (128 kiB) erase sectors and tiny (32 B) write sectors. 8 | 9 | 10 | --- 11 | 12 | The on-disk representation of the filesystem is a _log,_ containing things 13 | called _entries._ Stored objects are one kind of entry, called a _KV entry_ (for 14 | key-value). 15 | 16 | Each entry consists of a stream of bytes, padded to a whole number of write 17 | sectors. The stream of bytes consists of: 18 | 19 | - A generic entry header. 20 | - A type-specific subheader. 21 | - Any data contents. 22 | - Padding. 23 | - A type-specific subtrailer. 24 | - A generic entry trailer. 25 | 26 | Padding is between the contents and the trailers for reasons that will become 27 | apparent shortly. 28 | 29 | To write an entry, begin programming at the first sector moving toward the end. 30 | 31 | An entry is complete when its trailer sector finishes programming. 32 | 33 | An incomplete entry may occur if we lose power or crash before we're done 34 | programming the entry. You can detect an incomplete entry by comparing the 35 | header (which is written first) with the trailer (which is written last); if 36 | they don't match, the entry is incomplete. Because unwritten areas are expected 37 | to be detectable as blank (either because they fail ECC, or because they read as 38 | a predictable bit pattern), a blank trailer is also a clear indication of an 39 | incomplete entry. (There's also a special type code used to mark incomplete 40 | entries, below.) 41 | 42 | A incomplete entry _must still be treated as being in the log_ for space 43 | allocation purposes: detecting it as incomplete _requires_ that its trailing 44 | sector stay unused. However, it will not be treated as valid for lookup or 45 | garbage collection. 46 | 47 | 48 | The log has a _watermark_ indicating the break between valid/used data and 49 | invalid/free data. To recover this watermark on restart, 50 | 51 | - Begin reading from the log at the beginning. 52 | - Skip each entry that is found. 53 | - If an entry is incomplete, the watermark immediately follows its unwritten 54 | trailer sector. 55 | - If the sector after an entry (which would normally be the header of the 56 | following entry) is blank, set the watermark there. 57 | - If you run into the end of the allocated space, then there is no free space 58 | and the watermark is at the end of the space. 59 | 60 | On restart, if an incomplete entry is detected, we _program its trailer_ to 61 | indicate incompleteness and enable it to be skipped when reading the log in 62 | reverse. A special trailer type code is reserved to indicate an incomplete 63 | entry. The header is not modified, because of medium restrictions. 64 | 65 | 66 | The generic header contains: 67 | 68 | - A type indicator, giving the nature of the subheader 69 | - The all-1s and all-0s type indicators are both reserved. 70 | - Offset, in sectors, to sector containing the trailer. This allows an entry to 71 | be checked and skipped in a generic fashion without parsing subheaders. 72 | - A checksum, to make it less likely that arbitrary data is confused for a 73 | header. 74 | 75 | The generic trailer contains the same information. Trailers use different type 76 | codes from headers so they can't be confused for one another. 77 | 78 | 79 | The KV entry subheader contains: 80 | 81 | - Length of the key, in bytes 82 | - A cheaply computed hash of the key. This is not for integrity protection (we 83 | assume ECC on the underlying medium) but for fast lookup: if the length and 84 | hash don't match, there's no need to read the key. 85 | 86 | The subtrailer is identical. 87 | 88 | 89 | To find the latest KV entry for a key: 90 | 91 | - Start at the watermark (end of the log). 92 | - Repeat until you run out of log: 93 | - Read the sector that should contain the trailer. This read can be 94 | performed without blank-check because of the "write incomplete trailer on 95 | startup" behavior described above. 96 | - If it is relevant (type code is KV trailer, subtrailer key length and hash 97 | are correct), do a bytewise compare of the key contents. If that matches, 98 | we've found our entry. 99 | - Otherwise, skip it. 100 | 101 | This algorithm is worst-case linear in the total number of modifications that 102 | have been performed to the log since last GC. It uses no RAM beyond the 103 | watermark value and iteration state. 104 | 105 | 106 | An extension of that algorithm would maintain a cache of key hashes and record 107 | locations to move reads of at least frequently-used entries toward O(1). It 108 | would still have O(n) worst case performance. 109 | 110 | 111 | We could also define an entry that contains a key hash table, which would let us 112 | periodically "checkpoint" the filesystem and preserve fast access. If this 113 | hashtable is comprehensive, it also serves as an index of all valid previous 114 | entries and could accelerate GC. 115 | 116 | 117 | On GC, we want to copy only the _latest_ version of each entry. This discussion 118 | will focus on KV entries since they're the main one. Adding new entry forms may 119 | require GC alterations. 120 | 121 | - Start at the watermark and work backwards, like a lookup. 122 | - If an entry is incomplete, skip it. 123 | - Check if a later version of the entry has already been written. 124 | - Look up against the tracking structure. 125 | - If the tracking structure is inconclusive, do a lookup against the 126 | partially written target. 127 | - Otherwise, copy it into the new space. A bitwise copy will suffice, since all 128 | internal data is expressed in relative offsets. 129 | - Record its key in a tracking data structure 130 | - Probably use a bloom filter to track "possibly already written" 131 | - Probably also use a bounded-size hash table mapping key hashes to offsets 132 | within the target space. 133 | 134 | This is potentially n^2 if the tracking structure is too small: every update 135 | would require scanning all previous updates. Can I avoid this? 136 | 137 | 138 | 139 | Deleting an entry requires a new _delete_ entry. The alterations to the logic 140 | above that are required for handling deletes are: 141 | 142 | - On lookup, if a delete entry for a given key is found, report that the entry 143 | is missing. 144 | - On GC, delete entries do not get written to the target space, but _do_ get 145 | added to the tracking structure. 146 | - Note: this strongly suggests that the "have I copied this" scan needs to 147 | be on the processed portion of from-space, not to-space. 148 | - 149 | 150 | --- 151 | 152 | If basic checks succeed and there is an unambiguously current space, we can 153 | mount read-only. 154 | 155 | Barriers to mounting read-write include: 156 | 157 | - Incomplete entry with blank trailer sector. Would not be able to be skipped 158 | during backwards seek. Needs to have trailer sector written. 159 | - Unerased idle space. Would not be able to GC. Needs to be erased. 160 | 161 | A failure should only leave a _single_ incomplete entry at the end. Before 162 | another entry can be generated, it must be mounted read-write; for that to 163 | happen, the incomplete entry needs a filled in trailer sector, converting it to 164 | a waste entry. 165 | 166 | Because we require the other space to be erased before mounting read-write, we 167 | shouldn't be able to find it unerased _and also_ an incomplete entry. However, 168 | the process for correcting this situation seems quite clear; fix either problem 169 | in either order. 170 | 171 | 172 | -------------------------------------------------------------------------------- /examples/kvtool.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use clap::Parser; 6 | use anyhow::{Context, anyhow, bail}; 7 | use std::io::{Seek, Read, Write, SeekFrom}; 8 | use std::cell::RefCell; 9 | use num_traits::FromPrimitive; 10 | use lethe::low_level::Flash; 11 | 12 | #[derive(Parser)] 13 | struct Kvtool { 14 | #[clap(short)] 15 | sector_size: u8, 16 | 17 | #[clap(subcommand)] 18 | cmd: Cmd, 19 | 20 | image_file: std::path::PathBuf, 21 | } 22 | 23 | #[derive(Parser)] 24 | enum Cmd { 25 | Check, 26 | Repair, 27 | Format { 28 | #[clap(arg_enum)] 29 | space: ArgSpace, 30 | }, 31 | Erase { 32 | #[clap(arg_enum)] 33 | space: ArgSpace, 34 | #[clap(subcommand)] 35 | target: EraseTarget, 36 | }, 37 | Write { 38 | key: String, 39 | value: String, 40 | }, 41 | Read { 42 | key: String, 43 | }, 44 | Locate { 45 | key: String, 46 | }, 47 | Dump { 48 | #[clap(arg_enum)] 49 | space: ArgSpace, 50 | }, 51 | Evacuate { 52 | #[clap(arg_enum)] 53 | space: ArgSpace, 54 | }, 55 | Corrupt { 56 | #[clap(arg_enum)] 57 | space: ArgSpace, 58 | #[clap(subcommand)] 59 | target: EraseTarget, 60 | }, 61 | } 62 | 63 | #[derive(Parser)] 64 | enum EraseTarget { 65 | All, 66 | Sector { 67 | number: u32, 68 | }, 69 | } 70 | 71 | fn main() -> Result<(), anyhow::Error> { 72 | let args = Kvtool::from_args(); 73 | 74 | match args.sector_size { 75 | 32 => specialized_main::<32>(args)?, 76 | x => bail!("unsupported sector size {x}"), 77 | } 78 | 79 | Ok(()) 80 | } 81 | 82 | #[derive(Copy, Clone, Debug, clap::ArgEnum)] 83 | enum ArgSpace { 84 | Zero, 85 | One, 86 | } 87 | 88 | impl From for Space { 89 | fn from(a: ArgSpace) -> Self { 90 | match a { 91 | ArgSpace::Zero => Self::Zero, 92 | ArgSpace::One => Self::One, 93 | } 94 | } 95 | } 96 | 97 | fn specialized_main(args: Kvtool) -> Result<(), anyhow::Error> { 98 | let mut img = FlashImage::::open(&args.image_file) 99 | .with_context(|| { 100 | format!("opening image file {}", args.image_file.display()) 101 | })?; 102 | 103 | let mut buffer0 = [0; S]; 104 | let mut buffer1 = [0; S]; 105 | 106 | match args.cmd { 107 | Cmd::Repair => { 108 | with_mounted_image(img, |mut store| { 109 | match store.repair() { 110 | Ok(()) => println!("success"), 111 | Err(e) => println!("error: {e:?}"), 112 | } 113 | Ok(()) 114 | })?; 115 | } 116 | Cmd::Check => { 117 | for space in Space::ALL { 118 | println!("checking space {:?}", space); 119 | 120 | let result = lethe::low_level::check( 121 | &mut img, 122 | &mut buffer0, 123 | space, 124 | )?; 125 | 126 | use lethe::low_level::CheckResult; 127 | match result { 128 | CheckResult::ValidLog { generation, end, tail_erased, incomplete_write } => { 129 | println!("- contains valid log of {end} sectors"); 130 | println!("- marked as generation {generation}"); 131 | println!("- tail of space {} erased", 132 | if tail_erased { "is" } else { "IS NOT" }, 133 | ); 134 | println!("- final entry in log is {}", 135 | if incomplete_write { "INCOMPLETE" } else { "complete" }, 136 | ); 137 | } 138 | CheckResult::Bad(e) => { 139 | println!("- could not validate: {e:?}"); 140 | } 141 | } 142 | } 143 | } 144 | Cmd::Erase { space, target } => { 145 | let space = Space::from(space); 146 | match target { 147 | EraseTarget::All => { 148 | println!("erasing space {space:?}"); 149 | img.erase_space(space)?; 150 | } 151 | EraseTarget::Sector { number } => { 152 | println!("erasing sector {number} in space {space:?}"); 153 | img.erase_sector(space, number)?; 154 | } 155 | } 156 | } 157 | Cmd::Corrupt { space, target } => { 158 | let space = Space::from(space); 159 | match target { 160 | EraseTarget::All => { 161 | println!("corrupting space {space:?}"); 162 | img.erase_space(space)?; 163 | let nonsense = [0xAA; S]; 164 | for s in 0..img.sectors_per_space() { 165 | img.program_sector(space, s, &nonsense)?; 166 | } 167 | } 168 | EraseTarget::Sector { number } => { 169 | println!("corrupting sector {number} in space {space:?}"); 170 | img.erase_sector(space, number)?; 171 | let nonsense = [0xAA; S]; 172 | img.program_sector(space, number, &nonsense)?; 173 | } 174 | } 175 | } 176 | Cmd::Format { space } => { 177 | println!("formatting space {space:?}"); 178 | let r = lethe::low_level::format( 179 | &mut img, 180 | &mut buffer0, 181 | space.into(), 182 | 0, 183 | ); 184 | use lethe::low_level::FormatError; 185 | match r { 186 | Ok(()) => println!("success"), 187 | Err(FormatError::NeedsErase) => { 188 | println!("space must be erased first"); 189 | } 190 | Err(FormatError::Flash(e)) => { 191 | return Err(e).context("accessing image"); 192 | } 193 | } 194 | } 195 | Cmd::Write { key, value } => { 196 | with_writable_mounted_image(img, |mut store| { 197 | use lethe::low_level::WriteError; 198 | match store.write_kv(key.as_bytes(), value.as_bytes()) { 199 | Ok(()) => println!("ok"), 200 | Err(WriteError::NoSpace) => println!("no space"), 201 | Err(e) => println!("error: {e:?}"), 202 | } 203 | Ok(()) 204 | })?; 205 | } 206 | Cmd::Locate { key } => { 207 | with_mounted_image(img, |store| { 208 | match store.locate_kv(key.as_bytes()) { 209 | Ok(Some(x)) => println!("found at sector {x}"), 210 | Ok(None) => println!("not found"), 211 | Err(e) => println!("error: {e:?}"), 212 | } 213 | Ok(()) 214 | })?; 215 | } 216 | Cmd::Read { key } => { 217 | with_mounted_image(img, |store| { 218 | let mut out = [0; 1024]; 219 | match store.read_kv(key.as_bytes(), &mut out) { 220 | Ok(Some(n)) => { 221 | println!("{}", pretty_hex::pretty_hex(&&out[..n])); 222 | } 223 | Ok(None) => println!("not found"), 224 | Err(e) => println!("error: {e:?}"), 225 | } 226 | Ok(()) 227 | })?; 228 | } 229 | Cmd::Dump { space } => { 230 | let space = Space::from(space); 231 | let result = lethe::low_level::check( 232 | &mut img, 233 | &mut buffer0, 234 | space, 235 | )?; 236 | 237 | use lethe::low_level::CheckResult; 238 | let end = match result { 239 | CheckResult::ValidLog { end, .. } => { 240 | end 241 | } 242 | CheckResult::Bad(e) => { 243 | bail!("- could not validate: {e:?}"); 244 | } 245 | }; 246 | 247 | println!("dumping log contents from space {space:?}"); 248 | 249 | let mut seen_keys = std::collections::HashMap::new(); 250 | let mut sector = lethe::low_level::Constants::>::HEADER_SECTORS; 251 | 252 | while sector < end { 253 | println!("entry at sector {sector}"); 254 | let entry = lethe::low_level::read_entry_from_head( 255 | &mut img, 256 | &mut buffer0, 257 | space, 258 | sector, 259 | ).map_err(|e| { 260 | anyhow!("failed to read: {e:?}") 261 | })?; 262 | let kst = lethe::low_level::KnownSubtypes::from_u8(entry.meta.subtype); 263 | let next_sector = entry.next_sector; 264 | let contents_length = entry.meta.contents_length.get(); 265 | 266 | println!("- content length {}", contents_length); 267 | println!("- subtype {:#02x} ({})", 268 | entry.meta.subtype, 269 | match kst { 270 | Some(t) => format!("{:?}", t), 271 | None => "unknown".to_string(), 272 | }); 273 | 274 | match kst { 275 | Some(lethe::low_level::KnownSubtypes::Data) | Some(lethe::low_level::KnownSubtypes::Delete) => { 276 | let (subheader, _) = lethe::low_level::cast_prefix::(entry.submeta); 277 | println!("- key hash {:#08x}", subheader.key_hash.get()); 278 | 279 | let mut key = vec![0; subheader.key_length.get() as usize]; 280 | lethe::low_level::read_contents( 281 | &mut img, 282 | &mut buffer0, 283 | space, 284 | sector, 285 | 0, 286 | &mut key, 287 | ).map_err(|e| { 288 | anyhow!("failed to read key: {e:?}") 289 | })?; 290 | 291 | if let Some(prev) = seen_keys.insert(key.clone(), sector) { 292 | println!("- supercedes entry at {prev}"); 293 | } 294 | println!("Key {}", pretty_hex::pretty_hex(&key)); 295 | 296 | if kst == Some(lethe::low_level::KnownSubtypes::Delete) { 297 | println!("- this entry deletes the key"); 298 | } else { 299 | let value_len = contents_length - key.len() as u32; 300 | let mut val = vec![0; value_len as usize]; 301 | lethe::low_level::read_contents( 302 | &mut img, 303 | &mut buffer0, 304 | space, 305 | sector, 306 | key.len() as u32, 307 | &mut val, 308 | ).map_err(|e| { 309 | anyhow!("failed to read val: {e:?}") 310 | })?; 311 | println!("Value {}", pretty_hex::pretty_hex(&val)); 312 | } 313 | 314 | } 315 | _ => (), 316 | } 317 | 318 | println!(); 319 | 320 | sector = next_sector; 321 | } 322 | 323 | 324 | } 325 | Cmd::Evacuate { space } => { 326 | let space = Space::from(space); 327 | let result = lethe::low_level::check( 328 | &mut img, 329 | &mut buffer0, 330 | space, 331 | )?; 332 | 333 | use lethe::low_level::CheckResult; 334 | let end = match result { 335 | CheckResult::ValidLog { end, .. } => { 336 | end 337 | } 338 | CheckResult::Bad(e) => { 339 | bail!("- could not validate: {e:?}"); 340 | } 341 | }; 342 | 343 | println!("evacuating contents of space {:?} => {:?}", 344 | space, space.other()); 345 | 346 | lethe::low_level::evacuate( 347 | &mut img, 348 | &mut buffer0, 349 | &mut buffer1, 350 | space, 351 | end, 352 | ).map_err(|e| { 353 | anyhow!("flash access error: {e:?}") 354 | })?; 355 | 356 | println!("done"); 357 | } 358 | } 359 | 360 | Ok(()) 361 | } 362 | 363 | fn with_mounted_image( 364 | img: FlashImage, 365 | body: impl FnOnce(lethe::Store<'_, FlashImage>) -> anyhow::Result<()>, 366 | ) -> anyhow::Result<()> { 367 | let mut buffers = lethe::StoreBuffers { 368 | b0: [0; S], 369 | b1: [0; S], 370 | }; 371 | let store = match lethe::mount(img, &mut buffers) { 372 | Err(e) => bail!("could not mount: {:?}", e.cause()), 373 | Ok(store) => store, 374 | }; 375 | 376 | body(store) 377 | } 378 | 379 | fn with_writable_mounted_image( 380 | img: FlashImage, 381 | body: impl FnOnce(lethe::WritableStore<'_, FlashImage>) -> anyhow::Result<()>, 382 | ) -> anyhow::Result<()> { 383 | with_mounted_image(img, |store| { 384 | if !store.can_mount_writable() { 385 | bail!("can't mount store writable due to errors"); 386 | } 387 | 388 | let store = store.mount_writable().map_err(|_| ()).unwrap(); 389 | body(store) 390 | }) 391 | } 392 | 393 | struct FlashImage { 394 | file: RefCell, 395 | sectors_per_space: u32, 396 | } 397 | 398 | impl FlashImage { 399 | fn open(path: impl AsRef) -> Result { 400 | let file = std::fs::OpenOptions::new() 401 | .read(true) 402 | .write(true) 403 | .create(false) 404 | .open(path)?; 405 | let metadata = file.metadata()?; 406 | let file_len = metadata.len(); 407 | 408 | if file_len % S as u64 != 0 { 409 | bail!("file is not a whole number of sectors in length"); 410 | } 411 | 412 | let sectors = file_len / S as u64; 413 | if sectors % 2 != 0 { 414 | bail!("file contains an odd number of sectors"); 415 | } 416 | 417 | let sectors_per_space = u32::try_from(sectors / 2) 418 | .context("file too large")?; 419 | 420 | Ok(Self { 421 | file: file.into(), 422 | sectors_per_space, 423 | }) 424 | } 425 | 426 | fn erase_sector(&mut self, space: Space, sector: u32) -> Result<(), anyhow::Error> { 427 | let global_sector = usize::from(space) as u64 * self.sectors_per_space as u64 + sector as u64; 428 | let global_offset = global_sector * S as u64; 429 | let mut file = self.file.borrow_mut(); 430 | file.seek(SeekFrom::Start(global_offset))?; 431 | let data = [0xFF; S]; 432 | file.write_all(&data)?; 433 | Ok(()) 434 | } 435 | } 436 | 437 | use lethe::low_level::Space; 438 | 439 | impl lethe::low_level::Flash for FlashImage { 440 | type Sector = [u8; S]; 441 | type Error = std::io::Error; 442 | 443 | fn sectors_per_space(&self) -> u32 { 444 | self.sectors_per_space 445 | } 446 | 447 | fn read_sector(&self, space: Space, index: u32, dest: &mut Self::Sector) -> Result<(), Self::Error> { 448 | let global_sector = usize::from(space) as u64 * self.sectors_per_space as u64 + index as u64; 449 | let global_offset = global_sector * S as u64; 450 | 451 | let mut file = self.file.borrow_mut(); 452 | 453 | file.seek(SeekFrom::Start(global_offset))?; 454 | file.read_exact(dest)?; 455 | 456 | /* 457 | println!("sector {index}"); 458 | println!("{}", pretty_hex::pretty_hex(dest)); 459 | */ 460 | Ok(()) 461 | } 462 | 463 | fn can_program_sector(&self, space: Space, index: u32) -> Result { 464 | let mut b = [0; S]; 465 | self.read_sector(space, index, &mut b)?; 466 | Ok(b.iter().all(|&byte| byte == 0xFF)) 467 | } 468 | 469 | fn can_read_sector(&self, _space: Space, _index: u32) -> Result { 470 | // This backend allows arbitrary reads. 471 | Ok(true) 472 | } 473 | 474 | fn program_sector(&mut self, space: Space, index: u32, data: &Self::Sector) -> Result<(), Self::Error> { 475 | let global_sector = usize::from(space) as u64 * self.sectors_per_space as u64 + index as u64; 476 | let global_offset = global_sector * S as u64; 477 | 478 | let mut file = self.file.borrow_mut(); 479 | 480 | file.seek(SeekFrom::Start(global_offset))?; 481 | file.write_all(data)?; 482 | Ok(()) 483 | } 484 | 485 | fn erase_space(&mut self, space: Space) -> Result<(), Self::Error> { 486 | let global_sector = usize::from(space) as u64 * self.sectors_per_space as u64; 487 | let global_offset = global_sector * S as u64; 488 | 489 | let mut file = self.file.borrow_mut(); 490 | 491 | file.seek(SeekFrom::Start(global_offset))?; 492 | let data = [0xFF; S]; 493 | for _ in 0..self.sectors_per_space { 494 | file.write_all(&data)?; 495 | } 496 | Ok(()) 497 | } 498 | } 499 | -------------------------------------------------------------------------------- /examples/llsh.rs: -------------------------------------------------------------------------------- 1 | use rustyline::error::ReadlineError; 2 | 3 | fn main() -> Result<(), Box> { 4 | let mut rl = rustyline::Editor::<()>::new()?; 5 | 6 | loop { 7 | match rl.readline(">> ") { 8 | Ok(line) => println!("{line}"), 9 | Err(ReadlineError::Eof) => { 10 | println!("exiting."); 11 | break; 12 | } 13 | Err(ReadlineError::Interrupted) => { 14 | println!("^C"); 15 | } 16 | Err(ReadlineError::WindowResized) => { 17 | println!("wheeeee"); 18 | } 19 | Err(ReadlineError::Io(e)) => return Err(e.into()), 20 | Err(e) => { 21 | println!("unexpected error: {e:?}"); 22 | break; 23 | } 24 | } 25 | } 26 | 27 | Ok(()) 28 | } 29 | -------------------------------------------------------------------------------- /proposal.mkdn: -------------------------------------------------------------------------------- 1 | # Key-value store proposal 2 | 3 | For storage of persistent information and encrypted data on the RoT, I propose a 4 | simple key-value store, written in safe Rust. 5 | 6 | Bullet points (good and bad): 7 | 8 | - Stores associations between keys (sequences of bytes) and values (other 9 | sequences of bytes). 10 | - The current value can be looked up for a given key. 11 | - Values can be replaced, or the key-value pair deleted. 12 | - Can work on both direct-addressable (internal or memory-mapped) flash and 13 | non-addressable flash (e.g. SPI). 14 | - Can operate without much state in RAM, at the cost of potentially O(number of 15 | keys) lookups. Can also operate with RAM hashtables to accelerate this, with 16 | adjustable cost. (Tunable.) 17 | - Can record index information in flash, making future lookups constant-time 18 | without a RAM hashtable, at the cost of storage space. (Tunable.) 19 | - Resilient against crashes during updates. 20 | - Resilient against power failure during updates. 21 | - Supports atomic in-place version upgrades by hijacking the garbage collection 22 | process. 23 | - Can be mounted read-only for inspection by tools despite various forms of 24 | corruption. 25 | - Available as a portable lib crate for use in tests and tools. 26 | - Writes are typically O(1) but are worst-case O(number of entries in the store) 27 | if garbage collection is required. 28 | - Keeps half of flash in reserve for garbage collection, so a data store 29 | spanning 10 kiB can only store slightly less than 5 kiB of data. 30 | 31 | Assumptions (these appear to hold for our likely targets): 32 | 33 | - Flash is writable in units we'll call "sectors," which are at least 16 bytes. 34 | - Sector writes are atomic. 35 | - A sector must be erased to be written, and can be written once per erase. 36 | - Erase takes more time than writing and affects many sectors at once. 37 | - It is possible to tell if a sector can be written (has been erased) without 38 | hardware faults. 39 | - It is possible to tell if a sector can be read (has been written since erase) 40 | without hardware faults. 41 | - The flash has integrity protection using out-of-band information such as ECC 42 | bits per sector. 43 | - A failure during sector erase will either leave the sector erased, unchanged, 44 | or with invalid out-of-band information that makes it detectable as corrupt, 45 | but will never make up apparently valid data. 46 | 47 | Non-assumptions: 48 | 49 | - Does not rely on any clever flash behaviors like multiply programming pages. 50 | - Does not make assumptions about the contents of erased memory. 51 | 52 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | #![cfg_attr(not(test), no_std)] 6 | 7 | pub mod low_level; 8 | 9 | use core::borrow::BorrowMut; 10 | use crate::low_level::{Flash, Space}; 11 | use core::cell::RefCell; 12 | 13 | pub struct Store<'b, F: Flash> { 14 | flash: F, 15 | buffers: RefCell<&'b mut StoreBuffers>, 16 | current: Space, 17 | next_free: u32, 18 | incomplete_write: bool, 19 | tail_erased: bool, 20 | other_erased: bool, 21 | } 22 | 23 | /// Basic read-only store access. 24 | impl Store<'_, F> { 25 | /// Searches for the most recent data entry matching `key` and reads its 26 | /// value into `value_out`. 27 | /// 28 | /// At most `value_out.len()` bytes will be read; the actual value in the 29 | /// entry may be longer than this. (TODO: not great.) 30 | /// 31 | /// On success, returns `Ok(Some(bytes_read))`. If no matching entry is 32 | /// found, or if the key has been deleted, returns `Ok(None)`. 33 | /// 34 | /// Returns `Err` only if an error occurs accessing the flash dveice, or if 35 | /// the log is found to be corrupt. 36 | pub fn read_kv( 37 | &self, 38 | key: &[u8], 39 | value_out: &mut [u8], 40 | ) -> Result, low_level::ReadError> { 41 | let mut b = self.buffers.borrow_mut(); 42 | let loc = low_level::seek_kv_backwards( 43 | &self.flash, 44 | &mut b.b0, 45 | self.current, 46 | self.next_free, 47 | key, 48 | )?; 49 | 50 | if let Some(head_sector) = loc { 51 | let n = low_level::read_contents( 52 | &self.flash, 53 | &mut b.b0, 54 | self.current, 55 | head_sector, 56 | key.len() as u32, 57 | value_out, 58 | )?; 59 | Ok(Some(n)) 60 | } else { 61 | Ok(None) 62 | } 63 | } 64 | 65 | /// Searches for the most recent data entry matching `key` and returns its 66 | /// location in the current space. This is mostly useful for interacting 67 | /// with lower level APIs. 68 | /// 69 | /// If found, returns `Ok(Some(sector_number))`. 70 | /// 71 | /// If there is no entry for `key` or it has been deleted, returns 72 | /// `Ok(None)`. 73 | /// 74 | /// Returns `Err` only if an error occurs accessing the flash device, or if 75 | /// the log is found to be corrupt. 76 | pub fn locate_kv( 77 | &self, 78 | key: &[u8], 79 | ) -> Result, low_level::ReadError> { 80 | let mut b = self.buffers.borrow_mut(); 81 | low_level::seek_kv_backwards( 82 | &self.flash, 83 | &mut b.b0, 84 | self.current, 85 | self.next_free, 86 | key, 87 | ) 88 | } 89 | } 90 | 91 | /// Accessing check results, repair, mounting writable. 92 | impl<'b, F: Flash> Store<'b, F> { 93 | /// Checks whether the log check said we can mount this writable without 94 | /// further effort. 95 | pub fn can_mount_writable(&self) -> bool { 96 | self.incomplete_write == false 97 | && self.tail_erased == true 98 | && self.other_erased == true 99 | } 100 | 101 | /// Attempts to mount this writable without doing any repair actions. This 102 | /// will fail if the check found problems, which you can detect in advance 103 | /// by calling `can_mount_writable`. 104 | pub fn mount_writable(self) -> Result, Self> { 105 | if !self.can_mount_writable() { 106 | return Err(self); 107 | } 108 | 109 | Ok(WritableStore(self)) 110 | } 111 | 112 | /// Attempts to repair any issues that would prevent mounting the store 113 | /// writable. 114 | /// 115 | /// On success, `can_mount_writable` will return `true`, and 116 | /// `mount_writable` will succeed, modulo flash device failures. 117 | pub fn repair(&mut self) -> Result<(), RepairError> { 118 | // First, ensure that the idle space on the device is erased. This is 119 | // important to do first, because we may need to evacuate the log to the 120 | // other space to finish other repairs below. 121 | if !self.other_erased { 122 | self.flash.erase_space(self.current.other())?; 123 | self.other_erased = true; 124 | } 125 | 126 | // Second, check if we found programmed sectors after the log in the 127 | // current space. If this is the case, because we only assume we can 128 | // erase at the space level, we need to evacuate the log and switch 129 | // over. This will have the effect of clearing any incomplete write 130 | // condition, so we do this before repairing an incomplete write. 131 | if !self.tail_erased { 132 | let evacuated_space = self.current; 133 | let target_space = self.current.other(); 134 | // We've ensured that the idle space is erased, above. However, the 135 | // evacuation process is about to start writing to it, so clear the 136 | // flag in case it fails. 137 | self.other_erased = false; 138 | 139 | // Evacuate the entries from the current space. 140 | let buffers = self.buffers.get_mut(); 141 | let r = low_level::evacuate( 142 | &mut self.flash, 143 | &mut buffers.b0, 144 | &mut buffers.b1, 145 | evacuated_space, 146 | self.next_free, 147 | ); 148 | use low_level::ReadError; 149 | let target_watermark = match r { 150 | Err(ReadError::Flash(e)) => return Err(e.into()), 151 | Err(_) => return Err(RepairError::Corrupt), 152 | Ok(n) => n, 153 | }; 154 | // Switch the current space. 155 | self.current = target_space; 156 | self.next_free = target_watermark; 157 | self.incomplete_write = false; 158 | 159 | // Because evacuation only programs from the header to the end of 160 | // the log, we can set the tail_erased flag now. 161 | self.tail_erased = true; 162 | 163 | // Now, erase the space we evacuated. 164 | self.flash.erase_space(evacuated_space)?; 165 | 166 | // Finally, we can restore other_erased to false. 167 | self.other_erased = true; 168 | } else if self.incomplete_write { 169 | // We only check incomplete_write if tail_erased is true, for the 170 | // reasons discussed above. 171 | 172 | // We repair incomplete writes by filling in their trailer sector 173 | // with the Aborted marker. The incomplete write will start at our 174 | // end-of-log (next_free) location. 175 | let r = low_level::read_entry_from_head( 176 | &mut self.flash, 177 | &mut self.buffers.get_mut().b0, 178 | self.current, 179 | self.next_free, 180 | ); 181 | use low_level::ReadError; 182 | let entry = match r { 183 | Err(ReadError::Flash(e)) => return Err(e.into()), 184 | Err(_) => return Err(RepairError::Corrupt), 185 | Ok(entry) => entry, 186 | }; 187 | 188 | // Copy the entry metadata so we can stuff it into the trailer. 189 | let mut meta = *entry.meta; 190 | meta.subtype = low_level::KnownSubtypes::Aborted as u8; 191 | meta.sub_bytes = 0; 192 | 193 | let trailer = entry.next_sector - 1; 194 | 195 | let buffer = &mut self.buffers.get_mut().b0; 196 | let b = (*buffer).borrow_mut(); 197 | b.fill(0); 198 | *low_level::cast_suffix_mut(b).1 = meta; 199 | self.flash.program_sector(self.current, trailer, buffer)?; 200 | 201 | // Update the end-of-log pointer to include this entry and clear the 202 | // error. 203 | self.next_free = trailer + 1; 204 | self.incomplete_write = false; 205 | } 206 | 207 | debug_assert!(!self.incomplete_write); 208 | debug_assert!(self.tail_erased); 209 | debug_assert!(self.other_erased); 210 | 211 | Ok(()) 212 | } 213 | } 214 | 215 | #[derive(Copy, Clone, Debug)] 216 | pub enum RepairError { 217 | Corrupt, 218 | Flash(E), 219 | } 220 | 221 | impl From for RepairError { 222 | fn from(e: E) -> Self { 223 | Self::Flash(e) 224 | } 225 | } 226 | 227 | /// A data store that is available for both reads and writes. (Reads happen by 228 | /// `Deref` to `Store`.) 229 | pub struct WritableStore<'b, F: Flash>(Store<'b, F>); 230 | 231 | impl<'b, F: Flash> core::ops::Deref for WritableStore<'b, F> { 232 | type Target = Store<'b, F>; 233 | fn deref(&self) -> &Self::Target { 234 | &self.0 235 | } 236 | } 237 | 238 | impl WritableStore<'_, F> { 239 | /// Appends a data entry to the log assigning `value` to `key`. 240 | pub fn write_kv( 241 | &mut self, 242 | key: &[u8], 243 | value: &[u8], 244 | ) -> Result<(), low_level::WriteError> { 245 | low_level::write_kv( 246 | &mut self.0.flash, 247 | &mut self.0.buffers.get_mut().b0, 248 | self.0.current, 249 | self.0.next_free, 250 | key, 251 | value, 252 | )?; 253 | Ok(()) 254 | } 255 | } 256 | 257 | /// Buffers required for the data store implementation on flash device `F`. 258 | #[derive(Default)] 259 | pub struct StoreBuffers { 260 | pub b0: F::Sector, 261 | pub b1: F::Sector, 262 | } 263 | 264 | /// Mounts `flash` for (initially) read-only access, performing an integrity 265 | /// check. 266 | pub fn mount( 267 | mut flash: F, 268 | buffers: &mut StoreBuffers, 269 | ) -> Result, MountError> { 270 | match mount_inner(&mut flash, buffers) { 271 | Err(cause) => Err(MountError { flash, cause }), 272 | Ok((current, next_free, incomplete_write, tail_erased, other_erased)) => { 273 | Ok(Store { 274 | flash, 275 | buffers: buffers.into(), 276 | current, 277 | next_free, 278 | incomplete_write, 279 | tail_erased, 280 | other_erased, 281 | }) 282 | } 283 | } 284 | } 285 | 286 | fn mount_inner(flash: &mut F, buffers: &mut StoreBuffers) -> Result<(Space, u32, bool, bool, bool), MountErrorCause> { 287 | // Run log checks on both spaces. 288 | use low_level::CheckResult; 289 | let c0 = low_level::check(flash, &mut buffers.b0, Space::Zero)?; 290 | let c1 = low_level::check(flash, &mut buffers.b0, Space::One)?; 291 | 292 | let (current, other_erased) = match (&c0, &c1) { 293 | // If both spaces contain valid data, choose the one with the greater 294 | // generation number, using sequence number arithmetic. 295 | (CheckResult::ValidLog { generation: g0, .. }, CheckResult::ValidLog { generation: g1, .. }) => { 296 | use core::cmp::Ordering; 297 | match sequence_compare(*g0, *g1) { 298 | Ordering::Greater => (Space::Zero, false), 299 | Ordering::Less => (Space::One, false), 300 | Ordering::Equal => return Err(MountErrorCause::GenerationsAmbiguous), 301 | } 302 | } 303 | 304 | // If only one space is valid, choose it, obvs. 305 | (CheckResult::ValidLog { .. }, CheckResult::Bad(e)) => ( 306 | Space::Zero, 307 | *e == low_level::CheckError::Erased, 308 | ), 309 | (CheckResult::Bad(e), CheckResult::ValidLog { .. }) => ( 310 | Space::One, 311 | *e == low_level::CheckError::Erased, 312 | ), 313 | 314 | // If both spaces are bad... we can't proceed. 315 | (CheckResult::Bad(bad0), CheckResult::Bad(bad1)) => return Err(MountErrorCause::NoValidLogs( 316 | *bad0, 317 | *bad1, 318 | )), 319 | }; 320 | 321 | let current_result = match current { 322 | Space::Zero => c0, 323 | Space::One => c1, 324 | }; 325 | 326 | let (end, incomplete_write, tail_erased) = match current_result { 327 | CheckResult::ValidLog { end, incomplete_write, tail_erased, .. } => { 328 | (end, incomplete_write, tail_erased) 329 | } 330 | _ => unreachable!(), 331 | }; 332 | 333 | Ok((current, end, incomplete_write, tail_erased, other_erased)) 334 | } 335 | 336 | pub struct MountError { 337 | flash: F, 338 | cause: MountErrorCause, 339 | } 340 | 341 | impl MountError { 342 | pub fn into_inner(self) -> F { 343 | self.flash 344 | } 345 | 346 | pub fn cause(&self) -> &MountErrorCause { 347 | &self.cause 348 | } 349 | } 350 | 351 | #[derive(Copy, Clone, Debug)] 352 | pub enum MountErrorCause { 353 | GenerationsAmbiguous, 354 | NoValidLogs(low_level::CheckError, low_level::CheckError), 355 | Flash(E), 356 | } 357 | 358 | impl From for MountErrorCause { 359 | fn from(e: E) -> Self { 360 | Self::Flash(e) 361 | } 362 | } 363 | 364 | fn sequence_compare(a: u32, b: u32) -> core::cmp::Ordering { 365 | (a.wrapping_sub(b) as i32).cmp(&0) 366 | } 367 | -------------------------------------------------------------------------------- /src/low_level.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use core::mem::size_of; 6 | use core::marker::PhantomData; 7 | use core::borrow::{Borrow, BorrowMut}; 8 | use zerocopy::{AsBytes, FromBytes, Unaligned}; 9 | use num_traits::FromPrimitive; 10 | 11 | ////////////////////////////////////////////////////////////////////////////// 12 | // Convenience wrappers for zerocopy. 13 | 14 | pub fn cast_prefix(bytes: &[u8]) -> (&T, &[u8]) 15 | where T: FromBytes + Unaligned, 16 | { 17 | let (lv, rest) = zerocopy::LayoutVerified::<_, T>::new_unaligned_from_prefix(bytes) 18 | .expect("type does not fit in sector"); 19 | (lv.into_ref(), rest) 20 | } 21 | 22 | fn cast_prefix_mut(bytes: &mut [u8]) -> (&mut T, &mut [u8]) 23 | where T: AsBytes + FromBytes + Unaligned, 24 | { 25 | let (lv, rest) = zerocopy::LayoutVerified::<_, T>::new_unaligned_from_prefix(bytes) 26 | .expect("type does not fit in sector"); 27 | (lv.into_mut(), rest) 28 | } 29 | 30 | fn cast_suffix(bytes: &[u8]) -> (&[u8], &T) 31 | where T: FromBytes + Unaligned, 32 | { 33 | let (rest, lv) = zerocopy::LayoutVerified::<_, T>::new_unaligned_from_suffix(bytes) 34 | .expect("type does not fit in sector"); 35 | (rest, lv.into_ref()) 36 | } 37 | 38 | pub(crate) fn cast_suffix_mut(bytes: &mut [u8]) -> (&mut [u8], &mut T) 39 | where T: AsBytes + FromBytes + Unaligned, 40 | { 41 | let (rest, lv) = zerocopy::LayoutVerified::<_, T>::new_unaligned_from_suffix(bytes) 42 | .expect("type does not fit in sector"); 43 | (rest, lv.into_mut()) 44 | } 45 | 46 | ////////////////////////////////////////////////////////////////////////////// 47 | // At-rest layout. 48 | 49 | /// Shorthand for a `u16` in little-endian representation. 50 | type U16LE = zerocopy::U16; 51 | /// Shorthand for a `u32` in little-endian representation. 52 | type U32LE = zerocopy::U32; 53 | 54 | /// Header written to the start of a formatted space, to mark it as such. 55 | /// 56 | /// This will appear at the start of the first sector in a space. We take care 57 | /// to write this sector _last_ so that this header only appears on fully 58 | /// initialized spaces. 59 | #[derive(Copy, Clone, Debug, FromBytes, AsBytes, Unaligned)] 60 | #[repr(C)] 61 | pub struct SpaceHeader { 62 | /// Magic number (`EXPECTED_MAGIC`) distinguishing this from arbitrary data. 63 | pub magic: U32LE, 64 | /// Generation number (sequence number) of this space. This is used to 65 | /// tie-break if we find both spaces initialized. 66 | pub generation: U32LE, 67 | /// Base 2 logarithm of the sector size. This is used as a check to ensure 68 | /// that the implementation has been correctly configured for the space. 69 | pub l2_sector_size: u8, 70 | /// Reserved padding bytes, must be zero. 71 | pub pad: [u8; 3], 72 | /// CRC32 of the above data, in order. Used to be doubly sure this isn't 73 | /// arbitrary data. 74 | pub crc: U32LE, 75 | } 76 | 77 | impl SpaceHeader { 78 | /// Bits we expect to find in the `magic` field. (This is a random number.) 79 | pub const EXPECTED_MAGIC: u32 = 0x53_be_88_9f; 80 | /// Smallest practical sector size (16 bytes). 81 | pub const MIN_L2_SECTOR_SIZE: u8 = 4; 82 | /// Largest halfway-reasonable sector size (1 GiB). 83 | pub const MAX_L2_SECTOR_SIZE: u8 = 30; 84 | 85 | /// Basic internal integrity check of the header. Checks fields against 86 | /// static ranges and verifies the magic and CRC. This doesn't know the 87 | /// sector size you're expecting, so you'll need to check that separately. 88 | pub fn check(&self) -> bool { 89 | self.magic.get() == Self::EXPECTED_MAGIC 90 | && self.l2_sector_size >= Self::MIN_L2_SECTOR_SIZE 91 | && self.l2_sector_size <= Self::MAX_L2_SECTOR_SIZE 92 | && self.pad == [0; 3] 93 | && self.crc_valid() 94 | } 95 | 96 | /// Compute the _expected_ CRC given all the other contents of `self`. 97 | pub fn expected_crc(&self) -> u32 { 98 | let algo = crc::Crc::::new(&crc::CRC_32_ISO_HDLC); 99 | let mut digest = algo.digest(); 100 | digest.update(self.magic.as_bytes()); 101 | digest.update(self.generation.as_bytes()); 102 | digest.update(self.l2_sector_size.as_bytes()); 103 | digest.update(&self.pad); 104 | 105 | digest.finalize() 106 | } 107 | 108 | /// Checks if the CRC field correctly describes the other fields in `self`. 109 | pub fn crc_valid(&self) -> bool { 110 | self.crc.get() == self.expected_crc() 111 | } 112 | } 113 | 114 | /// Metadata record used to identify entries. 115 | /// 116 | /// The metadata is written at both offset 0 in the first sector of the entry, 117 | /// and in the final bytes of the last sector, to ensure that the entry can be 118 | /// processed from either direction. 119 | #[derive(Copy, Clone, Debug, FromBytes, AsBytes, Unaligned, Eq, PartialEq)] 120 | #[repr(C)] 121 | pub struct EntryMeta { 122 | /// Marker to designate an entry and help distinguish it from 123 | /// unprogrammed or random data. 124 | pub magic: U16LE, 125 | /// Type of entry, and thus sub-meta. See the `KnownSubtypes` enum for 126 | /// defined values. 127 | pub subtype: u8, 128 | /// Length of submeta in bytes. The length must be correct for the `subtype` 129 | /// -- this is _not_ intended to support variable-length submeta. 130 | pub sub_bytes: u8, 131 | /// Length of contents separating the header from the trailer. This length 132 | /// is in bytes; the actual contents will be followed by enough padding to 133 | /// justify the subtrailer/trailer to the end of the sector. 134 | pub contents_length: U32LE, 135 | } 136 | 137 | impl EntryMeta { 138 | /// Bits we expect to find in the `magic` field. 139 | pub const EXPECTED_MAGIC: u16 = 0xCB_F5; 140 | 141 | /// Number of bytes in the full header/trailer including both this and the 142 | /// associated sub-meta. 143 | pub fn meta_length(&self) -> usize { 144 | size_of::() + usize::from(self.sub_bytes) 145 | } 146 | 147 | /// Number of bytes in the entire entry this record describes, without 148 | /// padding for sector alignment. 149 | pub fn unpadded_entry_length(&self) -> usize { 150 | 2 * self.meta_length() + self.contents_length.get() as usize 151 | } 152 | 153 | /// Number of sectors in the entire entry this record describes. 154 | pub fn entry_sectors(&self) -> u32 { 155 | bytes_to_sectors::(self.unpadded_entry_length() as u32) 156 | } 157 | } 158 | 159 | /// Defined values for the `EntryMeta::subtype` field. 160 | #[derive(Copy, Clone, Debug, Eq, PartialEq, num_derive::FromPrimitive)] 161 | pub enum KnownSubtypes { 162 | // Note: 0 is reserved. 163 | 164 | /// An entry containing stored data. The sub-header/trailer will use the 165 | /// `DataSubMeta` format. 166 | Data = 0x01, 167 | /// An entry that serves to delete a previous entry. Uses the 168 | /// `DeleteSubMeta` format which happens to match data. 169 | Delete = 0x02, 170 | 171 | /// An entry that was incompletely written before a restart (and then 172 | /// repaired). Has no sub-header/trailer (length 0). 173 | /// 174 | /// `Aborted` is the only subtype that can appear in the trailer of an entry 175 | /// whose header reports a different type. 176 | Aborted = 0xFE, 177 | 178 | // Note: 0xFF is reserved. 179 | } 180 | 181 | /// Sub-metadata used for Data entries, representing key-value pairs. 182 | #[derive(Copy, Clone, Debug, FromBytes, AsBytes, Unaligned)] 183 | #[repr(C)] 184 | pub struct DataSubMeta { 185 | /// Number of bytes in the key. 186 | pub key_length: U32LE, 187 | /// Hash of the key bytes using FNV-1, to assist in key lookup. 188 | pub key_hash: U32LE, 189 | } 190 | 191 | impl DataSubMeta { 192 | /// Size of the sub-metadata, in bytes. 193 | pub const SIZE: usize = size_of::(); 194 | /// Same, but converted to `u8` for convenient use in the `sub_bytes` field. 195 | pub const SUB_BYTES: u8 = Self::SIZE as u8; 196 | } 197 | 198 | /// Computes the hash value corresponding to a particular key. 199 | pub fn hash_key(key: &[u8]) -> u32 { 200 | const KEY_HASH_KEY: u64 = 0; 201 | 202 | use core::hash::{Hash, Hasher}; 203 | 204 | let mut hasher = fnv::FnvHasher::with_key(KEY_HASH_KEY); 205 | key.hash(&mut hasher); 206 | let h = hasher.finish(); 207 | h as u32 ^ (h >> 32) as u32 208 | } 209 | 210 | /// Sub-metadata used for Delete entries. 211 | pub type DeleteSubMeta = DataSubMeta; 212 | 213 | 214 | ////////////////////////////////////////////////////////////////////////////// 215 | // Flash device interface. 216 | 217 | /// Designates one of the two spaces in a flash device. This is like a ranged 218 | /// integer, or a bool with application-specific names. 219 | #[derive(Copy, Clone, Debug, Eq, PartialEq)] 220 | pub enum Space { 221 | Zero = 0, 222 | One = 1, 223 | } 224 | 225 | impl Space { 226 | /// Convenient array of all spaces. 227 | pub const ALL: [Self; 2] = [Self::Zero, Self::One]; 228 | 229 | /// Given a space, get the _other_ one. 230 | pub fn other(self) -> Self { 231 | match self { 232 | Self::Zero => Self::One, 233 | Self::One => Self::Zero, 234 | } 235 | } 236 | } 237 | 238 | impl From for usize { 239 | fn from(s: Space) -> Self { 240 | match s { 241 | Space::Zero => 0, 242 | Space::One => 1, 243 | } 244 | } 245 | } 246 | 247 | /// Trait describing flash memory for the purposes of our datastore. 248 | pub trait Flash { 249 | /// Type of sector, which is typically a `[u8; N]` for some sector size `N`; 250 | /// this is a type alias rather than a `const` because of restrictions on 251 | /// the use of associated constants on type parameters in array sizes; see 252 | /// `rust-lang/rust#43408`. 253 | type Sector: Sized + BorrowMut<[u8]> + Borrow<[u8]>; 254 | 255 | /// Error type that can be produced during flash accesses. 256 | type Error; 257 | 258 | /// Returns the number of sectors per space (of two spaces) in this device. 259 | /// Note that this operation cannot fail; it's expected to be relatively 260 | /// quick and always return the same result. 261 | fn sectors_per_space(&self) -> u32; 262 | 263 | /// Reads sector `index` from `space` in the flash device, writing the data 264 | /// into `dest`. 265 | /// 266 | /// On success, `dest` should be fully overwritten with data from the 267 | /// device. On failure, you can leave `dest` partially or completely 268 | /// untouched. 269 | /// 270 | /// Depending on the underlying device, `read_sector` may return an error 271 | /// when applied to an erased sector, or may produce data. 272 | fn read_sector( 273 | &self, 274 | space: Space, 275 | index: u32, 276 | dest: &mut Self::Sector, 277 | ) -> Result<(), Self::Error>; 278 | 279 | /// Checks whether sector `index` in `space` in the flash device can be 280 | /// read. This is intended to signal that the sector has been programmed 281 | /// since last being erased. However, some devices will freely read 282 | /// erased-but-not-programmed sectors; for such devices, this function can 283 | /// simply return `true`. 284 | fn can_read_sector(&self, space: Space, index: u32) -> Result; 285 | 286 | /// Checks whether sector `index` in `space` in the flash device can be 287 | /// programmed (using `program_sector`). This is intended to check whether 288 | /// the sector has been erased. 289 | /// 290 | /// For devices that support multiple overlapping writes per sector, this 291 | /// should indicate whether the sector can be programmed _and then 292 | /// faithfully read back the intended data,_ rather than (say) 293 | /// bitwise-NANDing it with previous contents. 294 | fn can_program_sector( 295 | &self, 296 | space: Space, 297 | index: u32, 298 | ) -> Result; 299 | 300 | /// Writes `data` into sector `index` in `space` in the flash device. 301 | /// 302 | /// On success, the flash device _should_ report the same data from 303 | /// `read_sector` until the sector is erased -- modulo hardware failures, 304 | /// etc. On error, the sector _may or may not_ contain the data. It's 305 | /// probably best to assume that it needs to be erased. 306 | fn program_sector( 307 | &mut self, 308 | space: Space, 309 | index: u32, 310 | data: &Self::Sector, 311 | ) -> Result<(), Self::Error>; 312 | 313 | /// Erases the contents of `space`. 314 | fn erase_space( 315 | &mut self, 316 | space: Space, 317 | ) -> Result<(), Self::Error>; 318 | 319 | /// Compares `data.len()` bytes starting at `offset` from the start of 320 | /// sector `index` for equality. `offset` may be larger than a sector, for 321 | /// convenience. 322 | /// 323 | /// This is a "pushed compare" operation to take advantage of situations 324 | /// where we can do the compare without reading out every sector into RAM, 325 | /// such as directly-addressable flash. 326 | /// 327 | /// The `buffer` argument is loaned to the driver, which may arbitrarily 328 | /// scribble over its contents while doing the compare. 329 | /// 330 | /// The default implementation uses `read_sector`. 331 | fn compare_contents( 332 | &self, 333 | space: Space, 334 | buffer: &mut Self::Sector, 335 | mut index: u32, 336 | offset: u32, 337 | mut data: &[u8], 338 | ) -> Result { 339 | let sector_size = size_of::(); 340 | 341 | let mut offset = offset as usize; 342 | 343 | while offset >= sector_size { 344 | index += 1; 345 | offset -= sector_size; 346 | } 347 | 348 | while !data.is_empty() { 349 | self.read_sector(space, index, buffer)?; 350 | let n = sector_size - offset; 351 | let n = usize::min(n, data.len()); 352 | let (this_data, next_data) = data.split_at(n); 353 | if (*buffer).borrow()[offset..offset+n] != *this_data { 354 | return Ok(false); 355 | } 356 | 357 | index += 1; 358 | offset = 0; 359 | data = next_data; 360 | } 361 | 362 | Ok(true) 363 | } 364 | 365 | /// Compares two sequences of bytes in the flash device for equality. The 366 | /// sequences start at a sector boundary but can be any byte length, and do 367 | /// not need to be in the same space. 368 | /// 369 | /// This is a "pushed compare" operation for cases where the flash device 370 | /// can compare sequences of bytes with fewer copies. 371 | /// 372 | /// The default implementation uses `read_sector`. 373 | fn compare_internal( 374 | &self, 375 | space0: Space, 376 | mut sector0: u32, 377 | space1: Space, 378 | mut sector1: u32, 379 | mut length: u32, 380 | buffer0: &mut Self::Sector, 381 | buffer1: &mut Self::Sector, 382 | ) -> Result { 383 | while length > 0 { 384 | self.read_sector(space0, sector0, buffer0)?; 385 | self.read_sector(space1, sector1, buffer1)?; 386 | let n = length.min(size_of::() as u32); 387 | if (*buffer0).borrow()[..n as usize] != (*buffer1).borrow()[..n as usize] { 388 | return Ok(false); 389 | } 390 | sector0 += 1; 391 | sector1 += 1; 392 | length -= n; 393 | } 394 | Ok(true) 395 | } 396 | 397 | /// Copies a sequence of sectors from `from_space`/`from_sector` to 398 | /// `to_sector` in the other space. 399 | /// 400 | /// Drivers that can do a flash-to-flash copy without needing to copy all 401 | /// the data through RAM can implement this to do so. The default 402 | /// implementation uses `read_sector`/`program_sector`. 403 | fn copy_across( 404 | &mut self, 405 | from_space: Space, 406 | from_sector: u32, 407 | to_sector: u32, 408 | count: u32, 409 | buffer: &mut Self::Sector, 410 | ) -> Result<(), Self::Error> { 411 | let to_space = from_space.other(); 412 | let src = from_sector..from_sector + count; 413 | let dst = to_sector..to_sector + count; 414 | 415 | for (fs, ts) in src.zip(dst) { 416 | self.read_sector(from_space, fs, buffer)?; 417 | self.program_sector(to_space, ts, buffer)?; 418 | } 419 | Ok(()) 420 | } 421 | } 422 | 423 | /// Handy routine for converting a byte length to a sector count (rounded up) on 424 | /// a given flash device. 425 | /// 426 | /// Note: this is not `const` merely because of limitations on the use of trait 427 | /// bounds in `const fn` at the time of this writing. 428 | pub fn bytes_to_sectors(x: u32) -> u32 { 429 | let sector_size = size_of::() as u32; 430 | (x + sector_size - 1) / sector_size 431 | } 432 | 433 | /// Provides a way to hang constants off an implementation of the Flash trait 434 | /// without them being overrideable to incorrect values by an implementation. 435 | pub struct Constants(PhantomData); 436 | 437 | impl Constants { 438 | pub const HEADER_SECTORS: u32 = { 439 | let sector = size_of::(); 440 | ((size_of::() + sector - 1) / sector) as u32 441 | }; 442 | } 443 | 444 | 445 | ////////////////////////////////////////////////////////////////////////////// 446 | // Filesystem operations/algorithms: general entry structure, reading, and 447 | // search. 448 | 449 | /// Reads entry metadata given the address of its first (head) sector. 450 | /// 451 | /// This will fail if the entry metadata is corrupt: bad magic number, 452 | /// sub-header size out of range, or nonsensical content length. 453 | /// 454 | /// On success, the first sector of the entry is loaded into `buffer`, and an 455 | /// `EntryInfo` struct referencing `buffer` provides the parse results. The 456 | /// `EntryInfo::next_sector` field points to the sector _just past_ the tail 457 | /// sector of this entry -- which is where you'd need to apply 458 | /// `read_entry_from_head` again to continue forward through the log. 459 | pub fn read_entry_from_head<'b, F: Flash>( 460 | flash: &F, 461 | buffer: &'b mut F::Sector, 462 | current: Space, 463 | sector: u32, 464 | ) -> Result, ReadError> { 465 | flash.read_sector(current, sector, buffer)?; 466 | let data = (*buffer).borrow(); 467 | let (meta, data) = cast_prefix::(data); 468 | 469 | if meta.magic.get() != EntryMeta::EXPECTED_MAGIC { 470 | return Err(ReadError::BadMagic(sector)); 471 | } 472 | let submeta = data.get(..usize::from(meta.sub_bytes)) 473 | .ok_or(ReadError::BadSubBytes(sector))?; 474 | 475 | let next_sector = sector.checked_add(meta.entry_sectors::()) 476 | .ok_or(ReadError::BadLength(sector))?; 477 | 478 | Ok(EntryInfo { 479 | next_sector, 480 | meta, 481 | submeta, 482 | }) 483 | } 484 | 485 | /// Parsed information about an entry. 486 | #[derive(Copy, Clone, Debug)] 487 | pub struct EntryInfo<'a> { 488 | /// Sector number of the other end of this entry. Which end depends on which 489 | /// direction you were reading in. 490 | pub next_sector: u32, 491 | /// Entry metadata in sector buffer. 492 | pub meta: &'a EntryMeta, 493 | /// Entry sub-metadata in sector buffer. 494 | pub submeta: &'a [u8], 495 | } 496 | 497 | /// Things that can go wrong while reading low-level entries. 498 | #[derive(Copy, Clone, Debug)] 499 | pub enum ReadError { 500 | /// Magic number (given) was wrong. 501 | BadMagic(u32), 502 | /// Length of sub-metadata (given) was too large. 503 | BadSubBytes(u32), 504 | /// Content length of entry (given) was too large for device. 505 | BadLength(u32), 506 | 507 | // TODO: probably doesn't belong in low_level 508 | End(u32), 509 | 510 | Flash(E), 511 | } 512 | 513 | impl From for ReadError { 514 | fn from(e: E) -> Self { 515 | Self::Flash(e) 516 | } 517 | } 518 | 519 | /// Reads entry metadata given the address of the first sector past its final 520 | /// sector (tail). 521 | /// 522 | /// This will fail if the entry metadata is corrupt: bad magic number, 523 | /// sub-header size out of range, or nonsensical content length. 524 | /// 525 | /// On success, the last sector of the entry is loaded into `buffer`, and an 526 | /// `EntryInfo` struct referencing `buffer` provides the parse results. The 527 | /// `EntryInfo::next_sector` field points to the entry's head (first) sector -- 528 | /// which is where you'd need to apply `read_entry_from_tail` again to continue 529 | /// backward through the log. 530 | pub(crate) fn read_entry_from_tail<'b, F: Flash>( 531 | flash: &F, 532 | buffer: &'b mut F::Sector, 533 | current: Space, 534 | sector: u32, 535 | ) -> Result, ReadError> { 536 | // Adjust sector number to point at the tail instead of just past it. 537 | let sector = sector - 1; 538 | 539 | flash.read_sector(current, sector, buffer)?; 540 | let data = (*buffer).borrow(); 541 | let (data, meta) = cast_suffix::(data); 542 | 543 | if meta.magic.get() != EntryMeta::EXPECTED_MAGIC { 544 | return Err(ReadError::BadMagic(sector)); 545 | } 546 | let submeta_start = data.len().checked_sub(usize::from(meta.sub_bytes)) 547 | .ok_or(ReadError::BadSubBytes(sector))?; 548 | let submeta = &data[submeta_start..]; 549 | 550 | let next_trailer = sector 551 | .checked_sub(meta.entry_sectors::()) 552 | .ok_or(ReadError::BadLength(sector))?; 553 | let next_sector = next_trailer + 1; 554 | 555 | Ok(EntryInfo { 556 | next_sector, 557 | meta, 558 | submeta, 559 | }) 560 | } 561 | 562 | /// Processes entries in the log in space `current` starting from the end 563 | /// (`start_sector`) and working backwards. Each entry is presented to `filter`, 564 | /// which controls the traversal. 565 | /// 566 | /// If `filter` returns `Ignore` for an entry, traversal continues. If traversal 567 | /// hits the start of the log, this function returns `Ok(None)`. 568 | /// 569 | /// If it returns `Accept`, traversal stops, and the head sector number for the 570 | /// entry is returned. 571 | /// 572 | /// If it returns `Abort`, traversal stops, and the function returns `Ok(None)` 573 | /// as if no entry were found. (This is helpful for implementing Delete 574 | /// entries.) 575 | pub(crate) fn seek_backwards<'b, F: Flash>( 576 | flash: &F, 577 | buffer: &'b mut F::Sector, 578 | current: Space, 579 | start_sector: u32, 580 | mut filter: impl FnMut(&F, &mut F::Sector, u32, KnownSubMetas) -> Result, 581 | ) -> Result, ReadError> { 582 | let header_sectors = Constants::::HEADER_SECTORS; 583 | 584 | assert!(start_sector >= header_sectors); 585 | 586 | let mut sector = start_sector; 587 | 588 | while sector > header_sectors { 589 | let entry = read_entry_from_tail(flash, buffer, current, sector)?; 590 | 591 | let head_sector = entry.next_sector; 592 | let ksh = KnownSubMetas::new(entry.meta.subtype, entry.submeta); 593 | 594 | let r = filter(flash, buffer, head_sector, ksh)?; 595 | match r { 596 | EntryDecision::Ignore => (), 597 | EntryDecision::Accept => return Ok(Some(head_sector)), 598 | EntryDecision::Abort => return Ok(None), 599 | } 600 | 601 | sector = head_sector; 602 | } 603 | 604 | Ok(None) 605 | } 606 | 607 | /// Metadata passed by-value to an entry seek predicate. 608 | #[derive(Copy, Clone, Debug)] 609 | pub enum KnownSubMetas { 610 | Data(DataSubMeta), 611 | Delete(DeleteSubMeta), 612 | Aborted, 613 | Other(u8), 614 | } 615 | 616 | impl KnownSubMetas { 617 | pub fn new(subtype: u8, submeta: &[u8]) -> Self { 618 | match KnownSubtypes::from_u8(subtype) { 619 | Some(KnownSubtypes::Data) => Self::Data(*cast_prefix(submeta).0), 620 | Some(KnownSubtypes::Delete) => Self::Delete(*cast_prefix(submeta).0), 621 | Some(KnownSubtypes::Aborted) => Self::Aborted, 622 | None => Self::Other(subtype), 623 | } 624 | } 625 | } 626 | 627 | /// Decisions a predicate may make about an entry during log traversal. 628 | #[derive(Copy, Clone, Debug)] 629 | pub(crate) enum EntryDecision { 630 | /// Keep going. 631 | Ignore, 632 | /// This is the entry we were looking for. 633 | Accept, 634 | /// Something about this entry means that the log doesn't _contain_ the 635 | /// entry we were looking for. 636 | Abort, 637 | } 638 | 639 | /// Reads the contents bytes of an entry -- the stuff between the sub-header and 640 | /// sub-trailer. This is a raw access function used in the implementation of 641 | /// higher level access functions, and is exposed for use in tooling. 642 | /// 643 | /// When applied to a Data entry, this will read out the raw data record. Other 644 | /// entries do not currently have contents. 645 | /// 646 | /// The entry will be found starting at `head_sector` in `flash` from the 647 | /// `current` space. Data will be copied starting `offset` bytes into the 648 | /// contents, into `out`. `buffer` will be scribbled upon in the process as 649 | /// scratch. 650 | /// 651 | /// This works somewhat like POSIX read: 652 | /// 653 | /// - If there are `out.len()` bytes available to read starting at `offset`, 654 | /// this returns `Ok(out.len())` to indicate that it has filled `out`. 655 | /// 656 | /// - If it would overlap the end of the entry's contents, it only fills in as 657 | /// much of `out` as there are bytes available, and returns 658 | /// `Ok(bytes_filled)`. 659 | /// 660 | /// - If `offset` is exactly the length of the contents, returns `Ok(0)`. 661 | /// 662 | /// - If `offset` is _outside_ the length of the contents, returns `Err(End)`. 663 | pub fn read_contents( 664 | flash: &F, 665 | buffer: &mut F::Sector, 666 | current: Space, 667 | head_sector: u32, 668 | offset: u32, 669 | out: &mut [u8], 670 | ) -> Result> { 671 | let entry = read_entry_from_head( 672 | flash, 673 | buffer, 674 | current, 675 | head_sector, 676 | )?; 677 | // Reject offsets that are totally outside the contents. 678 | let len_after_offset = entry.meta.contents_length.get() 679 | .checked_sub(offset) 680 | .ok_or(ReadError::End(entry.meta.contents_length.get()))?; 681 | // Compute the sector-relative offset by adding the length of the headers. 682 | let abs_offset = size_of::() 683 | + usize::from(entry.meta.sub_bytes) 684 | + offset as usize; 685 | // Limit the length of the read to the size of the out buffer. 686 | let xfer_len = usize::min(len_after_offset as usize, out.len()); 687 | 688 | // Set up our loop variables by computing our starting sector, offset within 689 | // the sector, and length of output buffer. 690 | let mut sector = head_sector as usize 691 | + abs_offset / size_of::(); 692 | let mut offset = abs_offset % size_of::(); 693 | let mut out = &mut out[..xfer_len]; 694 | 695 | while !out.is_empty() { 696 | flash.read_sector( 697 | current, 698 | sector as u32, 699 | buffer, 700 | )?; 701 | 702 | let n = usize::min(size_of::() - offset, xfer_len); 703 | out[..n].copy_from_slice(&(*buffer).borrow()[offset..offset + n]); 704 | 705 | // To prepare for the next iteration, we zero offset (so that it's 706 | // non-zero only on the first iteration) and advance the others. 707 | offset = 0; 708 | sector += 1; 709 | out = &mut out[n..]; 710 | } 711 | 712 | Ok(xfer_len) 713 | } 714 | 715 | ////////////////////////////////////////////////////////////////////////////// 716 | // Filesystem operations/algorithms: general entry writing. 717 | 718 | /// Writes a new entry. 719 | /// 720 | /// The entry will be written starting at `start_sector` in `flash`, in the 721 | /// space `current`. 722 | /// 723 | /// The entry will be constructed from: 724 | /// 725 | /// - the slices in `pieces`, in order without padding, 726 | /// - enough padding to right-align the remainder, 727 | /// - `subtrailer` and `trailer` in that order. 728 | /// 729 | /// This means that, in practice, `pieces[0]` should be the header metadata and 730 | /// `pieces[1]` the submetadata. 731 | /// 732 | /// On success, this function returns the sector number of the next free sector 733 | /// after the entry is written. 734 | /// 735 | /// Sectors are written strictly in-order, so on a write failure, you'll be left 736 | /// in one of three possible states: 737 | /// 738 | /// 1. The failure prevented the header from being written. The contents of the 739 | /// log are unchanged. 740 | /// 2. The failure happened somewhere before the trailer was finished and the 741 | /// log now ends in an incomplete entry. 742 | /// 3. The failure happened during write of the final sector, but did not 743 | /// prevent the sector from being written; the log now ends in your complete 744 | /// entry as though no error occurred. (We allow for this case to make flash 745 | /// device implementation slightly easier.) 746 | /// 747 | /// You can use `check_entry` if you need to distinguish, or treat any error as 748 | /// potentially incomplete and attempt to repair the log. 749 | pub(crate) fn write_entry( 750 | flash: &mut F, 751 | buffer: &mut F::Sector, 752 | current: Space, 753 | start_sector: u32, 754 | pieces: &[&[u8]], 755 | subtrailer: &[u8], 756 | trailer: &[u8], 757 | ) -> Result> { 758 | // Work out the length without padding. 759 | let total_length = pieces.iter().map(|p| p.len()).sum::() 760 | + subtrailer.len() 761 | + trailer.len(); 762 | let total_length = u32::try_from(total_length).unwrap(); 763 | // Convert to sectors, which has the effect of including the padding. 764 | let total_sectors = bytes_to_sectors::(total_length); 765 | // Detect out-of-space. 766 | if flash.sectors_per_space() - start_sector < total_sectors { 767 | return Err(WriteError::NoSpace); 768 | } 769 | 770 | // Set up loop variables. Management of these variables is a little subtle 771 | // since we don't require `pieces` to be sector-aligned. 772 | let mut sector = start_sector; 773 | let mut data = buffer.borrow_mut(); 774 | 775 | // Gather all the `pieces` and write them contiguously. 776 | for mut piece in pieces.iter().cloned() { 777 | // Loop because a piece may be larger than a sector. 778 | while !piece.is_empty() { 779 | // Break both the piece and the remaining sector buffer into the 780 | // largest common chunk (piece0/data0) and the remainder 781 | // (piece1/data1). 782 | let n = usize::min(piece.len(), data.len()); 783 | let (piece0, piece1) = piece.split_at(n); 784 | let (data0, data1) = data.split_at_mut(n); 785 | 786 | data0.copy_from_slice(piece0); 787 | 788 | piece = piece1; 789 | data = data1; 790 | 791 | // Check for entirely filled sector buffer; write it out and reset 792 | // it. 793 | if data.is_empty() { 794 | drop(data); 795 | 796 | flash.program_sector(current, sector, buffer)?; 797 | sector += 1; 798 | 799 | data = buffer.borrow_mut(); 800 | } 801 | } 802 | } 803 | 804 | // Check how much space we have left in the final sector. 805 | if data.len() < subtrailer.len() + trailer.len() { 806 | // We can't fit the full trailer into the final sector, so we have to 807 | // burn a sector on the trailer by flushing this sector and starting a 808 | // new one. 809 | // 810 | // Zero-fill the remainder of the flushed sector to avoid writing 811 | // arbitrary goo. 812 | data.fill(0); 813 | drop(data); 814 | 815 | flash.program_sector(current, sector, buffer)?; 816 | sector += 1; 817 | 818 | data = buffer.borrow_mut(); 819 | } 820 | 821 | // Fill in the trailer. 822 | { 823 | // Split remaining sector tail into unused area, which will be filled 824 | // with padding, and the trailer part. 825 | let dl = data.len(); 826 | let unused = dl - trailer.len() - subtrailer.len(); 827 | let (pad, tail) = data.split_at_mut(unused); 828 | pad.fill(0); 829 | 830 | // Now split the tail into subtrailer and trailer regions and fill them 831 | // in. 832 | let (sub, tail) = tail.split_at_mut(subtrailer.len()); 833 | sub.copy_from_slice(subtrailer); 834 | tail.copy_from_slice(trailer); 835 | } 836 | 837 | // Write the final sector. 838 | drop(data); 839 | flash.program_sector(current, sector, buffer)?; 840 | 841 | Ok(sector + 1) 842 | } 843 | 844 | /// Things that can go wrong while writing an entry. 845 | #[derive(Copy, Clone, Debug, Eq, PartialEq)] 846 | pub enum WriteError { 847 | /// There is not enough space on the flash device to add this entry to the 848 | /// log. 849 | NoSpace, 850 | /// An underlying flash access error occurred. 851 | Flash(E), 852 | } 853 | 854 | impl From for WriteError { 855 | fn from(e: E) -> Self { 856 | Self::Flash(e) 857 | } 858 | } 859 | 860 | 861 | ////////////////////////////////////////////////////////////////////////////// 862 | // Data entry implementation. 863 | 864 | /// Searches backwards through the log for a data entry matching `key`. 865 | /// 866 | /// `start_sector` is the index of the sector _just past_ the valid data you 867 | /// wish to search. The search will start at `start_sector - 1` and run back to 868 | /// the space header, stopping prematurely if a matching entry is found. 869 | /// 870 | /// Return values: 871 | /// 872 | /// - `Ok(Some(n))` indicates that a matching entry was found, and its header 873 | /// sector is at index `n`. 874 | /// - `Ok(None)` indicates that there is no matching entry in the log, either 875 | /// because it does not exist, or because it has been deleted. 876 | /// - `Err(e)` if the search could not be completed because a corrupt entry was 877 | /// discovered, or if the underlying flash layer returns an error. 878 | pub(crate) fn seek_kv_backwards( 879 | flash: &F, 880 | buffer: &mut F::Sector, 881 | current: Space, 882 | start_sector: u32, 883 | key: &[u8], 884 | ) -> Result, ReadError> { 885 | let key_len = u32::try_from(key.len()) 886 | .expect("key too long"); 887 | let key_hash = hash_key(key); 888 | 889 | seek_backwards( 890 | flash, 891 | buffer, 892 | current, 893 | start_sector, 894 | |flash, buffer, index, ksub| { 895 | match ksub { 896 | KnownSubMetas::Data(sub) | KnownSubMetas::Delete(sub) => { 897 | // For these types, we want to check the key. 898 | if sub.key_hash.get() == key_hash 899 | && sub.key_length.get() == key_len 900 | { 901 | // A potential match! We need to compare the first 902 | // `key.len()` bytes after the header. 903 | let meta_bytes = size_of::() 904 | + size_of::(); 905 | let key_eq = flash.compare_contents( 906 | current, 907 | buffer, 908 | index, 909 | meta_bytes as u32, 910 | key, 911 | )?; 912 | if key_eq { 913 | // Now, the difference between Data and Delete comes 914 | // into play. 915 | if matches!(ksub, KnownSubMetas::Data(_)) { 916 | return Ok(EntryDecision::Accept) 917 | } else { 918 | // A delete entry causes us to early-abort with no 919 | // match: 920 | return Ok(EntryDecision::Abort) 921 | }; 922 | } 923 | } 924 | } 925 | _ => { 926 | // Everything else, we'll just skip. 927 | } 928 | } 929 | Ok(EntryDecision::Ignore) 930 | }, 931 | ) 932 | } 933 | 934 | /// Writes a new Data entry for `key` storing `value` and superceding any 935 | /// previous entry for `key`. 936 | /// 937 | /// The entry will be written starting at `start_sector` (inclusive) into 938 | /// `flash` in space `current`. `buffer` will be scribbled on by the 939 | /// implementation. 940 | pub(crate) fn write_kv( 941 | flash: &mut F, 942 | buffer: &mut F::Sector, 943 | current: Space, 944 | start_sector: u32, 945 | key: &[u8], 946 | value: &[u8], 947 | ) -> Result> { 948 | // Paranoid parameter validation: are any of these too large for a u32? On a 949 | // 32-bit platform this can't happen. 950 | let key_len = u32::try_from(key.len()) 951 | .expect("key too long"); 952 | let value_len = u32::try_from(value.len()) 953 | .expect("value too long"); 954 | let contents_length = key_len.checked_add(value_len) 955 | .expect("key+value too long"); 956 | 957 | // Construct header/trailer. The same bits do double-duty in both places. 958 | let meta = EntryMeta { 959 | magic: EntryMeta::EXPECTED_MAGIC.into(), 960 | subtype: KnownSubtypes::Data as u8, 961 | sub_bytes: DataSubMeta::SUB_BYTES, 962 | contents_length: contents_length.into(), 963 | }; 964 | let submeta = DataSubMeta { 965 | key_length: key_len.into(), 966 | key_hash: hash_key(key).into(), 967 | }; 968 | 969 | // Go! 970 | write_entry( 971 | flash, 972 | buffer, 973 | current, 974 | start_sector, 975 | &[ 976 | meta.as_bytes(), 977 | submeta.as_bytes(), 978 | key, 979 | value, 980 | ], 981 | // Note that order is reversed here: 982 | submeta.as_bytes(), 983 | meta.as_bytes(), 984 | ) 985 | } 986 | 987 | ////////////////////////////////////////////////////////////////////////////// 988 | // Space formatting and checking. 989 | 990 | /// Creates a new empty log in device `flash` and space `current`. 991 | /// 992 | /// This requires that the space has been erased. If it has not been erased, 993 | /// this will fail with `FormatError::NeedsErase`. 994 | /// 995 | /// Given an erased space, formatting is simply a matter of writing a valid 996 | /// space header, so this only winds up needing one sector write. 997 | pub fn format( 998 | flash: &mut F, 999 | buffer: &mut F::Sector, 1000 | current: Space, 1001 | initial_generation: u32, 1002 | ) -> Result<(), FormatError> { 1003 | // Check that the target space is entirely erased. 1004 | for s in 0..flash.sectors_per_space() { 1005 | if !flash.can_program_sector(current, s)? { 1006 | return Err(FormatError::NeedsErase); 1007 | } 1008 | } 1009 | 1010 | // Write the header. 1011 | let (header, tail) = cast_prefix_mut::((*buffer).borrow_mut()); 1012 | *header = SpaceHeader { 1013 | magic: SpaceHeader::EXPECTED_MAGIC.into(), 1014 | generation: initial_generation.into(), 1015 | l2_sector_size: size_of::().trailing_zeros() as u8, 1016 | pad: [0; 3], 1017 | crc: 0.into(), 1018 | }; 1019 | header.crc = header.expected_crc().into(); 1020 | tail.fill(0); 1021 | 1022 | flash.program_sector(current, 0, buffer)?; 1023 | 1024 | Ok(()) 1025 | } 1026 | 1027 | /// Things that can go wrong with `format`. 1028 | #[derive(Copy, Clone, Debug, Eq, PartialEq)] 1029 | pub enum FormatError { 1030 | NeedsErase, 1031 | Flash(E), 1032 | } 1033 | 1034 | impl From for FormatError { 1035 | fn from(e: E) -> Self { 1036 | Self::Flash(e) 1037 | } 1038 | } 1039 | 1040 | /// Scans the contents of `flash` to determine the state of space `current`. 1041 | /// 1042 | /// `buffer` will be scribbled on by the implementation as scratch space. 1043 | /// 1044 | /// This function will fail (return `Err`) only if it is unable to scan the 1045 | /// space. An `Ok` return means that the check finished, _not_ that the space 1046 | /// contains a valid log. See the `CheckResult` type for more details. 1047 | pub fn check( 1048 | flash: &mut F, 1049 | buffer: &mut F::Sector, 1050 | current: Space, 1051 | ) -> Result { 1052 | let sector_count = flash.sectors_per_space(); 1053 | 1054 | // Check if the first sector is programmable and use that as a proxy for 1055 | // "erased." Our space header contents are (deliberately) distinguishable 1056 | // from typical erased flash, even for devices that allow us to read back 1057 | // erased sectors, so this should be universal. 1058 | if flash.can_program_sector(current, 0)? { 1059 | // Well, there's no valid store here... let's distinguish between full 1060 | // and partial erase to help our caller out. 1061 | for sector in 1..sector_count { 1062 | if !flash.can_program_sector(current, sector)? { 1063 | // There's at least one un-erased sector; we're going to have to 1064 | // erase this space before we can use it. 1065 | return Ok(CheckResult::Bad(CheckError::PartiallyErased)); 1066 | } 1067 | } 1068 | 1069 | // This space is empty and could be used as an idle space. 1070 | return Ok(CheckResult::Bad(CheckError::Erased)); 1071 | } 1072 | 1073 | // Check the space header! 1074 | flash.read_sector(current, 0, buffer)?; 1075 | let (space_header, _) = cast_prefix::((*buffer).borrow()); 1076 | 1077 | if !space_header.check() { 1078 | return Ok(CheckResult::Bad(CheckError::BadSpaceHeader)); 1079 | } 1080 | if 1 << space_header.l2_sector_size != size_of::() { 1081 | return Ok(CheckResult::Bad(CheckError::WrongSectorSize)); 1082 | } 1083 | 1084 | // Copy the generation out so that we can let go of buffer0. 1085 | let generation = space_header.generation.get(); 1086 | drop(space_header); 1087 | 1088 | let mut sector = Constants::::HEADER_SECTORS; 1089 | let mut incomplete_write_end = None; 1090 | 1091 | // Work over the remaining sectors in the (alleged) log, treating them as 1092 | // log entries. 1093 | while sector < sector_count { 1094 | let r = check_entry(flash, buffer, current, sector)?; 1095 | 1096 | match r { 1097 | CheckEntryResult::ChecksPassed(next) => { 1098 | // Advance the sector pointer and _keep going!_ 1099 | sector = next; 1100 | } 1101 | 1102 | CheckEntryResult::HeadErased => { 1103 | // This is probably the end of the log in this space. Don't 1104 | // advance the sector pointer. 1105 | break; 1106 | } 1107 | CheckEntryResult::IncompleteWrite(next) => { 1108 | // Note that we need repair. DO NOT advance the sector pointer! 1109 | // We will leave the incomplete write just past the end of the 1110 | // valid log so that reads don't have to think about it. 1111 | incomplete_write_end = Some(next); 1112 | break; 1113 | } 1114 | 1115 | CheckEntryResult::HeadCorrupt 1116 | | CheckEntryResult::HeadTailMismatch 1117 | | CheckEntryResult::TailCorrupt => { 1118 | return Ok(CheckResult::Bad(CheckError::Corrupt(sector))); 1119 | } 1120 | 1121 | CheckEntryResult::PartiallyErased => { 1122 | return Ok(CheckResult::Bad(CheckError::UnprogrammedData(sector))); 1123 | } 1124 | } 1125 | } 1126 | 1127 | // Now, scan the rest of the space to see if we've really found the end of 1128 | // the log, i.e. whether the rest of the space is erased. 1129 | // 1130 | // We want to start the scan at the end of the log, _or_ just past an 1131 | // incomplete write, if one was found. 1132 | let mut tail_erased = true; 1133 | let scan_start = incomplete_write_end.unwrap_or(sector); 1134 | for s in scan_start..sector_count { 1135 | if !flash.can_program_sector(current, s)? { 1136 | // Welp, there's at least one unerased turd in the space past the 1137 | // end of the log, which means we can't mount this writable yet. 1138 | tail_erased = false; 1139 | break; 1140 | } 1141 | } 1142 | 1143 | // Return our findings. 1144 | Ok(CheckResult::ValidLog { 1145 | generation, 1146 | end: sector, 1147 | incomplete_write: incomplete_write_end.is_some(), 1148 | tail_erased, 1149 | }) 1150 | } 1151 | 1152 | /// Result of completing the `check` process. 1153 | #[derive(Copy, Clone, Debug, Eq, PartialEq)] 1154 | pub enum CheckResult { 1155 | /// There is no valid log in this space, for the reason documented in 1156 | /// `CheckError`. 1157 | Bad(CheckError), 1158 | 1159 | /// A valid data store was found, with a valid header and some number of 1160 | /// valid entries running up to sector index `end` (which is one past the 1161 | /// end of valid data). This indicates that the store can be mounted at 1162 | /// least read-only. 1163 | /// 1164 | /// To mount read-write, `tail_erased` must be true, and `incomplete_write` 1165 | /// must be false. Or, repair action must be taken. 1166 | ValidLog { 1167 | /// Generation number found in the space header. 1168 | generation: u32, 1169 | /// Index of first non-log sector in the space. 1170 | end: u32, 1171 | /// Whether all sectors in the space starting with `end` have been 1172 | /// erased (`true`) or if some haven't (`false`). 1173 | tail_erased: bool, 1174 | incomplete_write: bool, 1175 | } 1176 | } 1177 | 1178 | /// Errors reported by the `check` process. 1179 | #[derive(Copy, Clone, Debug, Eq, PartialEq)] 1180 | pub enum CheckError { 1181 | /// While there is no log in the space, the space is entirely erased. It can 1182 | /// be used as the idle space with no further work. 1183 | Erased, 1184 | /// Not all of the sectors in the space have been erased, but enough have 1185 | /// been erased to render the data structure useless. It must be fully 1186 | /// erased before it can be used. 1187 | PartiallyErased, 1188 | /// The initial sector of the space is programmed, but can't be validated as 1189 | /// a space header. 1190 | BadSpaceHeader, 1191 | /// The space begins with what appears to be a valid space header, but the 1192 | /// recorded sector size is wrong for this flash device. 1193 | WrongSectorSize, 1194 | 1195 | /// The store is corrupt starting with the "entry" at the given sector 1196 | /// index. You may be able to successfully read a prior version of the store 1197 | /// by treating this as the end of the log. 1198 | Corrupt(u32), 1199 | 1200 | /// An entry at the given sector number contains at least one unprogrammed 1201 | /// sector, making it unsafe to read. You may be able to successfully read a 1202 | /// prior version of the store by treating this as the end of the log. 1203 | UnprogrammedData(u32), 1204 | } 1205 | 1206 | /// Reads the entry starting at `head_sector` and checks that it's valid. 1207 | /// 1208 | /// Note that success (`Ok`) of this function just means that the check 1209 | /// completed without flash access errors. It does _not_ mean the entry is fine. 1210 | /// 1211 | /// There are three classes of `Ok` results from this operation. 1212 | /// 1213 | /// Results that indicate everything is fine: 1214 | /// - `ChecksPassed(next)` indicates a valid and matching head/tail, that all 1215 | /// data sectors between are programmed with _something,_ and the next entry 1216 | /// should begin at `next`. 1217 | /// 1218 | /// Results that indicate a problem with _this_ entry but don't necessarily 1219 | /// imply corruption of the overall structure: 1220 | /// 1221 | /// - `HeadErased` probably just indicates the end of the log. 1222 | /// - `IncompleteWrite` probably means the end of the log plus a power loss. 1223 | /// 1224 | /// Finally, the rest of the `CheckEntryResult` variants indicate entry 1225 | /// corruption. 1226 | pub(crate) fn check_entry( 1227 | flash: &mut F, 1228 | buffer: &mut F::Sector, 1229 | current: Space, 1230 | head_sector: u32, 1231 | ) -> Result { 1232 | // Check if the first sector is written. 1233 | if flash.can_program_sector(current, head_sector)? { 1234 | return Ok(CheckEntryResult::HeadErased); 1235 | } 1236 | // Attempt to read and parse the header. 1237 | let head_info = match read_entry_from_head(flash, buffer, current, head_sector) { 1238 | Err(ReadError::Flash(e)) => return Err(e), 1239 | Err(_) => return Ok(CheckEntryResult::HeadCorrupt), 1240 | Ok(entry) => entry, 1241 | }; 1242 | 1243 | // Using the length information from the header, locate the tail sector and 1244 | // see if it's programmed. Unprogrammed tail sector indicates an incomplete 1245 | // write. 1246 | if flash.can_program_sector(current, head_info.next_sector - 1)? { 1247 | return Ok(CheckEntryResult::IncompleteWrite(head_info.next_sector)); 1248 | } 1249 | 1250 | // Get ready to read the tail sector by freeing the buffer. 1251 | let next_entry = head_info.next_sector; 1252 | let head_meta = *head_info.meta; 1253 | drop(head_info); 1254 | 1255 | // Attempt to read and parse the trailer. 1256 | let tail_info = match read_entry_from_tail(flash, buffer, current, next_entry) { 1257 | Err(ReadError::Flash(e)) => return Err(e), 1258 | Err(_) => return Ok(CheckEntryResult::TailCorrupt), 1259 | Ok(entry) => entry, 1260 | }; 1261 | 1262 | // Make sure the basics match the header. 1263 | if tail_info.next_sector != head_sector 1264 | || tail_info.meta.contents_length != head_meta.contents_length 1265 | { 1266 | return Ok(CheckEntryResult::HeadTailMismatch); 1267 | } 1268 | 1269 | // We permit only one kind of mismatch between header and trailer: the 1270 | // trailer may be marked Aborted, regardless of how the header is marked. In 1271 | // this case we still require the other fields to match. 1272 | if tail_info.meta.subtype != KnownSubtypes::Aborted as u8 1273 | && (tail_info.meta.subtype != head_meta.subtype 1274 | || tail_info.meta.sub_bytes != head_meta.sub_bytes) 1275 | { 1276 | return Ok(CheckEntryResult::HeadTailMismatch); 1277 | } 1278 | 1279 | if tail_info.meta.subtype != KnownSubtypes::Aborted as u8 { 1280 | // For any other subtype we require the entry's sectors to be totally 1281 | // readable, which typically means programmed -- though some flashes 1282 | // will expose unprogrammed sectors as all FF, which we tolerate because 1283 | // it won't introduce flash read errors into log operations. 1284 | for s in head_sector + 1..next_entry { 1285 | if !flash.can_read_sector(current, s)? { 1286 | return Ok(CheckEntryResult::PartiallyErased); 1287 | } 1288 | } 1289 | } 1290 | 1291 | Ok(CheckEntryResult::ChecksPassed(next_entry)) 1292 | } 1293 | 1294 | /// Possible successful results of `check_entry`. 1295 | #[derive(Copy, Clone, Debug)] 1296 | pub enum CheckEntryResult { 1297 | /// The head sector for this entry is blank. This probably means you've 1298 | /// reached the end of the log. 1299 | HeadErased, 1300 | 1301 | /// The head sector for this entry is programmed and looks reasonable, but 1302 | /// the tail sector is blank. This suggests an entry that was being written 1303 | /// when we lost power or crashed. This condition is recoverable with 1304 | /// effort. 1305 | /// 1306 | /// The given sector index is the first sector _after_ the unprogrammed tial 1307 | /// section. 1308 | IncompleteWrite(u32), 1309 | 1310 | /// The head sector for this entry is corrupt or contains arbitrary data. 1311 | HeadCorrupt, 1312 | /// The head and tail sectors for this entry are both programmed, but do not 1313 | /// match one another, suggesting data corruption. 1314 | HeadTailMismatch, 1315 | /// The tail sector is programmed with invalid data, suggesting data 1316 | /// corruption. 1317 | TailCorrupt, 1318 | /// The head and tail sectors of this entry are valid, but at least one of 1319 | /// the data sectors between is blank. 1320 | PartiallyErased, 1321 | 1322 | /// The entry appears valid, and the next entry will be at the given sector 1323 | /// index. 1324 | ChecksPassed(u32), 1325 | } 1326 | 1327 | ////////////////////////////////////////////////////////////////////////////// 1328 | // Space evacuation / garbage collection. 1329 | 1330 | /// Processes the log in `from_space`, identifies the subset of its entries that 1331 | /// have not been superceded by later entries, and copies them into the opposite 1332 | /// space on the same flash device. 1333 | /// 1334 | /// The relative order of the entries is _not_ preserved, to simplify the 1335 | /// implementation. 1336 | /// 1337 | /// `watermark` is the number of sectors in `from_space` that have been written, 1338 | /// i.e. the length of the log including the space header. 1339 | /// 1340 | /// `buffer0` and `buffer1` are temporary storage space used by the 1341 | /// implementation, and their contents will be scribbled upon. 1342 | /// 1343 | /// On success, the space opposite to `from_space` contains a log that is 1344 | /// semantically equivalent to the one in `from_space`, but with all redundant 1345 | /// entries removed, and the generation counter advanced by 1. This function 1346 | /// returns the watermark for that new log. 1347 | /// 1348 | /// Failures may be `ReadError` indicating a problem processing the `from_space` 1349 | /// log, or errors from the underlying flash device indicating problems writing 1350 | /// the new log. 1351 | /// 1352 | /// This function takes care to write the new space header _last,_ ensuring that 1353 | /// the destination space will only `check` as a valid log if all writes have 1354 | /// completed. Any failure should leave the destination space in the `Erased` or 1355 | /// `PartiallyErased` state where it can be erased and attempted again if 1356 | /// necessary. 1357 | /// 1358 | /// TODO: this implementation is currently O(n^2) as it doesn't use any kind of 1359 | /// cache to track which entries have been evacuated, requiring a scan of 1360 | /// to-space any time a data entry is copied. This could be fixed by giving it 1361 | /// more RAM to track copies. 1362 | pub fn evacuate( 1363 | flash: &mut F, 1364 | buffer0: &mut F::Sector, 1365 | buffer1: &mut F::Sector, 1366 | from_space: Space, 1367 | watermark: u32, 1368 | ) -> Result> { 1369 | flash.read_sector(from_space, 0, buffer0)?; 1370 | let (space_header, _) = cast_prefix::((*buffer0).borrow()); 1371 | let from_generation = space_header.generation.get(); 1372 | let to_space = from_space.other(); 1373 | 1374 | // Read every entry in from-space starting at the most recent and working 1375 | // back, to ensure that we see any superceding entries before their 1376 | // predecessors. 1377 | let header_sectors = Constants::::HEADER_SECTORS; 1378 | let mut from_sector = watermark; 1379 | let mut to_sector = header_sectors; 1380 | while from_sector > header_sectors { 1381 | let entry = read_entry_from_tail(flash, buffer0, from_space, from_sector)?; 1382 | 1383 | let meta_bytes = entry.meta.meta_length(); 1384 | let entry_sectors = entry.meta.entry_sectors::(); 1385 | 1386 | let head_sector = entry.next_sector; 1387 | 1388 | let subtype = KnownSubtypes::from_u8(entry.meta.subtype); 1389 | let should_copy = match subtype { 1390 | Some(KnownSubtypes::Data) | Some(KnownSubtypes::Delete) => { 1391 | // These get copied over but only if they're not superceded. 1392 | // This means we need to determine whether an entry for this key 1393 | // has already been evacuated into to-space. 1394 | // 1395 | // We will probably want to add some sort of cache for this in 1396 | // the future, but since RAM usage will be limited, we will 1397 | // probably always need to be able to fall back to scanning the 1398 | // in-progress to-space data structure for a match. 1399 | 1400 | let (_, subtrailer) = cast_suffix::(entry.submeta); 1401 | let key_hash = subtrailer.key_hash.get(); 1402 | let key_len = subtrailer.key_length.get(); 1403 | 1404 | let pred = seek_backwards( 1405 | flash, 1406 | buffer0, 1407 | to_space, 1408 | to_sector, 1409 | |flash, buf, s, sub| { 1410 | match sub { 1411 | KnownSubMetas::Data(sub) | KnownSubMetas::Delete(sub) => { 1412 | if sub.key_hash.get() == key_hash 1413 | && sub.key_length.get() == key_len 1414 | { 1415 | let key_eq = flash.compare_internal( 1416 | from_space, 1417 | head_sector, 1418 | to_space, 1419 | s, 1420 | key_len + meta_bytes as u32, 1421 | buf, 1422 | buffer1, 1423 | )?; 1424 | 1425 | if key_eq { 1426 | // Match! 1427 | return Ok(EntryDecision::Accept); 1428 | } 1429 | } 1430 | } 1431 | _ => (), 1432 | } 1433 | Ok(EntryDecision::Ignore) 1434 | } 1435 | )?; 1436 | 1437 | if pred.is_some() { 1438 | false 1439 | } else { 1440 | true 1441 | } 1442 | } 1443 | Some(KnownSubtypes::Aborted) => { 1444 | // Not copied, no effect on store. 1445 | false 1446 | } 1447 | None => { 1448 | // We can't safely copy entries we don't understand, because 1449 | // they may contain internal references to other parts of the 1450 | // data structure. 1451 | panic!() 1452 | } 1453 | }; 1454 | 1455 | if should_copy { 1456 | flash.copy_across( 1457 | from_space, 1458 | head_sector, 1459 | to_sector, 1460 | entry_sectors, 1461 | buffer0, 1462 | )?; 1463 | to_sector += entry_sectors; 1464 | } 1465 | 1466 | from_sector = head_sector; 1467 | } 1468 | 1469 | let b0 = buffer0.borrow_mut(); 1470 | let (to_header, btail) = cast_prefix_mut(b0); 1471 | *to_header = SpaceHeader { 1472 | magic: SpaceHeader::EXPECTED_MAGIC.into(), 1473 | generation: from_generation.wrapping_add(1).into(), 1474 | l2_sector_size: size_of::().trailing_zeros() as u8, 1475 | pad: [0; 3], 1476 | crc: 0.into(), 1477 | }; 1478 | to_header.crc = to_header.expected_crc().into(); 1479 | btail.fill(0); 1480 | 1481 | flash.program_sector(to_space, 0, buffer0)?; 1482 | 1483 | Ok(to_sector) 1484 | } 1485 | 1486 | #[cfg(test)] 1487 | mod tests { 1488 | use super::*; 1489 | 1490 | pub struct FakeFlash { 1491 | sectors: [Vec>; 2], 1492 | } 1493 | 1494 | impl FakeFlash { 1495 | pub fn new(sector_count: usize) -> Self { 1496 | Self { 1497 | sectors: [ 1498 | vec![None; sector_count], 1499 | vec![None; sector_count], 1500 | ], 1501 | } 1502 | } 1503 | 1504 | pub fn sectors(&self, space: Space) -> &[Option<[u8; N]>] { 1505 | match space { 1506 | Space::Zero => &self.sectors[0], 1507 | Space::One => &self.sectors[1], 1508 | } 1509 | } 1510 | 1511 | pub fn sectors_mut(&mut self, space: Space) -> &mut [Option<[u8; N]>] { 1512 | match space { 1513 | Space::Zero => &mut self.sectors[0], 1514 | Space::One => &mut self.sectors[1], 1515 | } 1516 | } 1517 | 1518 | pub fn erase_sector(&mut self, space: Space, index: u32) { 1519 | self.sectors_mut(space)[index as usize] = None; 1520 | } 1521 | } 1522 | 1523 | #[derive(Copy, Clone, Debug, Eq, PartialEq)] 1524 | pub enum FakeFlashError {} 1525 | 1526 | impl Flash for FakeFlash { 1527 | type Sector = [u8; N]; 1528 | type Error = FakeFlashError; 1529 | 1530 | fn sectors_per_space(&self) -> u32 { 1531 | u32::try_from(self.sectors[0].len()).unwrap() 1532 | } 1533 | 1534 | fn read_sector(&self, space: Space, index: u32, dest: &mut Self::Sector) -> Result<(), Self::Error> { 1535 | let sectors = self.sectors(space); 1536 | *dest = sectors[index as usize] 1537 | .expect("read of unprogrammed sector"); 1538 | Ok(()) 1539 | } 1540 | 1541 | fn can_program_sector(&self, space: Space, index: u32) -> Result { 1542 | Ok(self.sectors(space)[index as usize].is_none()) 1543 | } 1544 | 1545 | fn can_read_sector(&self, space: Space, index: u32) -> Result { 1546 | Ok(self.sectors(space)[index as usize].is_some()) 1547 | } 1548 | 1549 | fn program_sector(&mut self, space: Space, index: u32, data: &Self::Sector) -> Result<(), Self::Error> { 1550 | let s = &mut self.sectors_mut(space)[index as usize]; 1551 | if s.is_some() { 1552 | panic!("attempt to double-program sector {index}"); 1553 | } 1554 | *s = Some(*data); 1555 | Ok(()) 1556 | } 1557 | 1558 | fn erase_space(&mut self, space: Space) -> Result<(), Self::Error> { 1559 | self.sectors_mut(space).fill(None); 1560 | Ok(()) 1561 | } 1562 | } 1563 | 1564 | #[test] 1565 | fn test_write_entry_single_sector_no_pad() { 1566 | let mut flash = FakeFlash::<16>::new(64); 1567 | let mut buffer = [0u8; 16]; 1568 | 1569 | let next_free = write_entry( 1570 | &mut flash, 1571 | &mut buffer, 1572 | Space::Zero, 1573 | 1, 1574 | &[ 1575 | &[0, 1], 1576 | &[2, 3], 1577 | &[4, 5, 6, 7], 1578 | ], 1579 | &[8, 9, 10, 11], 1580 | &[12, 13, 14, 15], 1581 | ).map_err(|_| ()).unwrap(); 1582 | 1583 | assert_eq!(next_free, 2); 1584 | 1585 | let mut expected = [0; 16]; 1586 | for (i, byte) in expected.iter_mut().enumerate() { 1587 | *byte = i as u8; 1588 | } 1589 | 1590 | assert_eq!(&flash.sectors(Space::Zero)[1], &Some(expected)); 1591 | } 1592 | 1593 | #[test] 1594 | fn test_write_entry_single_sector_pad() { 1595 | let mut flash = FakeFlash::<16>::new(64); 1596 | let mut buffer = [0u8; 16]; 1597 | 1598 | let next_free = write_entry( 1599 | &mut flash, 1600 | &mut buffer, 1601 | Space::Zero, 1602 | 1, 1603 | &[ 1604 | &[0, 1], 1605 | &[2, 3], 1606 | &[4, 5], 1607 | ], 1608 | &[8, 9, 10, 11], 1609 | &[12, 13, 14, 15], 1610 | ).map_err(|_| ()).unwrap(); 1611 | 1612 | assert_eq!(next_free, 2); 1613 | 1614 | let mut expected = [0; 16]; 1615 | for (i, byte) in expected.iter_mut().enumerate() { 1616 | *byte = i as u8; 1617 | } 1618 | // fill in padding 1619 | expected[6] = 0; 1620 | expected[7] = 0; 1621 | 1622 | assert_eq!(&flash.sectors(Space::Zero)[1], &Some(expected)); 1623 | } 1624 | 1625 | #[test] 1626 | fn test_empty_entry_zero_fill() { 1627 | let mut flash = FakeFlash::<16>::new(64); 1628 | let mut buffer = [0u8; 16]; 1629 | 1630 | let next_free = write_entry( 1631 | &mut flash, 1632 | &mut buffer, 1633 | Space::Zero, 1634 | 1, 1635 | &[], 1636 | &[], 1637 | &[], 1638 | ).map_err(|_| ()).unwrap(); 1639 | 1640 | // Should still burn a sector. 1641 | assert_eq!(next_free, 2); 1642 | 1643 | assert_eq!(&flash.sectors(Space::Zero)[1], &Some([0; 16])); 1644 | } 1645 | 1646 | #[test] 1647 | fn test_write_entry_two_sector_short_data() { 1648 | let mut flash = FakeFlash::<16>::new(64); 1649 | let mut buffer = [0u8; 16]; 1650 | 1651 | let header = [1; 4]; 1652 | let subheader = [2; 4]; 1653 | // Just enough data to evict the subtrailer/trailer from the first 1654 | // sector, but not enough to fill the sector. This tests trailing 1655 | // padding of sectors at the end of data. 1656 | let data = [0xAA; 4]; 1657 | let subtrailer = [3; 4]; 1658 | let trailer = [4; 4]; 1659 | 1660 | let next_free = write_entry( 1661 | &mut flash, 1662 | &mut buffer, 1663 | Space::Zero, 1664 | 1, 1665 | &[ 1666 | &header, 1667 | &subheader, 1668 | &data, 1669 | ], 1670 | &subtrailer, 1671 | &trailer, 1672 | ).map_err(|_| ()).unwrap(); 1673 | 1674 | // Should burn two sectors 1675 | assert_eq!(next_free, 3); 1676 | 1677 | // Header plus data with trailing padding: 1678 | assert_eq!(&flash.sectors(Space::Zero)[1], &Some([ 1679 | 1, 1, 1, 1, 2, 2, 2, 2, 1680 | 0xAA, 0xAA, 0xAA, 0xAA, 0, 0, 0, 0, 1681 | ])); 1682 | // Second sector is all padding plus trailer: 1683 | assert_eq!(&flash.sectors(Space::Zero)[2], &Some([ 1684 | 0, 0, 0, 0, 0, 0, 0, 0, 1685 | 3, 3, 3, 3, 4, 4, 4, 4, 1686 | ])); 1687 | } 1688 | 1689 | #[test] 1690 | fn test_write_entry_two_sector_longer_data() { 1691 | let mut flash = FakeFlash::<16>::new(64); 1692 | let mut buffer = [0u8; 16]; 1693 | 1694 | let header = [1; 4]; 1695 | let subheader = [2; 4]; 1696 | let data = [0xAA; 14]; 1697 | let subtrailer = [3; 4]; 1698 | let trailer = [4; 4]; 1699 | 1700 | let next_free = write_entry( 1701 | &mut flash, 1702 | &mut buffer, 1703 | Space::Zero, 1704 | 1, 1705 | &[ 1706 | &header, 1707 | &subheader, 1708 | &data, 1709 | ], 1710 | &subtrailer, 1711 | &trailer, 1712 | ).map_err(|_| ()).unwrap(); 1713 | 1714 | // Should burn two sectors 1715 | assert_eq!(next_free, 3); 1716 | 1717 | assert_eq!(&flash.sectors(Space::Zero)[1], &Some([ 1718 | 1, 1, 1, 1, 2, 2, 2, 2, 1719 | 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 1720 | ])); 1721 | // Test second sector with embedded padding: 1722 | assert_eq!(&flash.sectors(Space::Zero)[2], &Some([ 1723 | 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 1724 | 0, 0, 1725 | 3, 3, 3, 3, 4, 4, 4, 4, 1726 | ])); 1727 | } 1728 | 1729 | #[test] 1730 | fn test_read_entry_from_tail_single() { 1731 | let mut flash = FakeFlash::<32>::new(64); 1732 | // sector 0 is the space header. 1733 | // sector 1 will be both header and trailer: 1734 | // 00 magic, subtype, sub_bytes 1735 | // 04 contents_length 1736 | // 08 fake subheader 1737 | // 1738 | // 0c data 1739 | // 10 data 1740 | // 1741 | // 14 fake subheader 1742 | // 18 magic, subtype, sub_bytes 1743 | // 1C contents_length 1744 | let mut sec1 = [0; 32]; 1745 | sec1[0..=1].copy_from_slice(&EntryMeta::EXPECTED_MAGIC.to_le_bytes()); 1746 | sec1[2] = 0xAA; 1747 | sec1[3] = 4; 1748 | sec1[4..=7].copy_from_slice(&8_u32.to_le_bytes()); 1749 | 1750 | sec1[8..=0xB].copy_from_slice(&0xDEAD_BEEF_u32.to_le_bytes()); 1751 | 1752 | sec1[0x0C..=0x13].fill(0xFF); 1753 | 1754 | sec1[0x14..=0x17].copy_from_slice(&0xDEAD_BEEF_u32.to_le_bytes()); 1755 | 1756 | sec1[0x18..=0x19].copy_from_slice(&EntryMeta::EXPECTED_MAGIC.to_le_bytes()); 1757 | sec1[0x1A] = 0xAA; 1758 | sec1[0x1B] = 4; 1759 | sec1[0x1C..=0x1F].copy_from_slice(&8_u32.to_le_bytes()); 1760 | flash.program_sector(Space::Zero, 1, &sec1).unwrap(); 1761 | 1762 | let mut buffer = [0u8; 32]; 1763 | 1764 | // We request to read backwards starting at sector 2, not sector 1, 1765 | // because that's how the API crumbles 1766 | let entry_info = read_entry_from_tail( 1767 | &mut flash, 1768 | &mut buffer, 1769 | Space::Zero, 1770 | 2, 1771 | ).expect("entry should pass read validation"); 1772 | 1773 | assert_eq!(entry_info.meta, &EntryMeta { 1774 | magic: EntryMeta::EXPECTED_MAGIC.into(), 1775 | subtype: 0xAA, 1776 | sub_bytes: 4, 1777 | contents_length: 8.into(), 1778 | }); 1779 | assert_eq!(entry_info.next_sector, 1); 1780 | assert_eq!(entry_info.submeta, &0xDEAD_BEEF_u32.to_le_bytes()); 1781 | } 1782 | 1783 | #[test] 1784 | fn kv_entry_round_trip() { 1785 | let mut flash = FakeFlash::<32>::new(64); 1786 | let mut buffer = [0u8; 32]; 1787 | 1788 | let end_of_entry = write_kv( 1789 | &mut flash, 1790 | &mut buffer, 1791 | Space::Zero, 1792 | 1, 1793 | b"hello", 1794 | b"world", 1795 | ).expect("entry should write"); 1796 | 1797 | let result = seek_kv_backwards( 1798 | &mut flash, 1799 | &mut buffer, 1800 | Space::Zero, 1801 | end_of_entry, 1802 | b"hello", 1803 | ).expect("entry should read back"); 1804 | 1805 | assert_eq!(result, Some(1)); 1806 | 1807 | let neg_result = seek_kv_backwards( 1808 | &mut flash, 1809 | &mut buffer, 1810 | Space::Zero, 1811 | end_of_entry, 1812 | b"no", 1813 | ); 1814 | match neg_result { 1815 | Ok(None) => (), 1816 | Ok(Some(s)) => panic!("nonexistent entry 'found' at sector {s}"), 1817 | Err(e) => panic!("entry read should succeed, and yet, {:?}", e), 1818 | } 1819 | } 1820 | 1821 | #[test] 1822 | fn format_without_erase_fails() { 1823 | let mut flash = FakeFlash::<16>::new(64); 1824 | // Program an arbitrary sector. 1825 | flash.program_sector(Space::Zero, 7, &[0; 16]).unwrap(); 1826 | 1827 | const G: u32 = 0xBAAD_F00D; 1828 | let mut buffer = [0; 16]; 1829 | 1830 | assert_eq!( 1831 | format(&mut flash, &mut buffer, Space::Zero, G), 1832 | Err(FormatError::NeedsErase), 1833 | ); 1834 | } 1835 | 1836 | #[test] 1837 | fn format_then_check() { 1838 | let mut flash = FakeFlash::<16>::new(64); 1839 | const G: u32 = 0xBAAD_F00D; 1840 | let mut buffer = [0; 16]; 1841 | format(&mut flash, &mut buffer, Space::Zero, G) 1842 | .expect("format should succeed"); 1843 | 1844 | let result = check(&mut flash, &mut buffer, Space::Zero) 1845 | .expect("check should not fail"); 1846 | 1847 | assert_eq!(result, CheckResult::ValidLog { 1848 | generation: G, 1849 | end: 1, 1850 | tail_erased: true, 1851 | incomplete_write: false, 1852 | }); 1853 | } 1854 | 1855 | #[test] 1856 | fn format_write1_check() { 1857 | let mut flash = FakeFlash::<16>::new(64); 1858 | const G: u32 = 0xBAAD_F00D; 1859 | let mut buffer = [0; 16]; 1860 | format(&mut flash, &mut buffer, Space::Zero, G) 1861 | .expect("format should succeed"); 1862 | 1863 | let end_of_entry = 1864 | write_kv(&mut flash, &mut buffer, Space::Zero, 1, b"hi", b"there") 1865 | .expect("write should succeed"); 1866 | assert_eq!(end_of_entry, 1 + 3); 1867 | 1868 | let result = check(&mut flash, &mut buffer, Space::Zero) 1869 | .expect("check should not fail"); 1870 | 1871 | assert_eq!(result, CheckResult::ValidLog { 1872 | generation: G, 1873 | end: 1 + 3, 1874 | tail_erased: true, 1875 | incomplete_write: false, 1876 | }); 1877 | } 1878 | 1879 | #[test] 1880 | fn format_write3_check() { 1881 | let mut flash = FakeFlash::<16>::new(64); 1882 | const G: u32 = 0xBAAD_F00D; 1883 | let mut buffer = [0; 16]; 1884 | format(&mut flash, &mut buffer, Space::Zero, G) 1885 | .expect("format should succeed"); 1886 | 1887 | let mut p = 1; 1888 | for _ in 0..3 { 1889 | p = 1890 | write_kv(&mut flash, &mut buffer, Space::Zero, p, b"hi", b"there") 1891 | .expect("write should succeed"); 1892 | } 1893 | 1894 | let result = check(&mut flash, &mut buffer, Space::Zero) 1895 | .expect("check should not fail"); 1896 | 1897 | assert_eq!(result, CheckResult::ValidLog { 1898 | generation: G, 1899 | end: 1 + 3 * 3, 1900 | tail_erased: true, 1901 | incomplete_write: false, 1902 | }); 1903 | } 1904 | 1905 | #[test] 1906 | fn write_out_of_space() { 1907 | let mut flash = FakeFlash::<16>::new(16); 1908 | const G: u32 = 0xBAAD_F00D; 1909 | let mut buffer = [0; 16]; 1910 | format(&mut flash, &mut buffer, Space::Zero, G) 1911 | .expect("format should succeed"); 1912 | 1913 | let mut p = 1; 1914 | for _ in 0..5 { 1915 | p = 1916 | write_kv(&mut flash, &mut buffer, Space::Zero, p, b"hi", b"there") 1917 | .expect("write should succeed"); 1918 | } 1919 | 1920 | let final_result = write_kv(&mut flash, &mut buffer, Space::Zero, p, b"hi", b"there"); 1921 | assert_eq!(Err(WriteError::NoSpace), final_result); 1922 | 1923 | let result = check(&mut flash, &mut buffer, Space::Zero) 1924 | .expect("check should not fail"); 1925 | 1926 | assert_eq!(result, CheckResult::ValidLog { 1927 | generation: G, 1928 | end: 1 + 5 * 3, // final one should not be reflected 1929 | tail_erased: true, 1930 | incomplete_write: false, 1931 | }); 1932 | } 1933 | 1934 | #[test] 1935 | fn check_incomplete() { 1936 | let mut flash = FakeFlash::<16>::new(64); 1937 | const G: u32 = 0xBAAD_F00D; 1938 | let mut buffer = [0; 16]; 1939 | format(&mut flash, &mut buffer, Space::Zero, G) 1940 | .expect("format should succeed"); 1941 | 1942 | let end_of_entry = 1943 | write_kv(&mut flash, &mut buffer, Space::Zero, 1, b"hi", b"there") 1944 | .expect("write should succeed"); 1945 | // Sneakily erase all sectors but the first. 1946 | for s in 2..end_of_entry { 1947 | flash.erase_sector(Space::Zero, s); 1948 | } 1949 | 1950 | let result = check(&mut flash, &mut buffer, Space::Zero) 1951 | .expect("check should not fail"); 1952 | 1953 | assert_eq!(result, CheckResult::ValidLog { 1954 | generation: G, 1955 | end: 1, // does not include incomplete entry 1956 | tail_erased: true, 1957 | incomplete_write: true, // <-- the point 1958 | }); 1959 | } 1960 | 1961 | #[test] 1962 | fn check_tail_not_erased() { 1963 | let mut flash = FakeFlash::<16>::new(64); 1964 | const G: u32 = 0xBAAD_F00D; 1965 | let mut buffer = [0; 16]; 1966 | format(&mut flash, &mut buffer, Space::Zero, G) 1967 | .expect("format should succeed"); 1968 | 1969 | let end_of_entry = 1970 | write_kv(&mut flash, &mut buffer, Space::Zero, 1, b"hi", b"there") 1971 | .expect("write should succeed"); 1972 | // Sneakily program a sector after the entry. 1973 | flash.program_sector(Space::Zero, end_of_entry + 2, &[0; 16]).unwrap(); 1974 | 1975 | let result = check(&mut flash, &mut buffer, Space::Zero) 1976 | .expect("check should not fail"); 1977 | 1978 | assert_eq!(result, CheckResult::ValidLog { 1979 | generation: G, 1980 | end: 1 + 3, 1981 | tail_erased: false, // <--- the point 1982 | incomplete_write: false, 1983 | }); 1984 | } 1985 | 1986 | #[test] 1987 | fn check_data_erasure() { 1988 | let mut flash = FakeFlash::<16>::new(64); 1989 | const G: u32 = 0xBAAD_F00D; 1990 | let mut buffer = [0; 16]; 1991 | format(&mut flash, &mut buffer, Space::Zero, G) 1992 | .expect("format should succeed"); 1993 | 1994 | let end_of_entry = 1995 | write_kv(&mut flash, &mut buffer, Space::Zero, 1, b"hi", b"there") 1996 | .expect("write should succeed"); 1997 | assert_eq!(end_of_entry, 4); 1998 | // Nuke the central data sector. 1999 | flash.erase_sector(Space::Zero, 2); 2000 | 2001 | let result = check(&mut flash, &mut buffer, Space::Zero) 2002 | .expect("check should not fail"); 2003 | 2004 | assert_eq!(result, CheckResult::Bad(CheckError::UnprogrammedData(1))); 2005 | } 2006 | 2007 | #[test] 2008 | fn evacuate_basic() { 2009 | let mut flash = FakeFlash::<32>::new(64); 2010 | let mut buffer = [0u8; 32]; 2011 | const G: u32 = 1; 2012 | format(&mut flash, &mut buffer, Space::Zero, G) 2013 | .expect("format should succeed"); 2014 | 2015 | let end_of_entry = write_kv( 2016 | &mut flash, 2017 | &mut buffer, 2018 | Space::Zero, 2019 | 1, 2020 | b"hello", 2021 | b"world", 2022 | ).expect("entry should write"); 2023 | 2024 | let mut buffer1 = [0u8; 32]; 2025 | evacuate( 2026 | &mut flash, 2027 | &mut buffer, 2028 | &mut buffer1, 2029 | Space::Zero, 2030 | end_of_entry, 2031 | ).expect("evacuate should succeed"); 2032 | 2033 | let result = seek_kv_backwards( 2034 | &mut flash, 2035 | &mut buffer, 2036 | Space::One, 2037 | end_of_entry, 2038 | b"hello", 2039 | ).expect("entry should read back"); 2040 | 2041 | assert_eq!(result, Some(1)); 2042 | } 2043 | 2044 | #[test] 2045 | fn evacuate_overwrite() { 2046 | let mut flash = FakeFlash::<32>::new(64); 2047 | let mut buffer = [0u8; 32]; 2048 | const G: u32 = 1; 2049 | format(&mut flash, &mut buffer, Space::Zero, G) 2050 | .expect("format should succeed"); 2051 | 2052 | let end_of_entry = write_kv( 2053 | &mut flash, 2054 | &mut buffer, 2055 | Space::Zero, 2056 | 1, 2057 | b"hello", 2058 | b"world", 2059 | ).expect("entry should write"); 2060 | let end_of_entry = write_kv( 2061 | &mut flash, 2062 | &mut buffer, 2063 | Space::Zero, 2064 | end_of_entry, 2065 | b"hello", 2066 | b"there", 2067 | ).expect("entry should write"); 2068 | 2069 | let mut buffer1 = [0u8; 32]; 2070 | evacuate( 2071 | &mut flash, 2072 | &mut buffer, 2073 | &mut buffer1, 2074 | Space::Zero, 2075 | end_of_entry, 2076 | ).expect("evacuate should succeed"); 2077 | 2078 | /* 2079 | let result = seek_kv_backwards( 2080 | &mut flash, 2081 | &mut buffer, 2082 | Space::One, 2083 | end_of_entry, 2084 | b"hello", 2085 | ).expect("entry should read back"); 2086 | 2087 | assert_eq!(result, Some(1)); 2088 | */ 2089 | } 2090 | } 2091 | -------------------------------------------------------------------------------- /testimg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oxidecomputer/lethe/72be5803eb8c28b8576d305b3d6f691c97de656d/testimg --------------------------------------------------------------------------------