├── LICENSE ├── MIEncryption.h ├── MIEncryption.m ├── MIStore+Metadata.h ├── MIStore+Metadata.m ├── MIStore+Update.h ├── MIStore+Update.m ├── MIStore.h └── MIStore.m /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /MIEncryption.h: -------------------------------------------------------------------------------- 1 | // 2 | // NSObject+MIEncryption.h 3 | // Elpass 4 | // 5 | // Created by Blankwonder on 2019/11/28. 6 | // Copyright © 2019 Surge Networks. All rights reserved. 7 | // 8 | 9 | #import 10 | 11 | 12 | @interface NSData (MIEncryption) 13 | 14 | - (NSData *)secretboxOpenWithKey:(NSData *)key; 15 | - (NSData *)secretboxOpenWithKey:(NSData *)key nonce:(NSData *)nonce; 16 | 17 | 18 | - (NSData *)secretboxWithKey:(NSData *)key; 19 | - (NSData *)secretboxWithKey:(NSData *)key nonce:(NSData *)nonce; 20 | 21 | + (NSData *)securityRandomDataWithLength:(NSInteger)length; 22 | 23 | @end 24 | 25 | -------------------------------------------------------------------------------- /MIEncryption.m: -------------------------------------------------------------------------------- 1 | // 2 | // NSObject+MIEncryption.m 3 | // Elpass 4 | // 5 | // Created by Blankwonder on 2019/11/28. 6 | // Copyright © 2019 Surge Networks. All rights reserved. 7 | // 8 | 9 | #import "MIEncryption.h" 10 | #import 11 | 12 | 13 | @implementation NSData (MIEncryption) 14 | 15 | - (NSData *)secretboxOpenWithKey:(NSData *)key { 16 | return [self secretboxOpenWithKey:key nonce:nil]; 17 | } 18 | 19 | - (NSData *)secretboxOpenWithKey:(NSData *)key nonce:(NSData *)nonceData { 20 | unsigned char nonce[crypto_secretbox_NONCEBYTES]; 21 | memset(nonce, 0, sizeof(nonce)); 22 | 23 | if (nonceData) { 24 | if (nonceData.length != crypto_secretbox_NONCEBYTES) return nil; 25 | [nonceData getBytes:nonce length:nonceData.length]; 26 | } 27 | 28 | if (key.length != crypto_secretbox_KEYBYTES) return nil; 29 | if (self.length <= crypto_secretbox_MACBYTES) return nil; 30 | 31 | NSMutableData *decryptedData = [NSMutableData dataWithLength:self.length - crypto_secretbox_MACBYTES]; 32 | 33 | if (crypto_secretbox_open_easy(decryptedData.mutableBytes, self.bytes, self.length, nonce, key.bytes) != 0) { 34 | return nil; 35 | } 36 | 37 | return decryptedData; 38 | } 39 | 40 | - (NSData *)secretboxWithKey:(NSData *)key { 41 | return [self secretboxWithKey:key nonce:nil]; 42 | 43 | } 44 | 45 | - (NSData *)secretboxWithKey:(NSData *)key nonce:(NSData *)nonceData { 46 | unsigned char nonce[crypto_secretbox_NONCEBYTES]; 47 | memset(nonce, 0, sizeof(nonce)); 48 | 49 | if (nonceData) { 50 | if (nonceData.length != crypto_secretbox_NONCEBYTES) return nil; 51 | [nonceData getBytes:nonce length:nonceData.length]; 52 | } 53 | if (key.length != crypto_secretbox_KEYBYTES) return nil; 54 | 55 | 56 | NSMutableData *encryptedData = [NSMutableData dataWithLength:self.length + crypto_secretbox_MACBYTES]; 57 | if (crypto_secretbox_easy(encryptedData.mutableBytes, self.bytes, self.length, nonce, key.bytes) != 0) { 58 | return nil; 59 | } 60 | 61 | return encryptedData; 62 | } 63 | 64 | + (NSData *)securityRandomDataWithLength:(NSInteger)length { 65 | void *ptr = malloc(length); 66 | randombytes_buf(ptr, length); 67 | 68 | return [NSData dataWithBytesNoCopy:ptr length:length freeWhenDone:YES]; 69 | } 70 | 71 | @end 72 | -------------------------------------------------------------------------------- /MIStore+Metadata.h: -------------------------------------------------------------------------------- 1 | // 2 | // MIStore+Metadata.h 3 | // Elpass 4 | // 5 | // Created by Blankwonder on 2019/9/11. 6 | // Copyright © 2019 Surge Networks. All rights reserved. 7 | // 8 | 9 | 10 | #import "MIStore.h" 11 | 12 | @interface MIStore (Metadata) 13 | 14 | - (void)rebuildAllMetadataFromTrunk; 15 | - (BOOL)mergeMetadata; 16 | 17 | - (void)metadataIsReadyToMerge; 18 | 19 | - (void)writeItemMetadatasForBlock:(int)blockNumber; 20 | 21 | + (BOOL)verifyStoreIntegrityInPath:(NSString *)path; 22 | 23 | @end 24 | -------------------------------------------------------------------------------- /MIStore+Metadata.m: -------------------------------------------------------------------------------- 1 | // 2 | // MIStore+Metadata.m 3 | // Elpass 4 | // 5 | // Created by Blankwonder on 2019/9/11. 6 | // Copyright © 2019 Surge Networks. All rights reserved. 7 | // 8 | 9 | #import "MIStore+Metadata.h" 10 | #import "MIStore+Private.h" 11 | #import "MessagePack.h" 12 | #import 13 | #import "NSURL+KKDomain.h" 14 | #import "NSString+KKDomain.h" 15 | #import "MIEncryption.h" 16 | 17 | @implementation MIStore (Metadata) 18 | 19 | - (void)writeMetadataBlock:(int)blockNumber items:(NSArray *)items { 20 | [_driver createDirectory:kStorageDirectoryMetadata error:nil]; 21 | 22 | NSData *key = [self deriveKeyWithSubkeyID:MIStoreSubkeyIDMetadataMask + blockNumber size:crypto_secretbox_KEYBYTES]; 23 | 24 | NSArray *jsonArray = [items KD_arrayUsingMapEnumerateBlock:^id(MIItem *obj, NSUInteger idx) { 25 | return [obj jsonDictionaryForStore]; 26 | }]; 27 | 28 | NSData *plainData = [MessagePack packObject:jsonArray]; 29 | NSData *ciphertext = [plainData secretboxWithKey:key]; 30 | 31 | // NSString *path = [self metadataPathForBlock:blockNumber]; 32 | KDClassLog(@"Write %ld metadata payloads to: block %d", items.count, blockNumber); 33 | 34 | //#if DEBUG 35 | // KDClassLog(@"Payloads in metadata: %@", jsonArray); 36 | //#endif 37 | 38 | NSError *error = nil; 39 | BOOL success = [_driver writeData:ciphertext toPath:[NSString stringWithFormat:@"%d", blockNumber] directory:kStorageDirectoryMetadata error:&error]; 40 | KDLoggerPrintError(error); 41 | 42 | if (!success) { 43 | MIEncounterPanicError(error); 44 | } 45 | } 46 | 47 | - (void)rebuildAllMetadataFromTrunk { 48 | [self syncDispatch:^{ 49 | // NSString *dirPath = self.metadataFolderPath; 50 | 51 | // [_driver removeItemAtPath:dirPath error:nil]; 52 | // [_driver createDirectoryAtPath:dirPath withIntermediateDirectories:YES attributes:nil error:nil]; 53 | 54 | NSMutableDictionary *map = [NSMutableDictionary dictionaryWithCapacity:64]; 55 | 56 | for (MIItem *item in _trunk.itemMap.allValues) { 57 | int block = item.blockNumber; 58 | 59 | NSMutableArray *array = map[@(block)]; 60 | if (!array) { 61 | array = [NSMutableArray array]; 62 | map[@(block)] = array; 63 | } 64 | [array addObject:item]; 65 | } 66 | 67 | for (int i = 0; i < 64; i++) { 68 | [self writeMetadataBlock:i items:map[@(i)] ?: @[]]; 69 | } 70 | 71 | // [map enumerateKeysAndObjectsUsingBlock:^(NSNumber *key, NSMutableArray *array, BOOL * _Nonnull stop) { 72 | // [self writeMetadataBlock:key.intValue items:array]; 73 | // }]; 74 | 75 | }]; 76 | } 77 | 78 | //- (NSString *)metadataPathForBlock:(int)blockNumber { 79 | // return [self.metadataFolderPath stringByAppendingPathComponent:[NSString stringWithFormat:@"%d", blockNumber]]; 80 | //} 81 | 82 | 83 | - (void)writeItemMetadatasForBlock:(int)blockNumber { 84 | if (self.preventWriting) { 85 | KDClassLog(@"writeItemMetadatasForBlock while preventWriting = YES!!"); 86 | return; 87 | } 88 | NSMutableArray *array = [NSMutableArray array]; 89 | 90 | for (MIItem *item in _trunk.itemMap.allValues) { 91 | if (item.blockNumber == blockNumber) [array addObject:item]; 92 | } 93 | 94 | [self writeMetadataBlock:blockNumber items:array]; 95 | } 96 | 97 | 98 | - (BOOL)mergeMetadata { 99 | KDClassLog(@"mergeMetadata"); 100 | __block BOOL changed = NO; 101 | [self syncDispatch:^{ 102 | CFAbsoluteTime start = CFAbsoluteTimeGetCurrent(); 103 | 104 | NSArray *subpaths = [_driver contentsOfDirectory:kStorageDirectoryMetadata error:nil]; 105 | 106 | NSMutableSet *remainingUUIDs = [NSMutableSet setWithArray:_trunk.itemMap.allKeys]; 107 | 108 | NSMutableArray *updatedItems = [NSMutableArray array]; 109 | NSMutableArray *insertedItems = [NSMutableArray array]; 110 | 111 | NSMutableArray *metadataPayloads = [NSMutableArray arrayWithCapacity:_trunk.itemMap.count]; 112 | 113 | for (NSString *filename in subpaths) { 114 | int block = filename.intValue; 115 | if (block == 0 && ![filename isEqualToString:@"0"]) continue; 116 | 117 | 118 | if (![filename isEqualToString:[NSString stringWithFormat:@"%d", block]]) { 119 | KDClassLog(@"Invalid metadata filename: %@ (%d), remove it", filename, block); 120 | NSError *error = nil; 121 | 122 | [_driver removeItemAtPath:filename directory:kStorageDirectoryMetadata error:&error]; 123 | KDLoggerPrintError(error); 124 | 125 | continue; 126 | } 127 | 128 | NSData *blockData = [_driver readDataAtPath:filename directory:kStorageDirectoryMetadata]; 129 | 130 | NSData *key = [self deriveKeyWithSubkeyID:MIStoreSubkeyIDMetadataMask + block size:crypto_secretbox_KEYBYTES]; 131 | 132 | NSData *decrypted = [blockData secretboxOpenWithKey:key]; 133 | if (!decrypted) { 134 | KDClassLog(@"Failed to decrypt metadata file: %@, abort!", filename); 135 | return; 136 | } 137 | 138 | NSArray *items = [MessagePack unpackData:decrypted]; 139 | 140 | [metadataPayloads addObjectsFromArray:items]; 141 | } 142 | 143 | for (NSDictionary *payload in metadataPayloads) { 144 | NSString *uuid = payload[@"uuid"]; 145 | MIItem *trunkItem = _trunk.itemMap[uuid]; 146 | 147 | MIItem *item = [MIItem deserializeFromDictionary:payload]; 148 | 149 | if (!item) { 150 | continue; 151 | } 152 | 153 | if (!trunkItem) { 154 | item.store = self; 155 | _trunk.itemMap[uuid] = item; 156 | 157 | NSMutableArray *array = [_trunk itemArrayForClass:item.class]; 158 | [array addObject:item]; 159 | 160 | [insertedItems addObject:item]; 161 | } else { 162 | [remainingUUIDs removeObject:uuid]; 163 | 164 | if ([item isEqualToItem:trunkItem]) { 165 | //KDClassLog(@"%@: Identical", uuid) 166 | } else { 167 | [updatedItems addObject:trunkItem]; 168 | KDClassLog(@"Metadata object is different to trunk, merge: %@", uuid); 169 | 170 | #if DEBUG 171 | NSDictionary *trunkPayload = [trunkItem jsonDictionaryForStore]; 172 | KDDebuggerPrintDictionaryDiff(payload, trunkPayload); 173 | KDClassLog(@"Original payload in metadata: %@", payload); 174 | #endif 175 | 176 | [trunkItem yy_mergeAllPropertiesFrom:item]; 177 | } 178 | } 179 | } 180 | 181 | for (NSString *uuid in remainingUUIDs) { 182 | KDClassLog(@"Metadata object doesn't exist for trunk item, deleting: %@", uuid); 183 | 184 | MIItem *item = _trunk.itemMap[uuid]; 185 | 186 | [_trunk.itemMap removeObjectForKey:uuid]; 187 | 188 | NSMutableArray *array = [_trunk itemArrayForClass:item.class]; 189 | [array removeObject:item]; 190 | } 191 | 192 | KDClassLog(@"Metadata verification completed in %.0f ms, updated: %ld, deleted: %ld, inserted: %ld", (CFAbsoluteTimeGetCurrent() - start) * 1000, updatedItems.count, remainingUUIDs.count, insertedItems.count); 193 | 194 | if (remainingUUIDs.count + insertedItems.count > 0) { 195 | changed = YES; 196 | dispatch_async( dispatch_get_main_queue(),^{ 197 | [[NSNotificationCenter defaultCenter] postNotificationName:MIStoreDidUpdateList object:self]; 198 | }); 199 | } 200 | 201 | if (updatedItems.count > 0) { 202 | changed = YES; 203 | dispatch_async( dispatch_get_main_queue(),^{ 204 | [[NSNotificationCenter defaultCenter] postNotificationName:MIStoreDidUpdateItems object:self userInfo:@{@"items": updatedItems}]; 205 | }); 206 | } 207 | 208 | dispatch_async( dispatch_get_main_queue(),^{ 209 | [[NSNotificationCenter defaultCenter] postNotificationName:MIStoreDidCompleteMergingMetadata object:self]; 210 | }); 211 | }]; 212 | 213 | return changed; 214 | } 215 | 216 | - (void)metadataIsReadyToMerge { 217 | [self syncDispatch:^{ 218 | if (self.preventWriting) return; 219 | 220 | if (![MIStore verifyStoreIntegrityInPath:self.databasePath]) { 221 | KDClassLog(@"Data integrity verification failed, refuse to merge metadata!"); 222 | return; 223 | } 224 | 225 | BOOL changed = [self mergeMetadata]; 226 | if (changed) { 227 | KDClassLog(@"Metadata merged to trunk"); 228 | [self updateTags]; 229 | [self saveTrunkIfNecessary]; 230 | [self updateLastUpdateTimestampForTrunkItemMap]; 231 | } 232 | 233 | if (_shouldUpgradeToAllMetaData) { 234 | KDClassLog(@"Upgrade to all metadata"); 235 | 236 | [self rebuildAllMetadataFromTrunk]; 237 | [self _rewriteIndexPayloadWithAllMetadataFlag]; 238 | _shouldUpgradeToAllMetaData = NO; 239 | } 240 | }]; 241 | } 242 | 243 | + (BOOL)verifyStoreIntegrityInPath:(NSString *)path { 244 | NSData *data = [NSData dataWithContentsOfFile:[path stringByAppendingPathComponent:@"Index"]]; 245 | 246 | NSDictionary *index = [MessagePack unpackData:data]; 247 | 248 | if (!index) return NO; 249 | 250 | BOOL amd = (index[@"amd"] != nil); 251 | 252 | if (![[NSFileManager defaultManager] fileExistsAtPath:[path stringByAppendingPathComponent:@"Trunk"]]) { 253 | return NO; 254 | } 255 | 256 | if (amd) { 257 | NSMutableIndexSet *indexSet = [[NSMutableIndexSet alloc] init]; 258 | for (NSString *filename in [[NSFileManager defaultManager] contentsOfDirectoryAtPath:[path stringByAppendingPathComponent:kStorageDirectoryMetadata] error:nil]) { 259 | int block = filename.intValue; 260 | if (block == 0 && ![filename isEqualToString:@"0"]) continue; 261 | 262 | if (![filename isEqualToString:[NSString stringWithFormat:@"%d", block]]) { 263 | continue; 264 | } 265 | 266 | [indexSet addIndex:block]; 267 | } 268 | 269 | if (![indexSet containsIndexesInRange:NSMakeRange(0, 64)]) { 270 | for (int i = 0; i < 64; i++) { 271 | if (![indexSet containsIndex:i]) { 272 | KDClassLog(@"Missing metadata file: %d", i); 273 | } 274 | } 275 | 276 | return NO; 277 | } 278 | } 279 | 280 | return YES; 281 | } 282 | 283 | @end 284 | -------------------------------------------------------------------------------- /MIStore+Update.h: -------------------------------------------------------------------------------- 1 | // 2 | // MIStore+Update.h 3 | // Elpass 4 | // 5 | // Created by Blankwonder on 2019/10/12. 6 | // Copyright © 2019 Surge Networks. All rights reserved. 7 | // 8 | 9 | #import "MIStore.h" 10 | 11 | @interface MIStore (Update) 12 | 13 | - (void)beginBatchOperations; 14 | - (void)endBatchOperations; 15 | 16 | - (void)addItem:(MIItem *)item; 17 | - (void)deleteItem:(MIItem *)item; 18 | - (void)updateItem:(MIItem *)item block:(void (^)(MIItem *item))block; 19 | 20 | - (void)markItemFavorited:(MIItem *)item; 21 | - (void)unmarkItemFavorited:(MIItem *)item; 22 | 23 | - (void)archiveItem:(MIItem *)item; 24 | - (void)unarchiveItem:(MIItem *)item; 25 | 26 | - (void)deleteTag:(NSString *)tag; 27 | - (void)renameTag:(NSString *)oldTag to:(NSString *)newTag; 28 | 29 | - (void)addAttachment:(MIAttachment *)attachment completionHandler:(void (^)(NSError *error))completionHandler; 30 | 31 | - (void)setIconForItem:(MIItem *)item iconData:(NSData *)data; 32 | - (void)removeIconForItem:(MIItem *)item; 33 | - (void)mkdirIconFolder; 34 | 35 | 36 | - (void)updateTags; 37 | 38 | @end 39 | -------------------------------------------------------------------------------- /MIStore+Update.m: -------------------------------------------------------------------------------- 1 | // 2 | // MIStore+Update.m 3 | // Elpass 4 | // 5 | // Created by Blankwonder on 2019/10/12. 6 | // Copyright © 2019 Surge Networks. All rights reserved. 7 | // 8 | 9 | #import "MIStore+Update.h" 10 | #import "MIStore+Private.h" 11 | #import "MIStore+Metadata.h" 12 | #import 13 | #import "MIEncryption.h" 14 | #import "MILocalResourceCache.h" 15 | #import "MIRecentlyUsedManager.h" 16 | 17 | @implementation MIStore (Update) 18 | 19 | - (void)markItemFavorited:(MIItem *)item { 20 | if (self.readonly) return; 21 | 22 | [self syncDispatch:^{ 23 | if (item.favIdx != 0) return; 24 | 25 | NSArray *favItems = self.favoritedItems; 26 | 27 | int index = [(MIItem *)favItems.firstObject favIdx] + 1000; 28 | 29 | [self _internalUpdateItem:item block:^(MIItem *item) { 30 | item.favIdx = index; 31 | if (item.archived) { 32 | item.archived = NO; 33 | } 34 | }]; 35 | }]; 36 | } 37 | 38 | - (void)unmarkItemFavorited:(MIItem *)item { 39 | if (self.readonly) return; 40 | 41 | [self syncDispatch:^{ 42 | if (item.favIdx == 0) return; 43 | 44 | [self _internalUpdateItem:item block:^(MIItem *item) { 45 | item.favIdx = 0; 46 | }]; 47 | }]; 48 | } 49 | 50 | - (void)archiveItem:(MIItem *)item { 51 | if (self.readonly) return; 52 | 53 | [self syncDispatch:^{ 54 | if (item.archived) return; 55 | 56 | [self _internalUpdateItem:item block:^(MIItem *item) { 57 | item.archived = YES; 58 | if (item.favIdx != 0) item.favIdx = 0; 59 | }]; 60 | 61 | [MIRecentlyUsedManager.sharedInstance removeUsedItem:item.uuid]; 62 | }]; 63 | } 64 | 65 | - (void)unarchiveItem:(MIItem *)item { 66 | if (self.readonly) return; 67 | 68 | [self syncDispatch:^{ 69 | if (!item.archived) return; 70 | 71 | [self _internalUpdateItem:item block:^(MIItem *item) { 72 | item.archived = NO; 73 | }]; 74 | }]; 75 | } 76 | 77 | - (void)beginBatchOperations { 78 | KDClassLog(@"Enter batch operations mode"); 79 | [self syncDispatch:^{ 80 | KDAssert(_batchObjects == nil); 81 | _batchObjects = [NSMutableSet set]; 82 | }]; 83 | } 84 | 85 | - (void)endBatchOperations { 86 | KDClassLog(@"Exit batch operations mode"); 87 | 88 | [self syncDispatch:^{ 89 | if (_batchObjects.count > 0) { 90 | [self scheduleSaveTrunk]; 91 | 92 | NSMutableSet *blocks = [NSMutableSet set]; 93 | 94 | for (MIItem *item in _batchObjects) { 95 | [blocks addObject:@(item.blockNumber)]; 96 | } 97 | 98 | for (NSNumber *b in blocks) { 99 | [self writeItemMetadatasForBlock:b.intValue]; 100 | } 101 | 102 | [self updateTags]; 103 | 104 | NSArray *allItems = _batchObjects.allObjects; 105 | dispatch_async( dispatch_get_main_queue(),^{ 106 | [[NSNotificationCenter defaultCenter] postNotificationName:MIStoreDidUpdateList object:self]; 107 | [[NSNotificationCenter defaultCenter] postNotificationName:MIStoreDidUpdateItems object:self userInfo:@{@"batch": @YES, @"items": allItems}]; 108 | }); 109 | } 110 | _batchObjects = nil; 111 | }]; 112 | } 113 | 114 | 115 | - (void)addItem:(MIItem *)item { 116 | if (self.readonly) return; 117 | 118 | [self syncDispatch:^{ 119 | item.store = self; 120 | if (!item.uuid) item.uuid = [NSUUID UUID].UUIDString; 121 | if (item.createdAt == 0) item.createdAt = MIGetTimestamp(); 122 | if (item.updatedAt == 0) item.updatedAt = MIGetTimestamp(); 123 | 124 | KDClassLog(@"Adding item: %@", item.uuid); 125 | 126 | NSMutableArray *array = [_trunk itemArrayForClass:item.class]; 127 | [array addObject:item]; 128 | 129 | _trunk.itemMap[item.uuid] = item; 130 | 131 | if (_batchObjects) { 132 | [_batchObjects addObject:item]; 133 | } else { 134 | [self writeItemMetadatasForBlock:item.blockNumber]; 135 | [self scheduleSaveTrunk]; 136 | [self updateTags]; 137 | 138 | dispatch_async( dispatch_get_main_queue(),^{ 139 | [[NSNotificationCenter defaultCenter] postNotificationName:MIStoreDidUpdateList object:self]; 140 | [[NSNotificationCenter defaultCenter] postNotificationName:MIStoreDidAddItem object:item]; 141 | }); 142 | } 143 | }]; 144 | } 145 | 146 | - (void)deleteItem:(MIItem *)item { 147 | if (self.readonly) return; 148 | 149 | [self syncDispatch:^{ 150 | NSString *uuid = item.uuid; 151 | KDClassLog(@"Removing item: %@", uuid); 152 | 153 | [item setDeleted:YES]; 154 | [_trunk.itemMap removeObjectForKey:uuid]; 155 | NSMutableArray *array = [_trunk itemArrayForClass:item.class]; 156 | [array removeObject:item]; 157 | 158 | for (MIAttachment *a in item.attachments) { 159 | [self removeAttachmentWithUUIDIfNecessary:a.uuid]; 160 | } 161 | 162 | if (item.iconUUID) { 163 | // Item already removed from _trunk 164 | [self removeIconFileWithUUIDIfNecessary:item.iconUUID]; 165 | } 166 | 167 | if (_batchObjects) { 168 | [_batchObjects addObject:item]; 169 | } else { 170 | [self scheduleSaveTrunk]; 171 | [self writeItemMetadatasForBlock:item.blockNumber]; 172 | [self updateTags]; 173 | 174 | dispatch_async( dispatch_get_main_queue(),^{ 175 | [[NSNotificationCenter defaultCenter] postNotificationName:MIStoreDidUpdateList object:self]; 176 | }); 177 | } 178 | 179 | [MIRecentlyUsedManager.sharedInstance removeUsedItem:uuid]; 180 | }]; 181 | } 182 | 183 | - (void)_internalUpdateItem:(MIItem *)_item block:(void (^)(MIItem *item))block { 184 | if (self.readonly) return; 185 | 186 | [self syncDispatch:^{ 187 | MIItem *item = _item; 188 | KDClassLog(@"Updating item: %@", item.uuid); 189 | 190 | MIItem *trunkItem = _trunk.itemMap[item.uuid]; 191 | if (!trunkItem) { 192 | KDClassLog(@"Trunk item doesn't exist"); 193 | return; 194 | } 195 | if (trunkItem != item) { 196 | KDClassLog(@"Trunk item != item"); 197 | item = trunkItem; 198 | } 199 | 200 | NSMutableSet *oldAttachementUUIDs = [NSMutableSet set]; 201 | for (MIAttachment *a in item.attachments) { 202 | [oldAttachementUUIDs addObject:a.uuid]; 203 | } 204 | 205 | #if DEBUG 206 | KDClassLog(@"Before %@", item); 207 | NSDictionary *before = [item jsonDictionaryForStore]; 208 | #endif 209 | block(item); 210 | #if DEBUG 211 | KDClassLog(@"After %@", item); 212 | NSDictionary *after = [item jsonDictionaryForStore]; 213 | 214 | KDDebuggerPrintDictionaryDiff(before, after); 215 | #endif 216 | 217 | NSMutableSet *newAttachementUUIDs = [NSMutableSet set]; 218 | for (MIAttachment *a in item.attachments) { 219 | [newAttachementUUIDs addObject:a.uuid]; 220 | } 221 | 222 | [oldAttachementUUIDs minusSet:newAttachementUUIDs]; 223 | 224 | if (oldAttachementUUIDs.count > 0) { 225 | [oldAttachementUUIDs enumerateObjectsUsingBlock:^(NSString *obj, BOOL * _Nonnull stop) { 226 | [self removeAttachmentWithUUIDIfNecessary:obj]; 227 | }]; 228 | } 229 | 230 | _lastUpdatedAt = MIGetTimestamp(); 231 | 232 | if (_batchObjects) { 233 | [_batchObjects addObject:item]; 234 | } else { 235 | [self scheduleSaveTrunk]; 236 | [self writeItemMetadatasForBlock:item.blockNumber]; 237 | [self updateTags]; 238 | 239 | dispatch_async( dispatch_get_main_queue(),^{ 240 | [[NSNotificationCenter defaultCenter] postNotificationName:MIStoreDidUpdateItems object:self userInfo:@{@"items": @[item]}]; 241 | }); 242 | } 243 | }]; 244 | } 245 | 246 | 247 | - (void)updateItem:(MIItem *)item block:(void (^)(MIItem *item))block { 248 | if (self.readonly) return; 249 | 250 | [self _internalUpdateItem:item block:^(MIItem *item) { 251 | block(item); 252 | item.updatedAt = MIGetTimestamp(); 253 | _lastUpdatedAt = item.updatedAt; 254 | }]; 255 | } 256 | 257 | - (void)deleteTag:(NSString *)tag { 258 | [self syncDispatch:^{ 259 | BOOL alreadyInBatch = _batchObjects != nil; 260 | 261 | if (!alreadyInBatch) { 262 | [self beginBatchOperations]; 263 | } 264 | 265 | for (MIItem *item in self.allItemsIncludedArchived) { 266 | if ([item.tags containsObject:tag]) { 267 | [self updateItem:item block:^(MIItem *item) { 268 | NSMutableArray *tags = item.tags.mutableCopy; 269 | [tags removeObject:tag]; 270 | item.tags = tags; 271 | }]; 272 | } 273 | } 274 | 275 | if (!alreadyInBatch) { 276 | [self endBatchOperations]; 277 | } 278 | }]; 279 | } 280 | 281 | - (void)addAttachment:(MIAttachment *)attachment completionHandler:(void (^)(NSError *error))completionHandler { 282 | KDAssert(attachment.sourcePath); 283 | dispatch_async( dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT,0),^{ 284 | NSData *data = [NSData dataWithContentsOfFile:attachment.sourcePath]; 285 | 286 | if (!data) { 287 | completionHandler(MIStoreError(@"The attachment file doesn't exist", 10)); 288 | return; 289 | } 290 | 291 | NSData *key = [self deriveKeyWithSubkeyID:MIStoreSubkeyIDAttachment size:crypto_secretbox_KEYBYTES]; 292 | NSData *ciphertext = [data secretboxWithKey:key]; 293 | 294 | [_driver createDirectory:kStorageDirectoryAttachments error:nil]; 295 | 296 | NSError *error = nil; 297 | [_driver writeData:ciphertext toPath:attachment.uuid directory:kStorageDirectoryAttachments error:&error]; 298 | 299 | KDLoggerPrintError(error); 300 | 301 | [self asyncDispatch:^{ 302 | completionHandler(error); 303 | }]; 304 | }); 305 | } 306 | 307 | - (void)removeAttachmentWithUUIDIfNecessary:(NSString *)uuid { 308 | if (self.readonly) return; 309 | 310 | [self syncDispatch:^{ 311 | 312 | for (MIItem *item in _trunk.itemMap.allValues) { 313 | for (MIAttachment *a in item.attachments) { 314 | if ([a.uuid isEqualToString:uuid]) { 315 | KDClassLog(@"Attachment %@ is still used by another item, skip deleting", uuid); 316 | return; 317 | } 318 | } 319 | 320 | } 321 | KDClassLog(@"Delete attachment: %@", uuid); 322 | 323 | NSError *error = nil; 324 | [_driver removeItemAtPath:uuid directory:kStorageDirectoryAttachments error:&error]; 325 | KDLoggerPrintError(error); 326 | }]; 327 | } 328 | 329 | - (void)updateTags { 330 | NSMutableSet *set = [NSMutableSet set]; 331 | 332 | for (MIItem *item in _trunk.itemMap.allValues) { 333 | if (!item.archived && item.tags.count > 0) { 334 | [set addObjectsFromArray:item.tags]; 335 | } 336 | } 337 | 338 | NSArray *tags = [set.allObjects sortedArrayUsingSelector:@selector(caseInsensitiveCompare:)]; 339 | 340 | if (![_allTags isEqualToArray:tags]) { 341 | _allTags = tags; 342 | 343 | dispatch_async( dispatch_get_main_queue(),^{ 344 | [NSNotificationCenter.defaultCenter postNotificationName:MIStoreDidUpdateTags object:self]; 345 | }); 346 | } 347 | } 348 | 349 | - (void)renameTag:(NSString *)oldTag to:(NSString *)newTag { 350 | if ([oldTag isEqualToString:newTag]) return; 351 | [self syncDispatch:^{ 352 | BOOL alreadyInBatch = _batchObjects != nil; 353 | 354 | if (!alreadyInBatch) { 355 | [self beginBatchOperations]; 356 | } 357 | 358 | for (MIItem *item in self.allItemsIncludedArchived) { 359 | if ([item.tags containsObject:oldTag]) { 360 | [self updateItem:item block:^(MIItem *item) { 361 | NSMutableArray *tags = item.tags.mutableCopy; 362 | [tags removeObject:oldTag]; 363 | if (![tags containsObject:newTag]) { 364 | [tags addObject:newTag]; 365 | } 366 | item.tags = tags; 367 | }]; 368 | } 369 | } 370 | 371 | if (!alreadyInBatch) { 372 | [self endBatchOperations]; 373 | } 374 | 375 | }]; 376 | 377 | } 378 | 379 | - (void)setIconForItem:(MIItem *)item iconData:(NSData *)data { 380 | [self updateItem:item block:^(MIItem *item) { 381 | NSString *hash = [data KD_MD5]; 382 | 383 | NSString *previousIconHash = item.iconUUID; 384 | item.iconUUID = hash; 385 | 386 | [self mkdirIconFolder]; 387 | [_driver writeData:data toPath:item.iconUUID directory:kStorageDirectoryIcons error:nil]; 388 | 389 | [MILocalResourceCache.sharedInstance invalidCacheAtPath:item.iconUUID]; 390 | if (previousIconHash && ![hash isEqualToString:previousIconHash]) { 391 | [self asyncDispatch:^{ 392 | [self removeIconFileWithUUIDIfNecessary:previousIconHash]; 393 | }]; 394 | } 395 | }]; 396 | } 397 | 398 | - (void)mkdirIconFolder { 399 | [_driver createDirectory:kStorageDirectoryIcons error:nil]; 400 | } 401 | 402 | - (void)removeIconForItem:(MIItem *)item { 403 | if (!item.iconUUID) return; 404 | NSString *iconUUID = item.iconUUID; 405 | [self updateItem:item block:^(MIItem *item) { 406 | item.iconUUID = nil; 407 | }]; 408 | [self removeIconFileWithUUIDIfNecessary:iconUUID]; 409 | } 410 | 411 | - (void)removeIconFileWithUUIDIfNecessary:(NSString *)iconUUID { 412 | if (self.readonly) return; 413 | 414 | [self syncDispatch:^{ 415 | for (MIItem *item in _trunk.itemMap.allValues) { 416 | if ([item.iconUUID isEqualToString:iconUUID]) { 417 | KDClassLog(@"Icon %@ is still used by another item, skip deleting", iconUUID); 418 | return; 419 | } 420 | } 421 | KDClassLog(@"Delete icon file: %@", iconUUID); 422 | 423 | NSError *error = nil; 424 | [_driver removeItemAtPath:iconUUID directory:kStorageDirectoryIcons error:&error]; 425 | KDLoggerPrintError(error); 426 | 427 | [MILocalResourceCache.sharedInstance invalidCacheAtPath:iconUUID]; 428 | }]; 429 | } 430 | 431 | @end 432 | -------------------------------------------------------------------------------- /MIStore.h: -------------------------------------------------------------------------------- 1 | // 2 | // MIStore.h 3 | // Elpass iOS 4 | // 5 | // Created by Blankwonder on 2019/8/16. 6 | // Copyright © 2019 Surge Networks. All rights reserved. 7 | // 8 | 9 | #import 10 | #import "MIModalDefines.h" 11 | #import "KDOrderedDictionary.h" 12 | #import "MIPersistentStoreDriver.h" 13 | 14 | extern NSString *const MIStoreErrorDomain; 15 | extern NSError *MIStoreError(NSString *message, NSInteger code); 16 | 17 | typedef NS_ENUM(int, MIStoreState) { 18 | MIStoreStateNull, 19 | MIStoreStateLocked, 20 | MIStoreStateUnlocked, 21 | MIStoreStateDamaged, 22 | MIStoreStateUnsupportedVersion, 23 | 24 | }; 25 | 26 | @interface MIStoreTrunkData : NSObject 27 | 28 | @property (nonatomic) NSMutableArray *logins; 29 | @property (nonatomic) NSMutableDictionary *itemMap; 30 | @property (nonatomic) NSMutableArray *bankCards; 31 | @property (nonatomic) NSMutableArray *secureNotes; 32 | @property (nonatomic) NSMutableArray *identifications; 33 | @property (nonatomic) NSMutableArray *passwords; 34 | @property (nonatomic) NSMutableArray *softwareLicenses; 35 | @property (nonatomic) NSMutableArray *bankAccounts; 36 | 37 | - (void)rebuildCategoryArray; 38 | - (NSMutableArray *)itemArrayForClass:(Class)class; 39 | 40 | @end 41 | 42 | 43 | @interface MIStore : NSObject { 44 | MIModalDatabaseDescriptor *_descriptor; 45 | 46 | NSData *_encryptedDescriptorData; 47 | NSData *_encryptedDescriptorDataNonce; 48 | NSData *_masterPasswordSalt; 49 | 50 | MIStoreTrunkData *_trunk; 51 | 52 | dispatch_queue_t _queue; 53 | 54 | NSMutableSet *_batchObjects; 55 | 56 | NSArray *_allTags; 57 | 58 | MIPersistentStoreDriver *_driver; 59 | 60 | MITimestamp _lastUpdatedAt; 61 | BOOL _shouldUpgradeToAllMetaData; 62 | } 63 | 64 | - (instancetype)initWithPersistentStore:(MIPersistentStoreDriver *)driver NS_DESIGNATED_INITIALIZER; 65 | - (instancetype)init NS_UNAVAILABLE; 66 | 67 | - (NSData *)createNewDatabaseWithError:(NSError **)errorPtr masterPassword:(NSString *)password dbuuid:(NSString *)dbuuid; 68 | - (NSData *)changeMasterPassword:(NSString *)password; 69 | 70 | - (BOOL)loadDatabaseWithError:(NSError **)errorPtr; 71 | 72 | - (void)unlockWithMasterPassword:(NSString *)password completionHandler:(void (^)(BOOL success, NSError *error, NSData *key))completionHandler; 73 | - (void)unlockWithMasterKey:(NSData *)key completionHandler:(void (^)(BOOL success, NSError *error))completionHandler; 74 | 75 | - (void)verifyMasterPassword:(NSString *)password completionHandler:(void (^)(BOOL success, NSData *key))completionHandler; 76 | 77 | - (BOOL)isStoreFilesExist; 78 | 79 | @property (nonatomic) MIPersistentStoreDriver *driver; 80 | @property (nonatomic, readonly) MIStoreState state; 81 | @property (nonatomic, readonly) NSString *databasePath; 82 | 83 | @property (nonatomic, readonly) long trunkUpdatedAt; 84 | 85 | @property (nonatomic, readonly) MITimestamp lastUpdatedAt; 86 | 87 | 88 | @property (nonatomic) BOOL readonly; 89 | @property (nonatomic) BOOL demo; 90 | 91 | @property (nonatomic) BOOL preventWriting; 92 | 93 | 94 | @property (nonatomic, readonly) MIModalDatabaseDescriptor *descriptor; 95 | 96 | @property (nonatomic, readonly) NSData *inMemoryIndexData; 97 | 98 | - (NSData *)indexDataFromDisk; 99 | 100 | - (void)saveTrunk; 101 | - (BOOL)saveTrunkIfNecessary; 102 | 103 | - (void)lock; 104 | 105 | - (void)syncDispatch:(void (^)(void))block; 106 | - (void)asyncDispatch:(void (^)(void))block; 107 | - (id)syncDispatchReturn:(id (^)(void))block; 108 | - (void)syncIfPossibleOrAsync:(void (^)(void))block; 109 | 110 | - (void)_rewriteIndexPayloadWithAllMetadataFlag; 111 | 112 | //- (NSString *)attachmentPathWithUUID:(NSString *)uuid; 113 | //- (NSString *)iconPathWithUUID:(NSString *)uuid; 114 | 115 | - (NSString *)iconFullPathWithUUID:(NSString *)uuid; 116 | 117 | - (NSDate *)storeCreatedDate; 118 | 119 | @end 120 | 121 | 122 | extern NSString *const MIStoreDidUpdateList; 123 | extern NSString *const MIStoreDidUpdateItems; 124 | extern NSString *const MIStoreDidAddItem; 125 | extern NSString *const MIStoreDidCompleteMergingMetadata; 126 | extern NSString *const MIStoreDidUpdateTags; 127 | 128 | 129 | -------------------------------------------------------------------------------- /MIStore.m: -------------------------------------------------------------------------------- 1 | // 2 | // MIStore.m 3 | // Elpass iOS 4 | // 5 | // Created by Blankwonder on 2019/8/16. 6 | // Copyright © 2019 Surge Networks. All rights reserved. 7 | // 8 | 9 | #import "MIStore.h" 10 | #import "MessagePack.h" 11 | #import 12 | #import "NSURL+KKDomain.h" 13 | #import "NSString+KKDomain.h" 14 | #import "MIStore+Private.h" 15 | #import "MIStore+Metadata.h" 16 | #import "MIEncryption.h" 17 | 18 | NSError *MIStoreError(NSString *message, NSInteger code) { 19 | return [NSError errorWithDomain:MIStoreErrorDomain code:code userInfo:@{NSLocalizedDescriptionKey: message}]; 20 | } 21 | 22 | @implementation MIStore { 23 | KDGCDTimer *_trunkSaveTimer; 24 | 25 | MIStoreState _state; 26 | } 27 | 28 | - (instancetype)initWithPersistentStore:(MIPersistentStoreDriver *)driver { 29 | self = [super init]; 30 | 31 | _driver = driver; 32 | 33 | _queue = dispatch_queue_create(NSStringFromClass(self.class).UTF8String, DISPATCH_QUEUE_SERIAL); 34 | const void *key = (__bridge const void *)(_queue); 35 | dispatch_queue_set_specific (_queue, key, (void *)key, NULL); 36 | 37 | return self; 38 | } 39 | 40 | - (BOOL)loadDatabaseWithError:(NSError **)errorPtr { 41 | KDAssert(errorPtr); 42 | 43 | __block NSError *error; 44 | 45 | BOOL success = [(NSNumber *)[self syncDispatchReturn:^id{ 46 | // BOOL isDirectory; 47 | // if (![_driver fileExistsAtPath:path isDirectory:&isDirectory]) { 48 | // error = MIStoreError(@"File doesn't exist.", 1); 49 | // [self setState:MIStoreStateDamaged]; 50 | // 51 | // return @NO; 52 | // } 53 | // 54 | // if (!isDirectory) { 55 | // error = MIStoreError(@"The vault isn't a directory.", 2); 56 | // [self setState:MIStoreStateDamaged]; 57 | // 58 | // return @NO; 59 | // } 60 | 61 | NSString *indexPath = @"Index"; 62 | 63 | if (![_driver fileExistsAtPath:indexPath directory:nil]) { 64 | error = MIStoreError(@"The vault is damaged.", 3); 65 | [self setState:MIStoreStateDamaged]; 66 | 67 | return @NO; 68 | } 69 | 70 | NSData *data = [_driver readDataAtPath:indexPath directory:nil]; 71 | if (data.length == 0) { 72 | error = MIStoreError(@"Failed to open database.", 4); 73 | [self setState:MIStoreStateDamaged]; 74 | 75 | return @NO; 76 | } 77 | _inMemoryIndexData = data; 78 | 79 | NSDictionary *index = [MessagePack unpackData:data]; 80 | 81 | int version = [index[@"v"] intValue]; 82 | 83 | _masterPasswordSalt = index[@"s"]; 84 | _encryptedDescriptorData = index[@"d"]; 85 | _encryptedDescriptorDataNonce = index[@"dn"]; 86 | 87 | if (!index[@"amd"]) { 88 | _shouldUpgradeToAllMetaData = YES; 89 | } 90 | 91 | if (_masterPasswordSalt.length != crypto_pwhash_SALTBYTES || 92 | _encryptedDescriptorDataNonce.length != crypto_secretbox_NONCEBYTES || 93 | !_encryptedDescriptorData) { 94 | error = MIStoreError(@"The vault is damaged.", 5); 95 | [self setState:MIStoreStateDamaged]; 96 | 97 | return @NO; 98 | } 99 | 100 | if (version != 1) { 101 | error = MIStoreError(@"Unsupported vault version.", 6); 102 | 103 | [self setState:MIStoreStateUnsupportedVersion]; 104 | 105 | return @NO; 106 | } 107 | 108 | [self setState:MIStoreStateLocked]; 109 | 110 | return @YES; 111 | }] boolValue]; 112 | 113 | *errorPtr = error; 114 | 115 | return success; 116 | } 117 | 118 | - (NSData *)indexDataFromDisk { 119 | return [_driver readDataAtPath:@"Index" directory:nil]; 120 | } 121 | 122 | - (NSData *)textPasswordToData:(NSString *)password salt:(NSData *)salt { 123 | unsigned char key[crypto_secretbox_KEYBYTES]; 124 | 125 | if (crypto_pwhash(key, sizeof key, password.UTF8String, password.length, salt.bytes, 126 | crypto_pwhash_OPSLIMIT_SENSITIVE, crypto_pwhash_MEMLIMIT_MODERATE, 127 | crypto_pwhash_ALG_DEFAULT) != 0) { 128 | KDClassLog(@"crypto_pwhash failed!"); 129 | return nil; 130 | } 131 | 132 | return [NSData dataWithBytes:key length:crypto_secretbox_KEYBYTES]; 133 | } 134 | 135 | 136 | - (MIStoreTrunkData *)loadTrunkDataFromDisk { 137 | NSData *encryptedTrunkData = [_driver readDataAtPath:@"Trunk" directory:nil]; 138 | 139 | if (!encryptedTrunkData) { 140 | return nil; 141 | } 142 | 143 | NSData *decryptedTrunk = [encryptedTrunkData secretboxOpenWithKey:[self deriveKeyWithSubkeyID:MIStoreSubkeyIDTrunk size:crypto_secretbox_KEYBYTES]]; 144 | 145 | if (!decryptedTrunk) { 146 | KDClassLog(@"Failed to decrypt trunk data"); 147 | return nil; 148 | } 149 | 150 | NSDictionary *trunk = [MessagePack unpackData:decryptedTrunk]; 151 | 152 | NSMutableDictionary *itemMap = [NSMutableDictionary dictionary]; 153 | 154 | for (NSDictionary *i in trunk[@"items"]) { 155 | MIItem *item = [MIItem deserializeFromDictionary:i]; 156 | item.store = self; 157 | 158 | if (item) { 159 | itemMap[item.uuid] = item; 160 | } else { 161 | KDClassLog(@"Failed to deserialize obj: %@", i); 162 | } 163 | } 164 | 165 | MIStoreTrunkData *data = [[MIStoreTrunkData alloc] init]; 166 | data.itemMap = itemMap; 167 | [data rebuildCategoryArray]; 168 | 169 | return data; 170 | } 171 | 172 | - (void)unlockWithMasterKey:(NSData *)key completionHandler:(void (^)(BOOL success, NSError *error))completionHandler { 173 | [self syncDispatch:^{ 174 | if (_state == MIStoreStateUnlocked) { 175 | completionHandler(YES, nil); 176 | return; 177 | } 178 | 179 | NSData *decrypted = [_encryptedDescriptorData secretboxOpenWithKey:key nonce:_encryptedDescriptorDataNonce]; 180 | if (!decrypted) { 181 | KDClassLog(@"Datebase failed to unlocked"); 182 | completionHandler(NO, nil); 183 | return ; 184 | } 185 | 186 | NSDictionary *dic = [MessagePack unpackData:decrypted]; 187 | 188 | if (!dic) { 189 | KDClassLog(@"Failed to unpack data"); 190 | [self setState:MIStoreStateDamaged]; 191 | completionHandler(NO, MIStoreError(@"Failed to unpack data", 31)); 192 | return; 193 | } 194 | 195 | MIModalDatabaseDescriptor *descriptor = [MIModalDatabaseDescriptor yy_modelWithDictionary:dic]; 196 | 197 | _descriptor = descriptor; 198 | 199 | KDClassLog(@"Datebase unlocked: %@", descriptor.databaseUUID); 200 | 201 | MIStoreTrunkData *trunk = [self loadTrunkDataFromDisk]; 202 | 203 | if (!trunk) { 204 | KDClassLog(@"Failed to load trunk data"); 205 | [self setState:MIStoreStateDamaged]; 206 | 207 | completionHandler(NO, MIStoreError(@"Failed to load trunk data", 32)); 208 | 209 | return; 210 | } 211 | 212 | _trunk = trunk; 213 | [self updateLastUpdateTimestampForTrunkItemMap]; 214 | [self setState:MIStoreStateUnlocked]; 215 | KDClassLog(@"Item count: %ld, last updated at: %@", _trunk.itemMap.count, [NSDate dateWithTimeIntervalSince1970:_lastUpdatedAt]); 216 | 217 | completionHandler(YES, nil); 218 | 219 | }]; 220 | } 221 | 222 | - (void)lock { 223 | [self syncDispatch:^{ 224 | KDAssert(_state == MIStoreStateUnlocked); 225 | KDClassLog(@"Lock"); 226 | 227 | _trunk = nil; 228 | _descriptor = nil; 229 | [self setState:MIStoreStateLocked]; 230 | }]; 231 | } 232 | 233 | - (void)unlockWithMasterPassword:(NSString *)password completionHandler:(void (^)(BOOL success, NSError *error, NSData *key))completionHandler{ 234 | dispatch_async( dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT,0),^{ 235 | CFAbsoluteTime start = CFAbsoluteTimeGetCurrent(); 236 | 237 | NSData *key = [self textPasswordToData:password salt:_masterPasswordSalt]; 238 | KDClassLog(@"crypto_pwhash in %.0f ms", (CFAbsoluteTimeGetCurrent() - start) * 1000); 239 | [self unlockWithMasterKey:key completionHandler:^(BOOL success, NSError *error) { 240 | completionHandler(success, error, key); 241 | }]; 242 | }); 243 | } 244 | 245 | - (void)verifyMasterPassword:(NSString *)password completionHandler:(void (^)(BOOL success, NSData *key))completionHandler { 246 | dispatch_async( dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT,0),^{ 247 | CFAbsoluteTime start = CFAbsoluteTimeGetCurrent(); 248 | NSData *key = [self textPasswordToData:password salt:_masterPasswordSalt]; 249 | KDClassLog(@"crypto_pwhash in %.0f ms", (CFAbsoluteTimeGetCurrent() - start) * 1000); 250 | 251 | NSData *decrypted = [_encryptedDescriptorData secretboxOpenWithKey:key nonce:_encryptedDescriptorDataNonce]; 252 | if (!decrypted) { 253 | completionHandler(NO, key); 254 | } else { 255 | completionHandler(YES, key); 256 | } 257 | }); 258 | 259 | } 260 | 261 | - (NSData *)deriveKeyWithSubkeyID:(MIStoreSubkeyID)subkeyID size:(size_t)size { 262 | NSMutableData *data = [NSMutableData dataWithLength:size]; 263 | crypto_kdf_derive_from_key(data.mutableBytes, size, subkeyID, "________", _descriptor.masterKey.bytes); 264 | 265 | return data; 266 | } 267 | 268 | - (NSData *)createNewDatabaseWithError:(NSError **)errorPtr masterPassword:(NSString *)password dbuuid:(NSString *)dbuuid { 269 | KDAssert(errorPtr); 270 | KDAssert(dbuuid); 271 | KDAssert(password); 272 | // KDAssert(path); 273 | 274 | // if ([_driver fileExistsAtPath:path]) { 275 | // *errorPtr = MIStoreError(@"File already exists.", 7); 276 | // return nil; 277 | // } 278 | 279 | [_driver createDirectory:nil error:nil]; 280 | 281 | MIModalDatabaseDescriptor *descriptor = [[MIModalDatabaseDescriptor alloc] init]; 282 | 283 | descriptor.createdAt = NSDate.date; 284 | descriptor.databaseUUID = dbuuid; 285 | 286 | KDClassLog(@"Generating a new vault: %@", descriptor.databaseUUID); 287 | 288 | NSMutableData *masterKey = [NSMutableData dataWithLength:crypto_kdf_KEYBYTES]; 289 | randombytes_buf(masterKey.mutableBytes, crypto_kdf_KEYBYTES); 290 | descriptor.masterKey = masterKey; 291 | 292 | _descriptor = descriptor; 293 | 294 | NSData *key = [self changeMasterPassword:password]; 295 | 296 | [self setState:MIStoreStateUnlocked]; 297 | 298 | _trunk = [[MIStoreTrunkData alloc] init]; 299 | 300 | _trunk.logins = [NSMutableArray array]; 301 | _trunk.bankCards = [NSMutableArray array]; 302 | _trunk.secureNotes = [NSMutableArray array]; 303 | _trunk.identifications = [NSMutableArray array]; 304 | 305 | _trunk.itemMap = [NSMutableDictionary dictionary]; 306 | 307 | [self saveTrunk]; 308 | [self rebuildAllMetadataFromTrunk]; 309 | 310 | return key; 311 | } 312 | 313 | 314 | - (NSData *)changeMasterPassword:(NSString *)password { 315 | NSData *plainIndexData = [MessagePack packObject:[_descriptor yy_modelToJSONObject]]; 316 | 317 | NSData *saltData = [NSData securityRandomDataWithLength:crypto_pwhash_SALTBYTES]; 318 | NSData *key = [self textPasswordToData:password salt:saltData]; 319 | NSData *nonceData = [NSData securityRandomDataWithLength:crypto_secretbox_NONCEBYTES]; 320 | NSData *ciphertext = [plainIndexData secretboxWithKey:key nonce:nonceData]; 321 | 322 | _encryptedDescriptorData = ciphertext; 323 | _encryptedDescriptorDataNonce = nonceData; 324 | _masterPasswordSalt = saltData; 325 | 326 | NSDictionary *indexPayload = @{@"d": ciphertext, 327 | @"dn": nonceData, 328 | @"s": saltData, 329 | @"v": @1, 330 | @"amd": @YES 331 | }; 332 | 333 | _inMemoryIndexData = [MessagePack packObject:indexPayload]; 334 | 335 | if (_shouldUpgradeToAllMetaData) { 336 | KDClassLog(@"changeMasterPassword with _shouldUpgradeToAllMetaData set"); 337 | _shouldUpgradeToAllMetaData = NO; 338 | [self rebuildAllMetadataFromTrunk]; 339 | } 340 | 341 | BOOL res = [_driver writeData:_inMemoryIndexData toPath:@"Index" directory:nil error:nil]; 342 | if (!res) { 343 | KDClassLog(@"Failed to write index file!"); 344 | } 345 | 346 | return key; 347 | } 348 | 349 | 350 | 351 | - (void)_rewriteIndexPayloadWithAllMetadataFlag { 352 | NSMutableDictionary *indexPayload = [[MessagePack unpackData:_inMemoryIndexData] mutableCopy]; 353 | KDAssert(indexPayload.count != 0); 354 | indexPayload[@"amd"] = @1; 355 | _inMemoryIndexData = [MessagePack packObject:indexPayload]; 356 | 357 | BOOL res = [_driver writeData:_inMemoryIndexData toPath:@"Index" directory:nil error:nil]; 358 | if (!res) { 359 | KDClassLog(@"Failed to write index file!"); 360 | } 361 | } 362 | 363 | 364 | - (void)saveTrunk { 365 | [self syncDispatch:^{ 366 | if (_preventWriting) { 367 | KDClassLog(@"Try to save trunk while _preventWriting = YES!"); 368 | return; 369 | } 370 | 371 | [_trunkSaveTimer invalidate]; 372 | _trunkSaveTimer = nil; 373 | 374 | KDClassLog(@"saveTrunk"); 375 | NSArray *items = [_trunk.itemMap.allValues KD_arrayUsingMapEnumerateBlock:^id(MIItem *obj, NSUInteger idx) { 376 | return [obj jsonDictionaryForStore]; 377 | }]; 378 | 379 | NSDictionary *trunk = @{@"items": items, @"trunkUpdatedAt": @(time(NULL))}; 380 | 381 | NSData *unencryptedData = [MessagePack packObject:trunk]; 382 | NSData *key = [self deriveKeyWithSubkeyID:MIStoreSubkeyIDTrunk size:crypto_secretbox_KEYBYTES]; 383 | NSData *ciphertext = [unencryptedData secretboxWithKey:key]; 384 | 385 | 386 | NSError *error = nil; 387 | BOOL success = [_driver writeData:ciphertext toPath:@"Trunk" directory:nil error:&error]; 388 | KDLoggerPrintError(error); 389 | 390 | if (!success) { 391 | MIEncounterPanicError(error); 392 | } 393 | }]; 394 | } 395 | 396 | - (BOOL)saveTrunkIfNecessary { 397 | __block BOOL saved = NO; 398 | [self syncDispatch:^{ 399 | [_trunkSaveTimer invalidate]; 400 | _trunkSaveTimer = nil; 401 | 402 | MIStoreTrunkData *diskTrunkData = [self loadTrunkDataFromDisk]; 403 | 404 | if (![diskTrunkData.itemMap isEqualToDictionary:_trunk.itemMap]) { 405 | KDClassLog(@"Trunk need updates"); 406 | 407 | #if DEBUG 408 | KDDebuggerPrintDictionaryDiff(diskTrunkData.itemMap, _trunk.itemMap); 409 | #endif 410 | 411 | [self saveTrunk]; 412 | saved = YES; 413 | } else { 414 | KDClassLog(@"Trunk on disk is up to date"); 415 | } 416 | 417 | }]; 418 | 419 | return saved; 420 | } 421 | 422 | - (BOOL)isStoreFilesExist { 423 | if (![_driver fileExistsAtPath:@"Index" directory:nil]) return NO; 424 | if (![_driver fileExistsAtPath:@"Trunk" directory:nil]) return NO; 425 | return YES; 426 | } 427 | 428 | 429 | - (void)scheduleSaveTrunk { 430 | KDClassLog(@"scheduleSaveTrunk"); 431 | 432 | [_trunkSaveTimer invalidate]; 433 | _trunkSaveTimer = [KDGCDTimer onetimeTimerWithQueue:_queue after:0.5 handler:^{ 434 | [self saveTrunkIfNecessary]; 435 | }]; 436 | } 437 | 438 | 439 | - (BOOL)isOnSelfDispatchQueue { 440 | const void *key = (__bridge const void *)(_queue); 441 | void *res = dispatch_get_specific(key); 442 | return res == key; 443 | } 444 | 445 | - (void)syncDispatch:(void (^)(void))block { 446 | if ([self isOnSelfDispatchQueue]) { 447 | block(); 448 | } else { 449 | dispatch_sync(_queue, ^{ 450 | @autoreleasepool { 451 | block(); 452 | } 453 | }); 454 | } 455 | } 456 | 457 | - (id)syncDispatchReturn:(id (^)(void))block { 458 | if ([self isOnSelfDispatchQueue]) { 459 | return block(); 460 | } else { 461 | __block id result = nil; 462 | dispatch_sync(_queue, ^{ 463 | @autoreleasepool { 464 | result = block(); 465 | } 466 | }); 467 | return result; 468 | } 469 | } 470 | 471 | 472 | - (void)asyncDispatch:(void (^)(void))block { 473 | dispatch_async(_queue, ^{ 474 | @autoreleasepool { 475 | block(); 476 | } 477 | }); 478 | } 479 | 480 | - (void)syncIfPossibleOrAsync:(void (^)(void))block { 481 | if ([self isOnSelfDispatchQueue]) { 482 | block(); 483 | } else { 484 | dispatch_async(_queue, ^{ 485 | @autoreleasepool { 486 | block(); 487 | } 488 | }); 489 | } 490 | } 491 | 492 | 493 | - (void)setState:(MIStoreState)state { 494 | KDClassLog(@"State changed: %d -> %d", _state, state); 495 | _state = state; 496 | } 497 | 498 | - (MIStoreState)state { 499 | __block MIStoreState state; 500 | 501 | [self syncDispatch:^{ 502 | state = _state; 503 | }]; 504 | 505 | return state; 506 | } 507 | 508 | //- (NSString *)attachmentPathWithUUID:(NSString *)uuid { 509 | // NSString *dirPath = @"Attachments"; 510 | // 511 | // [KDStorageHelper mkdirIfNecessary:dirPath]; 512 | // 513 | // return [dirPath stringByAppendingPathComponent:uuid]; 514 | //} 515 | 516 | - (NSString *)iconFullPathWithUUID:(NSString *)uuid { 517 | return [[self.databasePath stringByAppendingPathComponent:@"Icons"] stringByAppendingPathComponent:uuid]; 518 | } 519 | 520 | - (NSDate *)storeCreatedDate { 521 | return [_driver fileCreateDateAtPath:@"Index" directory:nil]; 522 | } 523 | 524 | - (void)setPreventWriting:(BOOL)preventWriting { 525 | [self syncDispatch:^{ 526 | _preventWriting = preventWriting; 527 | }]; 528 | } 529 | 530 | - (NSString *)databasePath { 531 | return [(MIPersistentStoreDriverFilesystem *)_driver basePath]; 532 | } 533 | 534 | - (void)updateLastUpdateTimestampForTrunkItemMap { 535 | __block MITimestamp lastTimestamp = 0; 536 | 537 | [_trunk.itemMap enumerateKeysAndObjectsUsingBlock:^(NSString * _Nonnull key, MIItem * _Nonnull obj, BOOL * _Nonnull stop) { 538 | if (obj.updatedAt > lastTimestamp) { 539 | lastTimestamp = obj.updatedAt; 540 | } 541 | }]; 542 | 543 | _lastUpdatedAt = lastTimestamp; 544 | } 545 | 546 | @end 547 | 548 | 549 | @implementation MIStoreTrunkData 550 | 551 | - (void)rebuildCategoryArray { 552 | self.logins = [NSMutableArray array]; 553 | self.bankCards = [NSMutableArray array]; 554 | self.secureNotes = [NSMutableArray array]; 555 | self.identifications = [NSMutableArray array]; 556 | self.passwords = [NSMutableArray array]; 557 | self.softwareLicenses = [NSMutableArray array]; 558 | self.bankAccounts = [NSMutableArray array]; 559 | 560 | [self.itemMap enumerateKeysAndObjectsUsingBlock:^(NSString * _Nonnull key, MIItem * _Nonnull obj, BOOL * _Nonnull stop) { 561 | [[self itemArrayForClass:obj.class] addObject:obj]; 562 | }]; 563 | } 564 | 565 | 566 | - (NSMutableArray *)itemArrayForClass:(Class)class { 567 | if (class == MILoginItem.class) return self.logins; 568 | if (class == MIBankCardItem.class) return self.bankCards; 569 | if (class == MISecureNoteItem.class) return self.secureNotes; 570 | if (class == MIIdentificationItem.class) return self.identifications; 571 | if (class == MIPasswordItem.class) return self.passwords; 572 | if (class == MISoftwareLicenseItem.class) return self.softwareLicenses; 573 | if (class == MIBankAccountItem.class) return self.bankAccounts; 574 | 575 | if (class == MIPlaceholderItem.class) return nil; 576 | 577 | KDUtilThrowNoImplementationException 578 | } 579 | 580 | 581 | @end 582 | 583 | 584 | NSString *const MIStoreDidUpdateList = @"MIStoreDidUpdateList"; 585 | NSString *const MIStoreDidUpdateItems = @"MIStoreDidUpdateItems"; 586 | NSString *const MIStoreDidAddItem = @"MIStoreDidAddItem"; 587 | NSString *const MIStoreDidCompleteMergingMetadata = @"MIStoreDidCompleteMergingMetadata"; 588 | NSString *const MIStoreDidUpdateTags = @"MIStoreDidUpdateTags"; 589 | 590 | NSString *const MIStoreErrorDomain = @"MIStoreErrorDomain"; 591 | --------------------------------------------------------------------------------