├── .gitattributes ├── .gitignore ├── LICENSE ├── README.md ├── Resources └── Icon128.png ├── RuntimeArchiver.uplugin └── Source ├── RuntimeArchiver ├── Private │ ├── ArchiverGZip │ │ └── RuntimeArchiverGZip.cpp │ ├── ArchiverLZ4 │ │ └── RuntimeArchiverLZ4.cpp │ ├── ArchiverOodle │ │ └── RuntimeArchiverOodle.cpp │ ├── ArchiverRaw │ │ └── RuntimeArchiverRaw.cpp │ ├── ArchiverTar │ │ ├── RuntimeArchiverTar.cpp │ │ ├── RuntimeArchiverTarHeader.cpp │ │ ├── RuntimeArchiverTarHeader.h │ │ └── RuntimeArchiverTarOperations.h │ ├── ArchiverZip │ │ ├── RuntimeArchiverZip.cpp │ │ └── RuntimeArchiverZipIncludes.h │ ├── AsyncTasks │ │ ├── RuntimeArchiverArchiveAsyncTask.cpp │ │ └── RuntimeArchiverUnarchiveAsyncTask.cpp │ ├── RuntimeArchiver.cpp │ ├── RuntimeArchiverBase.cpp │ ├── RuntimeArchiverSubsystem.cpp │ ├── RuntimeArchiverUtilities.cpp │ └── Streams │ │ ├── RuntimeArchiverFileStream.cpp │ │ └── RuntimeArchiverMemoryStream.cpp ├── Public │ ├── ArchiverGZip │ │ └── RuntimeArchiverGZip.h │ ├── ArchiverLZ4 │ │ └── RuntimeArchiverLZ4.h │ ├── ArchiverOodle │ │ └── RuntimeArchiverOodle.h │ ├── ArchiverRaw │ │ └── RuntimeArchiverRaw.h │ ├── ArchiverTar │ │ └── RuntimeArchiverTar.h │ ├── ArchiverZip │ │ └── RuntimeArchiverZip.h │ ├── AsyncTasks │ │ ├── RuntimeArchiverArchiveAsyncTask.h │ │ └── RuntimeArchiverUnarchiveAsyncTask.h │ ├── RuntimeArchiver.h │ ├── RuntimeArchiverBase.h │ ├── RuntimeArchiverDefines.h │ ├── RuntimeArchiverSubsystem.h │ ├── RuntimeArchiverTypes.h │ ├── RuntimeArchiverUtilities.h │ └── Streams │ │ ├── RuntimeArchiverBaseStream.h │ │ ├── RuntimeArchiverFileStream.h │ │ └── RuntimeArchiverMemoryStream.h └── RuntimeArchiver.Build.cs └── ThirdParty └── miniz ├── miniz.c ├── miniz.h ├── miniz_common.h ├── miniz_export.h ├── miniz_tdef.c ├── miniz_tdef.h ├── miniz_tinfl.c ├── miniz_tinfl.h ├── miniz_zip.c └── miniz_zip.h /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled source files for the engine to use 2 | /Intermediate/ 3 | 4 | # Binary Files 5 | /Binaries/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Georgy Treshchev 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | **Important Notice:** 2 | This open-source version of the plugin is no longer maintained. It may contain bugs and lack some features. I recommend using the **Fab version** for the most up-to-date features, bug fixes, and ongoing support. The Fab version is available [here](https://www.fab.com/listings/9bc46d5b-b9e1-4b93-aede-194619108265). 3 | 4 | For support or questions, feel free to join the [Discord chat](https://georgy.dev/discord). -------------------------------------------------------------------------------- /Resources/Icon128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gtreshchev/RuntimeArchiver/f8ee8cf26565c714c2d96982a739c3ec93f89d10/Resources/Icon128.png -------------------------------------------------------------------------------- /RuntimeArchiver.uplugin: -------------------------------------------------------------------------------- 1 | { 2 | "FileVersion": 3, 3 | "Version": 1, 4 | "VersionName": "1.0", 5 | "FriendlyName": "Runtime Archiver", 6 | "Description": "Runtime Archiver plugin for Unreal Engine. Cross-platform archiving and unarchiving directories and files. Supports Zip, Oodle, Tar, LZ4 and GZip formats", 7 | "Category": "Other", 8 | "CreatedBy": "Georgy Treshchev", 9 | "CreatedByURL": "https://georgy.dev/", 10 | "DocsURL": "https://docs.georgy.dev/runtime-archiver/overview", 11 | "MarketplaceURL": "com.epicgames.launcher://ue/marketplace/product/2f22e9adc1dc4254a277aabb2320f3f9", 12 | "SupportURL": "https://georgy.dev/", 13 | "CanContainContent": true, 14 | "IsBetaVersion": false, 15 | "IsExperimentalVersion": false, 16 | "Installed": true, 17 | "Modules": [ 18 | { 19 | "Name": "RuntimeArchiver", 20 | "Type": "Runtime", 21 | "LoadingPhase": "Default" 22 | } 23 | ] 24 | } -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Private/ArchiverGZip/RuntimeArchiverGZip.cpp: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #include "ArchiverGZip/RuntimeArchiverGZip.h" 4 | #include "RuntimeArchiverDefines.h" 5 | #include "RuntimeArchiverUtilities.h" 6 | #include "ArchiverRaw/RuntimeArchiverRaw.h" 7 | #include "ArchiverTar/RuntimeArchiverTar.h" 8 | #include "Streams/RuntimeArchiverFileStream.h" 9 | #include "Streams/RuntimeArchiverMemoryStream.h" 10 | 11 | URuntimeArchiverGZip::URuntimeArchiverGZip() 12 | : LastCompressionLevel{ERuntimeArchiverCompressionLevel::Compression6} 13 | { 14 | } 15 | 16 | bool URuntimeArchiverGZip::CreateArchiveInStorage(FString ArchivePath) 17 | { 18 | if (!Super::CreateArchiveInStorage(ArchivePath)) 19 | { 20 | return false; 21 | } 22 | 23 | CompressedStream.Reset(new FRuntimeArchiverFileStream(ArchivePath, true)); 24 | if (!CompressedStream->IsValid()) 25 | { 26 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open gzip stream because it is not valid")); 27 | Reset(); 28 | return false; 29 | } 30 | 31 | if (!TarArchiver->CreateArchiveInMemory(0)) 32 | { 33 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to create gzip archive in storage '%s' due to tar archiver error"), *ArchivePath); 34 | Reset(); 35 | return false; 36 | } 37 | 38 | return true; 39 | } 40 | 41 | bool URuntimeArchiverGZip::CreateArchiveInMemory(int32 InitialAllocationSize) 42 | { 43 | if (!Super::CreateArchiveInMemory(InitialAllocationSize)) 44 | { 45 | return false; 46 | } 47 | 48 | CompressedStream.Reset(new FRuntimeArchiverMemoryStream(InitialAllocationSize)); 49 | if (!CompressedStream->IsValid()) 50 | { 51 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open gzip stream because it is not valid")); 52 | Reset(); 53 | return false; 54 | } 55 | 56 | if (!TarArchiver->CreateArchiveInMemory(InitialAllocationSize)) 57 | { 58 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to create gzip archive in memory due to tar archiver error")); 59 | Reset(); 60 | return false; 61 | } 62 | 63 | return true; 64 | } 65 | 66 | bool URuntimeArchiverGZip::OpenArchiveFromStorage(FString ArchivePath) 67 | { 68 | if (!Super::OpenArchiveFromStorage(ArchivePath)) 69 | { 70 | return false; 71 | } 72 | 73 | CompressedStream.Reset(new FRuntimeArchiverFileStream(ArchivePath, false)); 74 | if (!CompressedStream->IsValid()) 75 | { 76 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open gzip stream because it is not valid")); 77 | Reset(); 78 | return false; 79 | } 80 | 81 | TArray64 CompressedArchiveData; 82 | CompressedArchiveData.SetNumUninitialized(CompressedStream->Size()); 83 | 84 | if (!CompressedStream->Read(CompressedArchiveData.GetData(), CompressedArchiveData.Num())) 85 | { 86 | ReportError(ERuntimeArchiverErrorCode::GetError, TEXT("Unable to read gzip compressed stream to get archive data")); 87 | Reset(); 88 | return false; 89 | } 90 | 91 | TArray64 TarArchiveData; 92 | if (!URuntimeArchiverRaw::UncompressRawData(ERuntimeArchiverRawFormat::GZip, MoveTemp(CompressedArchiveData), TarArchiveData)) 93 | { 94 | ReportError(ERuntimeArchiverErrorCode::GetError, FString::Printf(TEXT("Unable to uncompress gzip to tar data to open archive from storage '%s'"), *ArchivePath)); 95 | Reset(); 96 | return false; 97 | } 98 | 99 | if (!TarArchiver->OpenArchiveFromMemory(TarArchiveData)) 100 | { 101 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open gzip archive from storage due to tar archiver error")); 102 | Reset(); 103 | return false; 104 | } 105 | 106 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully opened gzip archive '%s' in '%s' to read"), *GetName(), *ArchivePath); 107 | return true; 108 | } 109 | 110 | bool URuntimeArchiverGZip::OpenArchiveFromMemory(const TArray64& ArchiveData) 111 | { 112 | if (!Super::OpenArchiveFromMemory(ArchiveData)) 113 | { 114 | return false; 115 | } 116 | 117 | CompressedStream.Reset(new FRuntimeArchiverMemoryStream(ArchiveData)); 118 | if (!CompressedStream->IsValid()) 119 | { 120 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open gzip stream because it is not valid")); 121 | Reset(); 122 | return false; 123 | } 124 | 125 | TArray64 TarArchiveData; 126 | if (!URuntimeArchiverRaw::UncompressRawData(ERuntimeArchiverRawFormat::GZip, ArchiveData, TarArchiveData)) 127 | { 128 | ReportError(ERuntimeArchiverErrorCode::GetError, TEXT("Unable to uncompress gzip to tar data to open archive from memory")); 129 | Reset(); 130 | return false; 131 | } 132 | 133 | if (!TarArchiver->OpenArchiveFromMemory(TarArchiveData)) 134 | { 135 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open gzip archive from memory due to tar archiver error")); 136 | Reset(); 137 | return false; 138 | } 139 | 140 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully opened in-memory gzip archive '%s' to read"), *GetName()); 141 | return true; 142 | } 143 | 144 | bool URuntimeArchiverGZip::CloseArchive() 145 | { 146 | if (!Super::CloseArchive()) 147 | { 148 | return false; 149 | } 150 | 151 | if (Mode == ERuntimeArchiverMode::Write && Location == ERuntimeArchiverLocation::Storage) 152 | { 153 | TArray64 ArchiveData; 154 | if (!GetArchiveData(ArchiveData)) 155 | { 156 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to get archive data to close archive")); 157 | return false; 158 | } 159 | 160 | if (!CompressedStream->Seek(0)) 161 | { 162 | ReportError(ERuntimeArchiverErrorCode::CloseError, TEXT("Unable to seek first position in gzip archive to close archive")); 163 | return false; 164 | } 165 | 166 | if (!CompressedStream->Write(ArchiveData.GetData(), ArchiveData.Num())) 167 | { 168 | ReportError(ERuntimeArchiverErrorCode::CloseError, TEXT("Unable to write archive data to gzip compressed stream to close archive")); 169 | return false; 170 | } 171 | } 172 | 173 | if (!TarArchiver->CloseArchive()) 174 | { 175 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to close gzip archive due to tar archiver error")); 176 | return false; 177 | } 178 | 179 | Reset(); 180 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully closed gzip archive '%s'"), *GetName()); 181 | return true; 182 | } 183 | 184 | bool URuntimeArchiverGZip::GetArchiveData(TArray64& ArchiveData) 185 | { 186 | if (!Super::GetArchiveData(ArchiveData)) 187 | { 188 | return false; 189 | } 190 | 191 | if (Mode == ERuntimeArchiverMode::Read) 192 | { 193 | ArchiveData.SetNumUninitialized(CompressedStream->Size()); 194 | 195 | if (!CompressedStream->Seek(0)) 196 | { 197 | ReportError(ERuntimeArchiverErrorCode::GetError, TEXT("Unable to seek first position in gzip archive to get archive data")); 198 | return false; 199 | } 200 | 201 | if (!CompressedStream->Read(ArchiveData.GetData(), ArchiveData.Num())) 202 | { 203 | ReportError(ERuntimeArchiverErrorCode::GetError, TEXT("Unable to read gzip compressed stream to get archive data")); 204 | return false; 205 | } 206 | } 207 | else if (Mode == ERuntimeArchiverMode::Write) 208 | { 209 | TArray64 TarArchiveData; 210 | if (!TarArchiver->GetArchiveData(TarArchiveData)) 211 | { 212 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to get gzip archive data due to tar archiver error")); 213 | return false; 214 | } 215 | 216 | if (!URuntimeArchiverRaw::CompressRawData(ERuntimeArchiverRawFormat::GZip, LastCompressionLevel, TarArchiveData, ArchiveData)) 217 | { 218 | ReportError(ERuntimeArchiverErrorCode::GetError, TEXT("Unable to compress tar to gzip data to get archive data")); 219 | return false; 220 | } 221 | } 222 | 223 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully retrieved gzip archive data from memory with size '%lld'"), ArchiveData.Num()); 224 | return true; 225 | } 226 | 227 | bool URuntimeArchiverGZip::GetArchiveEntries(int32& NumOfArchiveEntries) 228 | { 229 | if (!Super::GetArchiveEntries(NumOfArchiveEntries)) 230 | { 231 | return false; 232 | } 233 | 234 | if (!TarArchiver->GetArchiveEntries(NumOfArchiveEntries)) 235 | { 236 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to the get number of gzip entries due to tar archiver error")); 237 | return false; 238 | } 239 | 240 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully retrieved %d gzip entries"), NumOfArchiveEntries); 241 | return true; 242 | } 243 | 244 | bool URuntimeArchiverGZip::GetArchiveEntryInfoByName(FString EntryName, FRuntimeArchiveEntry& EntryInfo) 245 | { 246 | if (!Super::GetArchiveEntryInfoByName(EntryName, EntryInfo)) 247 | { 248 | return false; 249 | } 250 | 251 | if (!TarArchiver->GetArchiveEntryInfoByName(MoveTemp(EntryName), EntryInfo)) 252 | { 253 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to get gzip entry by name due to tar archiver error")); 254 | return false; 255 | } 256 | 257 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully retrieved gzip entry '%s' by name"), *EntryInfo.Name); 258 | return true; 259 | } 260 | 261 | bool URuntimeArchiverGZip::GetArchiveEntryInfoByIndex(int32 EntryIndex, FRuntimeArchiveEntry& EntryInfo) 262 | { 263 | if (!Super::GetArchiveEntryInfoByIndex(EntryIndex, EntryInfo)) 264 | { 265 | return false; 266 | } 267 | 268 | if (!TarArchiver->GetArchiveEntryInfoByIndex(EntryIndex, EntryInfo)) 269 | { 270 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to get gzip entry by index due to tar archiver error")); 271 | return false; 272 | } 273 | 274 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully retrieved gzip entry '%s' by index"), *EntryInfo.Name); 275 | return true; 276 | } 277 | 278 | bool URuntimeArchiverGZip::AddEntryFromMemory(FString EntryName, const TArray64& DataToBeArchived, ERuntimeArchiverCompressionLevel CompressionLevel) 279 | { 280 | if (!Super::AddEntryFromMemory(EntryName, DataToBeArchived, CompressionLevel)) 281 | { 282 | return false; 283 | } 284 | 285 | if (!TarArchiver->AddEntryFromMemory(EntryName, DataToBeArchived, CompressionLevel)) 286 | { 287 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to add gzip entry due to tar archiver error")); 288 | return false; 289 | } 290 | 291 | LastCompressionLevel = CompressionLevel; 292 | 293 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully added gzip entry '%s' with size %lld bytes from memory"), *EntryName, DataToBeArchived.Num()); 294 | return true; 295 | } 296 | 297 | bool URuntimeArchiverGZip::ExtractEntryToMemory(const FRuntimeArchiveEntry& EntryInfo, TArray64& UnarchivedData) 298 | { 299 | if (!Super::ExtractEntryToMemory(EntryInfo, UnarchivedData)) 300 | { 301 | return false; 302 | } 303 | 304 | if (!TarArchiver->ExtractEntryToMemory(EntryInfo, UnarchivedData)) 305 | { 306 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to extract gzip entry due to tar archiver error")); 307 | return false; 308 | } 309 | 310 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully extracted gzip entry '%s' into memory"), *EntryInfo.Name); 311 | return true; 312 | } 313 | 314 | bool URuntimeArchiverGZip::Initialize() 315 | { 316 | if (!Super::Initialize()) 317 | { 318 | return false; 319 | } 320 | 321 | TarArchiver.Reset(Cast(CreateRuntimeArchiver(this, URuntimeArchiverTar::StaticClass()))); 322 | if (!TarArchiver.IsValid()) 323 | { 324 | ReportError(ERuntimeArchiverErrorCode::NotInitialized, TEXT("Unable to allocate memory for gzip archiver")); 325 | return false; 326 | } 327 | 328 | return true; 329 | } 330 | 331 | bool URuntimeArchiverGZip::IsInitialized() const 332 | { 333 | return Super::IsInitialized() && TarArchiver.IsValid() && TarArchiver->IsInitialized(); 334 | } 335 | 336 | void URuntimeArchiverGZip::Reset() 337 | { 338 | if (TarArchiver.IsValid()) 339 | { 340 | TarArchiver->Reset(); 341 | TarArchiver.Reset(); 342 | } 343 | 344 | CompressedStream.Reset(); 345 | Super::Reset(); 346 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully uninitialized gzip archiver '%s'"), *GetName()); 347 | } 348 | 349 | void URuntimeArchiverGZip::ReportError(ERuntimeArchiverErrorCode ErrorCode, const FString& ErrorString) const 350 | { 351 | Super::ReportError(ErrorCode, ErrorString); 352 | } 353 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Private/ArchiverLZ4/RuntimeArchiverLZ4.cpp: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #include "ArchiverLZ4/RuntimeArchiverLZ4.h" 4 | #include "RuntimeArchiverDefines.h" 5 | #include "RuntimeArchiverUtilities.h" 6 | #include "ArchiverRaw/RuntimeArchiverRaw.h" 7 | #include "ArchiverTar/RuntimeArchiverTar.h" 8 | #include "Streams/RuntimeArchiverFileStream.h" 9 | #include "Streams/RuntimeArchiverMemoryStream.h" 10 | 11 | URuntimeArchiverLZ4::URuntimeArchiverLZ4() 12 | : LastCompressionLevel{ERuntimeArchiverCompressionLevel::Compression6} 13 | { 14 | } 15 | 16 | bool URuntimeArchiverLZ4::CreateArchiveInStorage(FString ArchivePath) 17 | { 18 | if (!Super::CreateArchiveInStorage(ArchivePath)) 19 | { 20 | return false; 21 | } 22 | 23 | CompressedStream.Reset(new FRuntimeArchiverFileStream(ArchivePath, true)); 24 | if (!CompressedStream->IsValid()) 25 | { 26 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open lz4 stream because it is not valid")); 27 | Reset(); 28 | return false; 29 | } 30 | 31 | if (!TarArchiver->CreateArchiveInMemory(0)) 32 | { 33 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to create lz4 archive in storage '%s' due to tar archiver error"), *ArchivePath); 34 | Reset(); 35 | return false; 36 | } 37 | 38 | return true; 39 | } 40 | 41 | bool URuntimeArchiverLZ4::CreateArchiveInMemory(int32 InitialAllocationSize) 42 | { 43 | if (!Super::CreateArchiveInMemory(InitialAllocationSize)) 44 | { 45 | return false; 46 | } 47 | 48 | CompressedStream.Reset(new FRuntimeArchiverMemoryStream(InitialAllocationSize)); 49 | if (!CompressedStream->IsValid()) 50 | { 51 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open lz4 stream because it is not valid")); 52 | Reset(); 53 | return false; 54 | } 55 | 56 | if (!TarArchiver->CreateArchiveInMemory(InitialAllocationSize)) 57 | { 58 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to create lz4 archive in memory due to tar archiver error")); 59 | Reset(); 60 | return false; 61 | } 62 | 63 | return true; 64 | } 65 | 66 | bool URuntimeArchiverLZ4::OpenArchiveFromStorage(FString ArchivePath) 67 | { 68 | if (!Super::OpenArchiveFromStorage(ArchivePath)) 69 | { 70 | return false; 71 | } 72 | 73 | CompressedStream.Reset(new FRuntimeArchiverFileStream(ArchivePath, false)); 74 | if (!CompressedStream->IsValid()) 75 | { 76 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open lz4 stream because it is not valid")); 77 | Reset(); 78 | return false; 79 | } 80 | 81 | TArray64 CompressedArchiveData; 82 | CompressedArchiveData.SetNumUninitialized(CompressedStream->Size()); 83 | 84 | if (!CompressedStream->Read(CompressedArchiveData.GetData(), CompressedArchiveData.Num())) 85 | { 86 | ReportError(ERuntimeArchiverErrorCode::GetError, TEXT("Unable to read lz4 compressed stream to get archive data")); 87 | Reset(); 88 | return false; 89 | } 90 | 91 | TArray64 TarArchiveData; 92 | if (!URuntimeArchiverRaw::UncompressRawData(ERuntimeArchiverRawFormat::LZ4, MoveTemp(CompressedArchiveData), TarArchiveData)) 93 | { 94 | ReportError(ERuntimeArchiverErrorCode::GetError, FString::Printf(TEXT("Unable to uncompress lz4 to tar data to open archive from storage '%s'"), *ArchivePath)); 95 | Reset(); 96 | return false; 97 | } 98 | 99 | if (!TarArchiver->OpenArchiveFromMemory(TarArchiveData)) 100 | { 101 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open lz4 archive from storage due to tar archiver error")); 102 | Reset(); 103 | return false; 104 | } 105 | 106 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully opened lz4 archive '%s' in '%s' to read. Compressed size %lld, uncompressed %lld"), *GetName(), *ArchivePath, TarArchiveData.Num(), CompressedStream->Size()); 107 | return true; 108 | } 109 | 110 | bool URuntimeArchiverLZ4::OpenArchiveFromMemory(const TArray64& ArchiveData) 111 | { 112 | if (!Super::OpenArchiveFromMemory(ArchiveData)) 113 | { 114 | return false; 115 | } 116 | 117 | CompressedStream.Reset(new FRuntimeArchiverMemoryStream(ArchiveData)); 118 | if (!CompressedStream->IsValid()) 119 | { 120 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open lz4 stream because it is not valid")); 121 | Reset(); 122 | return false; 123 | } 124 | 125 | TArray64 TarArchiveData; 126 | if (!URuntimeArchiverRaw::UncompressRawData(ERuntimeArchiverRawFormat::LZ4, ArchiveData, TarArchiveData)) 127 | { 128 | ReportError(ERuntimeArchiverErrorCode::GetError, TEXT("Unable to uncompress lz4 to tar data to open archive from memory")); 129 | Reset(); 130 | return false; 131 | } 132 | 133 | if (!TarArchiver->OpenArchiveFromMemory(TarArchiveData)) 134 | { 135 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open lz4 archive from memory due to tar archiver error")); 136 | Reset(); 137 | return false; 138 | } 139 | 140 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully opened in-memory lz4 archive '%s' to read"), *GetName()); 141 | return true; 142 | } 143 | 144 | bool URuntimeArchiverLZ4::CloseArchive() 145 | { 146 | if (!Super::CloseArchive()) 147 | { 148 | return false; 149 | } 150 | 151 | if (Mode == ERuntimeArchiverMode::Write && Location == ERuntimeArchiverLocation::Storage) 152 | { 153 | TArray64 ArchiveData; 154 | if (!GetArchiveData(ArchiveData)) 155 | { 156 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to get archive data to close archive")); 157 | return false; 158 | } 159 | 160 | if (!CompressedStream->Seek(0)) 161 | { 162 | ReportError(ERuntimeArchiverErrorCode::CloseError, TEXT("Unable to seek first position in lz4 archive to close archive")); 163 | return false; 164 | } 165 | 166 | if (!CompressedStream->Write(ArchiveData.GetData(), ArchiveData.Num())) 167 | { 168 | ReportError(ERuntimeArchiverErrorCode::CloseError, TEXT("Unable to write archive data to lz4 compressed stream to close archive")); 169 | return false; 170 | } 171 | } 172 | 173 | if (!TarArchiver->CloseArchive()) 174 | { 175 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to close lz4 archive due to tar archiver error")); 176 | return false; 177 | } 178 | 179 | Reset(); 180 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully closed lz4 archive '%s'"), *GetName()); 181 | return true; 182 | } 183 | 184 | bool URuntimeArchiverLZ4::GetArchiveData(TArray64& ArchiveData) 185 | { 186 | if (!Super::GetArchiveData(ArchiveData)) 187 | { 188 | return false; 189 | } 190 | 191 | if (Mode == ERuntimeArchiverMode::Read) 192 | { 193 | ArchiveData.SetNumUninitialized(CompressedStream->Size()); 194 | 195 | if (!CompressedStream->Seek(0)) 196 | { 197 | ReportError(ERuntimeArchiverErrorCode::GetError, TEXT("Unable to seek first position in lz4 archive to get archive data")); 198 | return false; 199 | } 200 | 201 | if (!CompressedStream->Read(ArchiveData.GetData(), ArchiveData.Num())) 202 | { 203 | ReportError(ERuntimeArchiverErrorCode::GetError, TEXT("Unable to read lz4 compressed stream to get archive data")); 204 | return false; 205 | } 206 | } 207 | else if (Mode == ERuntimeArchiverMode::Write) 208 | { 209 | TArray64 TarArchiveData; 210 | if (!TarArchiver->GetArchiveData(TarArchiveData)) 211 | { 212 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to get lz4 archive data due to tar archiver error")); 213 | return false; 214 | } 215 | 216 | if (!URuntimeArchiverRaw::CompressRawData(ERuntimeArchiverRawFormat::LZ4, LastCompressionLevel, TarArchiveData, ArchiveData)) 217 | { 218 | ReportError(ERuntimeArchiverErrorCode::GetError, TEXT("Unable to compress tar to lz4 data to get archive data")); 219 | return false; 220 | } 221 | } 222 | 223 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully retrieved lz4 archive data from memory with size '%lld'"), ArchiveData.Num()); 224 | return true; 225 | } 226 | 227 | bool URuntimeArchiverLZ4::GetArchiveEntries(int32& NumOfArchiveEntries) 228 | { 229 | if (!Super::GetArchiveEntries(NumOfArchiveEntries)) 230 | { 231 | return false; 232 | } 233 | 234 | if (!TarArchiver->GetArchiveEntries(NumOfArchiveEntries)) 235 | { 236 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to the get number of lz4 entries due to tar archiver error")); 237 | return false; 238 | } 239 | 240 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully retrieved %d lz4 entries"), NumOfArchiveEntries); 241 | return true; 242 | } 243 | 244 | bool URuntimeArchiverLZ4::GetArchiveEntryInfoByName(FString EntryName, FRuntimeArchiveEntry& EntryInfo) 245 | { 246 | if (!Super::GetArchiveEntryInfoByName(EntryName, EntryInfo)) 247 | { 248 | return false; 249 | } 250 | 251 | if (!TarArchiver->GetArchiveEntryInfoByName(MoveTemp(EntryName), EntryInfo)) 252 | { 253 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to get lz4 entry by name due to tar archiver error")); 254 | return false; 255 | } 256 | 257 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully retrieved lz4 entry '%s' by name"), *EntryInfo.Name); 258 | return true; 259 | } 260 | 261 | bool URuntimeArchiverLZ4::GetArchiveEntryInfoByIndex(int32 EntryIndex, FRuntimeArchiveEntry& EntryInfo) 262 | { 263 | if (!Super::GetArchiveEntryInfoByIndex(EntryIndex, EntryInfo)) 264 | { 265 | return false; 266 | } 267 | 268 | if (!TarArchiver->GetArchiveEntryInfoByIndex(EntryIndex, EntryInfo)) 269 | { 270 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to get lz4 entry by index due to tar archiver error")); 271 | return false; 272 | } 273 | 274 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully retrieved lz4 entry '%s' by index"), *EntryInfo.Name); 275 | return true; 276 | } 277 | 278 | bool URuntimeArchiverLZ4::AddEntryFromMemory(FString EntryName, const TArray64& DataToBeArchived, ERuntimeArchiverCompressionLevel CompressionLevel) 279 | { 280 | if (!Super::AddEntryFromMemory(EntryName, DataToBeArchived, CompressionLevel)) 281 | { 282 | return false; 283 | } 284 | 285 | if (!TarArchiver->AddEntryFromMemory(EntryName, DataToBeArchived, CompressionLevel)) 286 | { 287 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to add lz4 entry due to tar archiver error")); 288 | return false; 289 | } 290 | 291 | LastCompressionLevel = CompressionLevel; 292 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully added lz4 entry '%s' with size %lld bytes from memory"), *EntryName, DataToBeArchived.Num()); 293 | return true; 294 | } 295 | 296 | bool URuntimeArchiverLZ4::ExtractEntryToMemory(const FRuntimeArchiveEntry& EntryInfo, TArray64& UnarchivedData) 297 | { 298 | if (!Super::ExtractEntryToMemory(EntryInfo, UnarchivedData)) 299 | { 300 | return false; 301 | } 302 | 303 | if (!TarArchiver->ExtractEntryToMemory(EntryInfo, UnarchivedData)) 304 | { 305 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to extract lz4 entry due to tar archiver error")); 306 | return false; 307 | } 308 | 309 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully extracted lz4 entry '%s' into memory"), *EntryInfo.Name); 310 | return true; 311 | } 312 | 313 | bool URuntimeArchiverLZ4::Initialize() 314 | { 315 | if (!Super::Initialize()) 316 | { 317 | return false; 318 | } 319 | 320 | TarArchiver.Reset(Cast(CreateRuntimeArchiver(this, URuntimeArchiverTar::StaticClass()))); 321 | if (!TarArchiver.IsValid()) 322 | { 323 | ReportError(ERuntimeArchiverErrorCode::NotInitialized, TEXT("Unable to allocate memory for lz4 archiver")); 324 | return false; 325 | } 326 | 327 | return true; 328 | } 329 | 330 | bool URuntimeArchiverLZ4::IsInitialized() const 331 | { 332 | return Super::IsInitialized() && TarArchiver.IsValid() && TarArchiver->IsInitialized(); 333 | } 334 | 335 | void URuntimeArchiverLZ4::Reset() 336 | { 337 | if (TarArchiver.IsValid()) 338 | { 339 | TarArchiver->Reset(); 340 | TarArchiver.Reset(); 341 | } 342 | 343 | CompressedStream.Reset(); 344 | Super::Reset(); 345 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully uninitialized lz4 archiver '%s'"), *GetName()); 346 | } 347 | 348 | void URuntimeArchiverLZ4::ReportError(ERuntimeArchiverErrorCode ErrorCode, const FString& ErrorString) const 349 | { 350 | Super::ReportError(ErrorCode, ErrorString); 351 | } 352 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Private/ArchiverOodle/RuntimeArchiverOodle.cpp: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #include "ArchiverOodle/RuntimeArchiverOodle.h" 4 | #include "RuntimeArchiverDefines.h" 5 | #include "RuntimeArchiverUtilities.h" 6 | #include "ArchiverRaw/RuntimeArchiverRaw.h" 7 | #include "ArchiverTar/RuntimeArchiverTar.h" 8 | #include "Streams/RuntimeArchiverFileStream.h" 9 | #include "Streams/RuntimeArchiverMemoryStream.h" 10 | 11 | URuntimeArchiverOodle::URuntimeArchiverOodle() 12 | : LastCompressionLevel{ERuntimeArchiverCompressionLevel::Compression6} 13 | { 14 | } 15 | 16 | bool URuntimeArchiverOodle::CreateArchiveInStorage(FString ArchivePath) 17 | { 18 | if (!Super::CreateArchiveInStorage(ArchivePath)) 19 | { 20 | return false; 21 | } 22 | 23 | CompressedStream.Reset(new FRuntimeArchiverFileStream(ArchivePath, true)); 24 | if (!CompressedStream->IsValid()) 25 | { 26 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open Oodle stream because it is not valid")); 27 | Reset(); 28 | return false; 29 | } 30 | 31 | if (!TarArchiver->CreateArchiveInMemory(0)) 32 | { 33 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to create Oodle archive in storage '%s' due to tar archiver error"), *ArchivePath); 34 | Reset(); 35 | return false; 36 | } 37 | 38 | return true; 39 | } 40 | 41 | bool URuntimeArchiverOodle::CreateArchiveInMemory(int32 InitialAllocationSize) 42 | { 43 | if (!Super::CreateArchiveInMemory(InitialAllocationSize)) 44 | { 45 | return false; 46 | } 47 | 48 | CompressedStream.Reset(new FRuntimeArchiverMemoryStream(InitialAllocationSize)); 49 | if (!CompressedStream->IsValid()) 50 | { 51 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open Oodle stream because it is not valid")); 52 | Reset(); 53 | return false; 54 | } 55 | 56 | if (!TarArchiver->CreateArchiveInMemory(InitialAllocationSize)) 57 | { 58 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to create Oodle archive in memory due to tar archiver error")); 59 | Reset(); 60 | return false; 61 | } 62 | 63 | return true; 64 | } 65 | 66 | bool URuntimeArchiverOodle::OpenArchiveFromStorage(FString ArchivePath) 67 | { 68 | if (!Super::OpenArchiveFromStorage(ArchivePath)) 69 | { 70 | return false; 71 | } 72 | 73 | CompressedStream.Reset(new FRuntimeArchiverFileStream(ArchivePath, false)); 74 | if (!CompressedStream->IsValid()) 75 | { 76 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open Oodle stream because it is not valid")); 77 | Reset(); 78 | return false; 79 | } 80 | 81 | TArray64 CompressedArchiveData; 82 | CompressedArchiveData.SetNumUninitialized(CompressedStream->Size()); 83 | 84 | if (!CompressedStream->Read(CompressedArchiveData.GetData(), CompressedArchiveData.Num())) 85 | { 86 | ReportError(ERuntimeArchiverErrorCode::GetError, TEXT("Unable to read Oodle compressed stream to get archive data")); 87 | Reset(); 88 | return false; 89 | } 90 | 91 | TArray64 TarArchiveData; 92 | if (!URuntimeArchiverRaw::UncompressRawData(ERuntimeArchiverRawFormat::Oodle, MoveTemp(CompressedArchiveData), TarArchiveData)) 93 | { 94 | ReportError(ERuntimeArchiverErrorCode::GetError, FString::Printf(TEXT("Unable to uncompress Oodle to tar data to open archive from storage '%s'"), *ArchivePath)); 95 | Reset(); 96 | return false; 97 | } 98 | 99 | if (!TarArchiver->OpenArchiveFromMemory(TarArchiveData)) 100 | { 101 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open Oodle archive from storage due to tar archiver error")); 102 | Reset(); 103 | return false; 104 | } 105 | 106 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully opened Oodle archive '%s' in '%s' to read"), *GetName(), *ArchivePath); 107 | return true; 108 | } 109 | 110 | bool URuntimeArchiverOodle::OpenArchiveFromMemory(const TArray64& ArchiveData) 111 | { 112 | if (!Super::OpenArchiveFromMemory(ArchiveData)) 113 | { 114 | return false; 115 | } 116 | 117 | CompressedStream.Reset(new FRuntimeArchiverMemoryStream(ArchiveData)); 118 | if (!CompressedStream->IsValid()) 119 | { 120 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open Oodle stream because it is not valid")); 121 | Reset(); 122 | return false; 123 | } 124 | 125 | TArray64 TarArchiveData; 126 | if (!URuntimeArchiverRaw::UncompressRawData(ERuntimeArchiverRawFormat::Oodle, ArchiveData, TarArchiveData)) 127 | { 128 | ReportError(ERuntimeArchiverErrorCode::GetError, TEXT("Unable to uncompress Oodle to tar data to open archive from memory")); 129 | Reset(); 130 | return false; 131 | } 132 | 133 | if (!TarArchiver->OpenArchiveFromMemory(TarArchiveData)) 134 | { 135 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open Oodle archive from memory due to tar archiver error")); 136 | Reset(); 137 | return false; 138 | } 139 | 140 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully opened in-memory Oodle archive '%s' to read"), *GetName()); 141 | return true; 142 | } 143 | 144 | bool URuntimeArchiverOodle::CloseArchive() 145 | { 146 | if (!Super::CloseArchive()) 147 | { 148 | return false; 149 | } 150 | 151 | if (Mode == ERuntimeArchiverMode::Write && Location == ERuntimeArchiverLocation::Storage) 152 | { 153 | TArray64 ArchiveData; 154 | if (!GetArchiveData(ArchiveData)) 155 | { 156 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to get archive data to close archive")); 157 | return false; 158 | } 159 | 160 | if (!CompressedStream->Seek(0)) 161 | { 162 | ReportError(ERuntimeArchiverErrorCode::CloseError, TEXT("Unable to seek first position in Oodle archive to close archive")); 163 | return false; 164 | } 165 | 166 | if (!CompressedStream->Write(ArchiveData.GetData(), ArchiveData.Num())) 167 | { 168 | ReportError(ERuntimeArchiverErrorCode::CloseError, TEXT("Unable to write archive data to Oodle compressed stream to close archive")); 169 | return false; 170 | } 171 | } 172 | 173 | if (!TarArchiver->CloseArchive()) 174 | { 175 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to close Oodle archive due to tar archiver error")); 176 | return false; 177 | } 178 | 179 | Reset(); 180 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully closed Oodle archive '%s'"), *GetName()); 181 | return true; 182 | } 183 | 184 | bool URuntimeArchiverOodle::GetArchiveData(TArray64& ArchiveData) 185 | { 186 | if (!Super::GetArchiveData(ArchiveData)) 187 | { 188 | return false; 189 | } 190 | 191 | if (Mode == ERuntimeArchiverMode::Read) 192 | { 193 | ArchiveData.SetNumUninitialized(CompressedStream->Size()); 194 | 195 | if (!CompressedStream->Seek(0)) 196 | { 197 | ReportError(ERuntimeArchiverErrorCode::GetError, TEXT("Unable to seek first position in Oodle archive to get archive data")); 198 | return false; 199 | } 200 | 201 | if (!CompressedStream->Read(ArchiveData.GetData(), ArchiveData.Num())) 202 | { 203 | ReportError(ERuntimeArchiverErrorCode::GetError, TEXT("Unable to read Oodle compressed stream to get archive data")); 204 | return false; 205 | } 206 | } 207 | else if (Mode == ERuntimeArchiverMode::Write) 208 | { 209 | TArray64 TarArchiveData; 210 | if (!TarArchiver->GetArchiveData(TarArchiveData)) 211 | { 212 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to get Oodle archive data due to tar archiver error")); 213 | return false; 214 | } 215 | 216 | if (!URuntimeArchiverRaw::CompressRawData(ERuntimeArchiverRawFormat::Oodle, LastCompressionLevel, TarArchiveData, ArchiveData)) 217 | { 218 | ReportError(ERuntimeArchiverErrorCode::GetError, TEXT("Unable to compress tar to Oodle data to get archive data")); 219 | return false; 220 | } 221 | } 222 | 223 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully retrieved Oodle archive data from memory with size '%lld'"), ArchiveData.Num()); 224 | return true; 225 | } 226 | 227 | bool URuntimeArchiverOodle::GetArchiveEntries(int32& NumOfArchiveEntries) 228 | { 229 | if (!Super::GetArchiveEntries(NumOfArchiveEntries)) 230 | { 231 | return false; 232 | } 233 | 234 | if (!TarArchiver->GetArchiveEntries(NumOfArchiveEntries)) 235 | { 236 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to the get number of Oodle entries due to tar archiver error")); 237 | return false; 238 | } 239 | 240 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully retrieved %d Oodle entries"), NumOfArchiveEntries); 241 | return true; 242 | } 243 | 244 | bool URuntimeArchiverOodle::GetArchiveEntryInfoByName(FString EntryName, FRuntimeArchiveEntry& EntryInfo) 245 | { 246 | if (!Super::GetArchiveEntryInfoByName(EntryName, EntryInfo)) 247 | { 248 | return false; 249 | } 250 | 251 | if (!TarArchiver->GetArchiveEntryInfoByName(MoveTemp(EntryName), EntryInfo)) 252 | { 253 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to get Oodle entry by name due to tar archiver error")); 254 | return false; 255 | } 256 | 257 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully retrieved Oodle entry '%s' by name"), *EntryInfo.Name); 258 | return true; 259 | } 260 | 261 | bool URuntimeArchiverOodle::GetArchiveEntryInfoByIndex(int32 EntryIndex, FRuntimeArchiveEntry& EntryInfo) 262 | { 263 | if (!Super::GetArchiveEntryInfoByIndex(EntryIndex, EntryInfo)) 264 | { 265 | return false; 266 | } 267 | 268 | if (!TarArchiver->GetArchiveEntryInfoByIndex(EntryIndex, EntryInfo)) 269 | { 270 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to get Oodle entry by index due to tar archiver error")); 271 | return false; 272 | } 273 | 274 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully retrieved Oodle entry '%s' by index"), *EntryInfo.Name); 275 | return true; 276 | } 277 | 278 | bool URuntimeArchiverOodle::AddEntryFromMemory(FString EntryName, const TArray64& DataToBeArchived, ERuntimeArchiverCompressionLevel CompressionLevel) 279 | { 280 | if (!Super::AddEntryFromMemory(EntryName, DataToBeArchived, CompressionLevel)) 281 | { 282 | return false; 283 | } 284 | 285 | if (!TarArchiver->AddEntryFromMemory(EntryName, DataToBeArchived, CompressionLevel)) 286 | { 287 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to add Oodle entry due to tar archiver error")); 288 | return false; 289 | } 290 | 291 | LastCompressionLevel = CompressionLevel; 292 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully added Oodle entry '%s' with size %lld bytes from memory"), *EntryName, DataToBeArchived.Num()); 293 | return true; 294 | } 295 | 296 | bool URuntimeArchiverOodle::ExtractEntryToMemory(const FRuntimeArchiveEntry& EntryInfo, TArray64& UnarchivedData) 297 | { 298 | if (!Super::ExtractEntryToMemory(EntryInfo, UnarchivedData)) 299 | { 300 | return false; 301 | } 302 | 303 | if (!TarArchiver->ExtractEntryToMemory(EntryInfo, UnarchivedData)) 304 | { 305 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to extract Oodle entry due to tar archiver error")); 306 | return false; 307 | } 308 | 309 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully extracted Oodle entry '%s' into memory"), *EntryInfo.Name); 310 | return true; 311 | } 312 | 313 | bool URuntimeArchiverOodle::Initialize() 314 | { 315 | if (!Super::Initialize()) 316 | { 317 | return false; 318 | } 319 | 320 | TarArchiver.Reset(Cast(CreateRuntimeArchiver(this, URuntimeArchiverTar::StaticClass()))); 321 | if (!TarArchiver.IsValid()) 322 | { 323 | ReportError(ERuntimeArchiverErrorCode::NotInitialized, TEXT("Unable to allocate memory for Oodle archiver")); 324 | return false; 325 | } 326 | 327 | return true; 328 | } 329 | 330 | bool URuntimeArchiverOodle::IsInitialized() const 331 | { 332 | return Super::IsInitialized() && TarArchiver.IsValid() && TarArchiver->IsInitialized(); 333 | } 334 | 335 | void URuntimeArchiverOodle::Reset() 336 | { 337 | if (TarArchiver.IsValid()) 338 | { 339 | TarArchiver->Reset(); 340 | TarArchiver.Reset(); 341 | } 342 | 343 | CompressedStream.Reset(); 344 | Super::Reset(); 345 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully uninitialized Oodle archiver '%s'"), *GetName()); 346 | } 347 | 348 | void URuntimeArchiverOodle::ReportError(ERuntimeArchiverErrorCode ErrorCode, const FString& ErrorString) const 349 | { 350 | Super::ReportError(ErrorCode, ErrorString); 351 | } 352 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Private/ArchiverRaw/RuntimeArchiverRaw.cpp: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #include "ArchiverRaw/RuntimeArchiverRaw.h" 4 | #include "RuntimeArchiverDefines.h" 5 | #include "Async/Async.h" 6 | #include "Misc/EngineVersionComparison.h" 7 | #include "Misc/Compression.h" 8 | #if UE_VERSION_NEWER_THAN(5, 0, 0) 9 | #include "Compression/OodleDataCompressionUtil.h" 10 | #endif 11 | 12 | namespace 13 | { 14 | /** 15 | * Converting a raw format enum to a name used in the Unreal Engine compressor 16 | */ 17 | EName ToName(ERuntimeArchiverRawFormat Format) 18 | { 19 | switch (Format) 20 | { 21 | case ERuntimeArchiverRawFormat::Oodle: 22 | { 23 | #if UE_VERSION_NEWER_THAN(5, 0, 0) 24 | return NAME_Oodle; 25 | #else 26 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Oodle format is not supported in %s %d"), TEXT(EPIC_PRODUCT_NAME), ENGINE_MAJOR_VERSION); 27 | return NAME_None; 28 | #endif 29 | } 30 | case ERuntimeArchiverRawFormat::GZip: 31 | { 32 | return NAME_Gzip; 33 | } 34 | case ERuntimeArchiverRawFormat::LZ4: 35 | { 36 | return NAME_LZ4; 37 | } 38 | default: 39 | return NAME_None; 40 | } 41 | } 42 | 43 | /** 44 | * Check if the specified format is valid 45 | */ 46 | bool IsFormatValid(const FName& FormatName) 47 | { 48 | // There is a bug in the engine where the LZ4 format cannot be found, even though it exists 49 | if (FormatName == NAME_LZ4) 50 | { 51 | return true; 52 | } 53 | 54 | if (!FCompression::IsFormatValid(FormatName)) 55 | { 56 | return false; 57 | } 58 | 59 | return true; 60 | } 61 | } 62 | 63 | #if UE_VERSION_NEWER_THAN(5, 0, 0) 64 | /** 65 | * Convert plugin-specific archiver data to Oodle compressor-specific data 66 | */ 67 | namespace OodleConversation 68 | { 69 | FOodleDataCompression::ECompressor GetCompressor(ERuntimeArchiverCompressionLevel CompressionLevel) 70 | { 71 | switch (CompressionLevel) 72 | { 73 | case ERuntimeArchiverCompressionLevel::Compression0: 74 | return FOodleDataCompression::ECompressor::Selkie; 75 | case ERuntimeArchiverCompressionLevel::Compression1: 76 | return FOodleDataCompression::ECompressor::Selkie; 77 | case ERuntimeArchiverCompressionLevel::Compression2: 78 | return FOodleDataCompression::ECompressor::Mermaid; 79 | case ERuntimeArchiverCompressionLevel::Compression3: 80 | return FOodleDataCompression::ECompressor::Mermaid; 81 | case ERuntimeArchiverCompressionLevel::Compression4: 82 | return FOodleDataCompression::ECompressor::Mermaid; 83 | case ERuntimeArchiverCompressionLevel::Compression5: 84 | return FOodleDataCompression::ECompressor::Kraken; 85 | case ERuntimeArchiverCompressionLevel::Compression6: 86 | return FOodleDataCompression::ECompressor::Kraken; 87 | case ERuntimeArchiverCompressionLevel::Compression7: 88 | return FOodleDataCompression::ECompressor::Kraken; 89 | case ERuntimeArchiverCompressionLevel::Compression8: 90 | return FOodleDataCompression::ECompressor::Leviathan; 91 | case ERuntimeArchiverCompressionLevel::Compression9: 92 | return FOodleDataCompression::ECompressor::Leviathan; 93 | case ERuntimeArchiverCompressionLevel::Compression10: 94 | return FOodleDataCompression::ECompressor::Leviathan; 95 | default: 96 | return FOodleDataCompression::ECompressor::NotSet; 97 | } 98 | } 99 | 100 | FOodleDataCompression::ECompressionLevel GetCompressionLevel(ERuntimeArchiverCompressionLevel CompressionLevel) 101 | { 102 | switch (CompressionLevel) 103 | { 104 | case ERuntimeArchiverCompressionLevel::Compression0: 105 | return FOodleDataCompression::ECompressionLevel::HyperFast1; 106 | case ERuntimeArchiverCompressionLevel::Compression1: 107 | return FOodleDataCompression::ECompressionLevel::SuperFast; 108 | case ERuntimeArchiverCompressionLevel::Compression2: 109 | return FOodleDataCompression::ECompressionLevel::VeryFast; 110 | case ERuntimeArchiverCompressionLevel::Compression3: 111 | return FOodleDataCompression::ECompressionLevel::Fast; 112 | case ERuntimeArchiverCompressionLevel::Compression4: 113 | return FOodleDataCompression::ECompressionLevel::Normal; 114 | case ERuntimeArchiverCompressionLevel::Compression5: 115 | return FOodleDataCompression::ECompressionLevel::Fast; 116 | case ERuntimeArchiverCompressionLevel::Compression6: 117 | return FOodleDataCompression::ECompressionLevel::Normal; 118 | case ERuntimeArchiverCompressionLevel::Compression7: 119 | return FOodleDataCompression::ECompressionLevel::Optimal1; 120 | case ERuntimeArchiverCompressionLevel::Compression8: 121 | return FOodleDataCompression::ECompressionLevel::Optimal2; 122 | case ERuntimeArchiverCompressionLevel::Compression9: 123 | return FOodleDataCompression::ECompressionLevel::Optimal3; 124 | case ERuntimeArchiverCompressionLevel::Compression10: 125 | return FOodleDataCompression::ECompressionLevel::Optimal4; 126 | default: 127 | return FOodleDataCompression::ECompressionLevel::None; 128 | } 129 | } 130 | } 131 | #endif 132 | 133 | void URuntimeArchiverRaw::CompressRawDataAsync(ERuntimeArchiverRawFormat RawFormat, ERuntimeArchiverCompressionLevel CompressionLevel, TArray UncompressedData, const FRuntimeArchiverRawMemoryResult& OnResult) 134 | { 135 | CompressRawDataAsync(RawFormat, CompressionLevel, TArray64(MoveTemp(UncompressedData)), 136 | FRuntimeArchiverRawMemoryResultNative::CreateLambda([OnResult](TArray64 CompressedData64) 137 | { 138 | if (CompressedData64.Num() > TNumericLimits::SizeType>::Max()) 139 | { 140 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Array with int32 size (max length: %d) cannot fit int64 size data (retrieved length: %lld)\nA standard byte array can hold a maximum of 2 GB of data"), TNumericLimits::SizeType>::Max(), CompressedData64.Num()); 141 | OnResult.ExecuteIfBound(TArray()); 142 | return; 143 | } 144 | OnResult.ExecuteIfBound(TArray(MoveTemp(CompressedData64))); 145 | })); 146 | } 147 | 148 | void URuntimeArchiverRaw::CompressRawDataAsync(ERuntimeArchiverRawFormat RawFormat, ERuntimeArchiverCompressionLevel CompressionLevel, TArray64 UncompressedData, const FRuntimeArchiverRawMemoryResultNative& OnResult) 149 | { 150 | AsyncTask(ENamedThreads::AnyBackgroundHiPriTask, [RawFormat, CompressionLevel, UncompressedData = MoveTemp(UncompressedData), OnResult]() mutable 151 | { 152 | TArray64 CompressedData; 153 | CompressRawData(RawFormat, CompressionLevel, MoveTemp(UncompressedData), CompressedData); 154 | 155 | AsyncTask(ENamedThreads::GameThread, [OnResult, CompressedData = MoveTemp(CompressedData)]() mutable 156 | { 157 | OnResult.ExecuteIfBound(MoveTemp(CompressedData)); 158 | }); 159 | }); 160 | } 161 | 162 | bool URuntimeArchiverRaw::CompressRawData(ERuntimeArchiverRawFormat RawFormat, ERuntimeArchiverCompressionLevel CompressionLevel, const TArray64& UncompressedData, TArray64& CompressedData) 163 | { 164 | const FName FormatName = ToName(RawFormat); 165 | if (!IsFormatValid(FormatName)) 166 | { 167 | UE_LOG(LogRuntimeArchiver, Error, TEXT("The specified format '%s' is not valid"), *FormatName.ToString()); 168 | return false; 169 | } 170 | 171 | #if UE_VERSION_NEWER_THAN(5, 0, 0) 172 | if (FormatName.IsEqual(NAME_Oodle)) 173 | { 174 | if (!FOodleCompressedArray::CompressTArray64(CompressedData, UncompressedData, OodleConversation::GetCompressor(CompressionLevel), OodleConversation::GetCompressionLevel(CompressionLevel))) 175 | { 176 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to compress data for '%s' format"), *FormatName.ToString()); 177 | return false; 178 | } 179 | 180 | return true; 181 | } 182 | #endif 183 | 184 | int32 CompressedSize = GuessCompressedSize(RawFormat, UncompressedData); 185 | if (CompressedSize <= 0) 186 | { 187 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to get compressed data size for '%s' format"), *FormatName.ToString()); 188 | return false; 189 | } 190 | 191 | TArray64 TempCompressedData; 192 | TempCompressedData.SetNumUninitialized(CompressedSize); 193 | 194 | if (!FCompression::CompressMemory(FormatName, TempCompressedData.GetData(), CompressedSize, UncompressedData.GetData(), UncompressedData.Num())) 195 | { 196 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to compress data for '%s' format"), *FormatName.ToString()); 197 | return false; 198 | } 199 | 200 | TempCompressedData.SetNum(CompressedSize, true); 201 | CompressedData = MoveTemp(TempCompressedData); 202 | return true; 203 | } 204 | 205 | void URuntimeArchiverRaw::UncompressRawDataAsync(ERuntimeArchiverRawFormat RawFormat, TArray CompressedData, const FRuntimeArchiverRawMemoryResult& OnResult) 206 | { 207 | UncompressRawDataAsync(RawFormat, TArray64(MoveTemp(CompressedData)), 208 | FRuntimeArchiverRawMemoryResultNative::CreateLambda([OnResult](TArray64 UncompressedData64) 209 | { 210 | if (UncompressedData64.Num() > TNumericLimits::SizeType>::Max()) 211 | { 212 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Array with int32 size (max length: %d) cannot fit int64 size data (retrieved length: %lld)\nA standard byte array can hold a maximum of 2 GB of data"), TNumericLimits::SizeType>::Max(), UncompressedData64.Num()); 213 | OnResult.ExecuteIfBound(TArray()); 214 | return; 215 | } 216 | OnResult.ExecuteIfBound(TArray(MoveTemp(UncompressedData64))); 217 | })); 218 | } 219 | 220 | void URuntimeArchiverRaw::UncompressRawDataAsync(ERuntimeArchiverRawFormat RawFormat, TArray64 CompressedData, const FRuntimeArchiverRawMemoryResultNative& OnResult) 221 | { 222 | AsyncTask(ENamedThreads::AnyBackgroundHiPriTask, [RawFormat, CompressedData = MoveTemp(CompressedData), OnResult]() mutable 223 | { 224 | TArray64 UncompressedData; 225 | UncompressRawData(RawFormat, CompressedData = MoveTemp(CompressedData), UncompressedData); 226 | 227 | AsyncTask(ENamedThreads::GameThread, [OnResult, UncompressedData = MoveTemp(UncompressedData)]() mutable 228 | { 229 | OnResult.ExecuteIfBound(MoveTemp(UncompressedData)); 230 | }); 231 | }); 232 | } 233 | 234 | bool URuntimeArchiverRaw::UncompressRawData(ERuntimeArchiverRawFormat RawFormat, TArray64 CompressedData, TArray64& UncompressedData) 235 | { 236 | const FName FormatName = ToName(RawFormat); 237 | 238 | if (!IsFormatValid(FormatName)) 239 | { 240 | UE_LOG(LogRuntimeArchiver, Error, TEXT("The specified format '%s' is not valid"), *FormatName.ToString()); 241 | return false; 242 | } 243 | 244 | #if UE_VERSION_NEWER_THAN(5, 0, 0) 245 | if (FormatName.IsEqual(NAME_Oodle)) 246 | { 247 | if (!FOodleCompressedArray::DecompressToTArray64(UncompressedData, CompressedData)) 248 | { 249 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to uncompress data for '%s' format"), *FormatName.ToString()); 250 | return false; 251 | } 252 | return true; 253 | } 254 | #endif 255 | 256 | const int64 UncompressedSize = GuessUncompressedSize(RawFormat, CompressedData); 257 | if (UncompressedSize <= 0) 258 | { 259 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to get compressed data size for '%s' format"), *FormatName.ToString()); 260 | return false; 261 | } 262 | 263 | TArray64 TempUncompressedData; 264 | TempUncompressedData.SetNumUninitialized(UncompressedSize); 265 | 266 | if (!FCompression::UncompressMemory(FormatName, TempUncompressedData.GetData(), UncompressedSize, CompressedData.GetData(), CompressedData.Num())) 267 | { 268 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to uncompress data for '%s' format"), *FormatName.ToString()); 269 | return false; 270 | } 271 | 272 | UncompressedData = MoveTemp(TempUncompressedData); 273 | return true; 274 | } 275 | 276 | int64 URuntimeArchiverRaw::GuessCompressedSize(ERuntimeArchiverRawFormat RawFormat, const TArray64& UncompressedData) 277 | { 278 | const FName FormatName = ToName(RawFormat); 279 | if (!IsFormatValid(FormatName)) 280 | { 281 | UE_LOG(LogRuntimeArchiver, Error, TEXT("The specified format '%s' is not valid"), *FormatName.ToString()); 282 | return false; 283 | } 284 | #if UE_VERSION_NEWER_THAN(5, 0, 0) 285 | return FCompression::GetMaximumCompressedSize(FormatName, UncompressedData.Num()); 286 | #else 287 | return FCompression::CompressMemoryBound(FormatName, UncompressedData.Num()); 288 | #endif 289 | } 290 | 291 | int64 URuntimeArchiverRaw::GuessUncompressedSize(ERuntimeArchiverRawFormat RawFormat, const TArray64& CompressedData) 292 | { 293 | if (CompressedData.Num() <= 0) 294 | { 295 | return 0; 296 | } 297 | 298 | switch (RawFormat) 299 | { 300 | case ERuntimeArchiverRawFormat::GZip: 301 | { 302 | const uint8_t* DataPtr = CompressedData.GetData() + CompressedData.Num() - 4; 303 | 304 | return (static_cast(DataPtr[0]) << 0) + 305 | (static_cast(DataPtr[1]) << 8) + 306 | (static_cast(DataPtr[2]) << 16) + 307 | (static_cast(DataPtr[3]) << 24); 308 | } 309 | case ERuntimeArchiverRawFormat::LZ4: 310 | { 311 | return static_cast(CompressedData.Num() * 255); 312 | } 313 | default: 314 | { 315 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to determine the uncompressed size of the format '%s'"), *UEnum::GetValueAsString(RawFormat)); 316 | } 317 | } 318 | 319 | return 0; 320 | } 321 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Private/ArchiverTar/RuntimeArchiverTar.cpp: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #include "ArchiverTar/RuntimeArchiverTar.h" 4 | #include "RuntimeArchiverDefines.h" 5 | #include "RuntimeArchiverTarOperations.h" 6 | #include "RuntimeArchiverUtilities.h" 7 | #include "ArchiverTar/RuntimeArchiverTarHeader.h" 8 | #include "Streams/RuntimeArchiverFileStream.h" 9 | #include "Streams/RuntimeArchiverMemoryStream.h" 10 | #include "Misc/Paths.h" 11 | 12 | bool URuntimeArchiverTar::CreateArchiveInStorage(FString ArchivePath) 13 | { 14 | if (!Super::CreateArchiveInStorage(ArchivePath)) 15 | { 16 | return false; 17 | } 18 | 19 | FPaths::NormalizeFilename(ArchivePath); 20 | 21 | if (!TarEncapsulator->OpenFile(ArchivePath, true)) 22 | { 23 | ReportError(ERuntimeArchiverErrorCode::NotInitialized, FString::Printf(TEXT("Unable to open tar archive '%s' for writing"), *ArchivePath)); 24 | Reset(); 25 | return false; 26 | } 27 | 28 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully created tar archive '%s' in '%s'"), *GetName(), *ArchivePath); 29 | 30 | return true; 31 | } 32 | 33 | bool URuntimeArchiverTar::CreateArchiveInMemory(int32 InitialAllocationSize) 34 | { 35 | if (!Super::CreateArchiveInMemory(InitialAllocationSize)) 36 | { 37 | return false; 38 | } 39 | 40 | if (!TarEncapsulator->OpenMemory(TArray64(), InitialAllocationSize, true)) 41 | { 42 | ReportError(ERuntimeArchiverErrorCode::NotInitialized, TEXT("Unable to initialize tar archive in memory")); 43 | Reset(); 44 | return false; 45 | } 46 | 47 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully created tar archive '%s' in memory"), *GetName()); 48 | 49 | return true; 50 | } 51 | 52 | bool URuntimeArchiverTar::OpenArchiveFromStorage(FString ArchivePath) 53 | { 54 | if (!Super::OpenArchiveFromStorage(ArchivePath)) 55 | { 56 | return false; 57 | } 58 | 59 | FPaths::NormalizeFilename(ArchivePath); 60 | 61 | if (!TarEncapsulator->OpenFile(ArchivePath, false)) 62 | { 63 | ReportError(ERuntimeArchiverErrorCode::NotInitialized, FString::Printf(TEXT("Unable to open tar archive '%s' for writing"), *ArchivePath)); 64 | Reset(); 65 | return false; 66 | } 67 | 68 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully opened tar archive '%s' in '%s' to read"), *GetName(), *ArchivePath); 69 | 70 | return true; 71 | } 72 | 73 | bool URuntimeArchiverTar::OpenArchiveFromMemory(const TArray64& ArchiveData) 74 | { 75 | if (!Super::OpenArchiveFromMemory(ArchiveData)) 76 | { 77 | return false; 78 | } 79 | 80 | if (!TarEncapsulator->OpenMemory(ArchiveData, 0, false)) 81 | { 82 | ReportError(ERuntimeArchiverErrorCode::NotInitialized, TEXT("Unable to open in-memory tar archive to read")); 83 | Reset(); 84 | return false; 85 | } 86 | 87 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully opened in-memory tar archive '%s' to read"), *GetName()); 88 | 89 | return true; 90 | } 91 | 92 | bool URuntimeArchiverTar::CloseArchive() 93 | { 94 | if (!Super::CloseArchive()) 95 | { 96 | return false; 97 | } 98 | 99 | Reset(); 100 | 101 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully closed tar archive '%s'"), *GetName()); 102 | 103 | return true; 104 | } 105 | 106 | bool URuntimeArchiverTar::GetArchiveData(TArray64& ArchiveData) 107 | { 108 | if (!Super::GetArchiveData(ArchiveData)) 109 | { 110 | return false; 111 | } 112 | 113 | const bool bSuccess = TarEncapsulator->GetArchiveData(ArchiveData); 114 | 115 | if (!bSuccess) 116 | { 117 | ReportError(ERuntimeArchiverErrorCode::GetError, TEXT("Unable to get tar archive data from memory")); 118 | return false; 119 | } 120 | 121 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully retrieved tar archive data from memory with size '%lld'"), ArchiveData.Num()); 122 | 123 | return true; 124 | } 125 | 126 | bool URuntimeArchiverTar::GetArchiveEntries(int32& NumOfArchiveEntries) 127 | { 128 | if (!Super::GetArchiveEntries(NumOfArchiveEntries)) 129 | { 130 | return false; 131 | } 132 | 133 | if (!TarEncapsulator->GetArchiveEntries(NumOfArchiveEntries)) 134 | { 135 | ReportError(ERuntimeArchiverErrorCode::GetError, TEXT("Unable to get number of tar entries")); 136 | return false; 137 | } 138 | 139 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully retrieved %d tar entries"), NumOfArchiveEntries); 140 | return true; 141 | } 142 | 143 | bool URuntimeArchiverTar::GetArchiveEntryInfoByName(FString EntryName, FRuntimeArchiveEntry& EntryInfo) 144 | { 145 | if (!Super::GetArchiveEntryInfoByName(EntryName, EntryInfo)) 146 | { 147 | return false; 148 | } 149 | 150 | FTarHeader Header; 151 | int32 EntryIndex; 152 | 153 | // Searching for a header by the entry name 154 | const bool bFound = TarEncapsulator->FindIf([&EntryName](const FTarHeader& PossibleHeader, int32 PossibleIndex) 155 | { 156 | return EntryName.Equals(StringCast(PossibleHeader.GetName()).Get()); 157 | }, Header, EntryIndex, true); 158 | 159 | if (!bFound) 160 | { 161 | ReportError(ERuntimeArchiverErrorCode::GetError, FString::Printf(TEXT("Unable to find the entry index under the entry name '%s'"), *EntryName)); 162 | return false; 163 | } 164 | 165 | if (!FTarHeader::ToEntry(Header, EntryIndex, EntryInfo)) 166 | { 167 | ReportError(ERuntimeArchiverErrorCode::GetError, FString::Printf(TEXT("Failed to convert tar header to entry with entry name '%s'"), *EntryName)); 168 | return false; 169 | } 170 | 171 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully retrieved tar entry '%s' by name"), *EntryInfo.Name); 172 | 173 | return true; 174 | } 175 | 176 | bool URuntimeArchiverTar::GetArchiveEntryInfoByIndex(int32 EntryIndex, FRuntimeArchiveEntry& EntryInfo) 177 | { 178 | if (!Super::GetArchiveEntryInfoByIndex(EntryIndex, EntryInfo)) 179 | { 180 | return false; 181 | } 182 | 183 | FTarHeader Header; 184 | 185 | // Searching for a header by the entry index 186 | const bool bFound = TarEncapsulator->FindIf([EntryIndex](const FTarHeader& PossibleHeader, int32 PossibleIndex) 187 | { 188 | return EntryIndex == PossibleIndex; 189 | }, Header, EntryIndex, true); 190 | 191 | if (!bFound) 192 | { 193 | ReportError(ERuntimeArchiverErrorCode::GetError, FString::Printf(TEXT("Unable to find tar header at index %d"), EntryIndex)); 194 | return false; 195 | } 196 | 197 | if (!FTarHeader::ToEntry(Header, EntryIndex, EntryInfo)) 198 | { 199 | ReportError(ERuntimeArchiverErrorCode::GetError, FString::Printf(TEXT("Failed to convert tar header to entry with entry index '%d'"), EntryIndex)); 200 | return false; 201 | } 202 | 203 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully retrieved tar entry '%s' by index"), *EntryInfo.Name); 204 | 205 | return true; 206 | } 207 | 208 | bool URuntimeArchiverTar::AddEntryFromMemory(FString EntryName, const TArray64& DataToBeArchived, ERuntimeArchiverCompressionLevel CompressionLevel) 209 | { 210 | if (!Super::AddEntryFromMemory(EntryName, DataToBeArchived, CompressionLevel)) 211 | { 212 | return false; 213 | } 214 | 215 | // Adding directory entries separately as required by the tar specification 216 | { 217 | // Parsing directories from entry name 218 | TArray Directories = URuntimeArchiverUtilities::ParseDirectories(EntryName); 219 | 220 | for (const FString& Directory : Directories) 221 | { 222 | FTarHeader Header; 223 | int32 Index; 224 | 225 | // Skip if the archive already contains this directory entry 226 | const bool bFound = TarEncapsulator->FindIf([&Directory](const FTarHeader& PotentialHeader, int32 Index) 227 | { 228 | return Directory.Equals(StringCast(PotentialHeader.GetName()).Get()); 229 | }, Header, Index, true); 230 | 231 | if (!bFound) 232 | { 233 | if (!FTarHeader::GenerateHeader(Directory, 0, FDateTime::Now(), true, Header)) 234 | { 235 | ReportError(ERuntimeArchiverErrorCode::AddError, FString::Printf(TEXT("Unable to generate directory header for for entry '%s' to write from memory"), *Directory)); 236 | return false; 237 | } 238 | 239 | // Directories only require writing a header, no data 240 | if (!TarEncapsulator->WriteHeader(Header)) 241 | { 242 | ReportError(ERuntimeArchiverErrorCode::AddError, FString::Printf(TEXT("Unable to write header for entry '%s' from memory"), *Directory)); 243 | return false; 244 | } 245 | } 246 | } 247 | } 248 | 249 | FTarHeader Header; 250 | if (!FTarHeader::GenerateHeader(EntryName, DataToBeArchived.Num(), FDateTime::Now(), false, Header)) 251 | { 252 | ReportError(ERuntimeArchiverErrorCode::AddError, FString::Printf(TEXT("Unable to generate file header for for entry '%s' to write from memory"), *EntryName)); 253 | return false; 254 | } 255 | 256 | // Archiving data 257 | { 258 | if (!TarEncapsulator->WriteHeader(Header)) 259 | { 260 | ReportError(ERuntimeArchiverErrorCode::AddError, FString::Printf(TEXT("Unable to write header for entry '%s' from memory"), *EntryName)); 261 | return false; 262 | } 263 | 264 | if (!TarEncapsulator->WriteData(DataToBeArchived)) 265 | { 266 | ReportError(ERuntimeArchiverErrorCode::AddError, FString::Printf(TEXT("Unable to write data for entry '%s' from memory"), *EntryName)); 267 | return false; 268 | } 269 | } 270 | 271 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully added tar entry '%s' with size %lld bytes from memory"), *EntryName, DataToBeArchived.Num()); 272 | 273 | return true; 274 | } 275 | 276 | bool URuntimeArchiverTar::ExtractEntryToMemory(const FRuntimeArchiveEntry& EntryInfo, TArray64& UnarchivedData) 277 | { 278 | if (!Super::ExtractEntryToMemory(EntryInfo, UnarchivedData)) 279 | { 280 | return false; 281 | } 282 | 283 | int32 NumOfArchiveEntries; 284 | if (!GetArchiveEntries(NumOfArchiveEntries) || EntryInfo.Index > (NumOfArchiveEntries - 1)) 285 | { 286 | ReportError(ERuntimeArchiverErrorCode::InvalidArgument, FString::Printf(TEXT("Tar entry index %d is invalid. Min index: 0, Max index: %d"), EntryInfo.Index, (NumOfArchiveEntries - 1))); 287 | return false; 288 | } 289 | 290 | FTarHeader Header; 291 | int32 Index; 292 | 293 | // Make sure we have such entry 294 | const bool bFound = TarEncapsulator->FindIf([&EntryInfo](const FTarHeader& PotentialHeader, int32 Index) 295 | { 296 | return EntryInfo.Name.Equals(StringCast(PotentialHeader.GetName()).Get()); 297 | }, Header, Index, false); 298 | 299 | if (!bFound) 300 | { 301 | ReportError(ERuntimeArchiverErrorCode::ExtractError, FString::Printf(TEXT("Unable to find tar entry '%s' to write into memory"), *EntryInfo.Name)); 302 | return false; 303 | } 304 | 305 | UnarchivedData.SetNumUninitialized(Header.GetSize()); 306 | 307 | if (!TarEncapsulator->ReadData(UnarchivedData)) 308 | { 309 | ReportError(ERuntimeArchiverErrorCode::AddError, FString::Printf(TEXT("Unable to read data from tar entry '%s' to write into memory"), *EntryInfo.Name)); 310 | UnarchivedData.Empty(); 311 | return false; 312 | } 313 | 314 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully extracted tar entry '%s' into memory"), *EntryInfo.Name); 315 | 316 | return true; 317 | } 318 | 319 | bool URuntimeArchiverTar::Initialize() 320 | { 321 | if (!Super::Initialize()) 322 | { 323 | return false; 324 | } 325 | 326 | TarEncapsulator.Reset(new FRuntimeArchiverTarEncapsulator); 327 | 328 | if (!TarEncapsulator) 329 | { 330 | ReportError(ERuntimeArchiverErrorCode::NotInitialized, TEXT("Unable to allocate memory for tar archiver")); 331 | return false; 332 | } 333 | 334 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully initialized tar archiver '%s'"), *GetName()); 335 | 336 | return true; 337 | } 338 | 339 | bool URuntimeArchiverTar::IsInitialized() const 340 | { 341 | return Super::IsInitialized() && TarEncapsulator.IsValid() && TarEncapsulator->IsValid(); 342 | } 343 | 344 | void URuntimeArchiverTar::Reset() 345 | { 346 | TarEncapsulator.Reset(); 347 | 348 | Super::Reset(); 349 | 350 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully uninitialized tar archiver '%s'"), *GetName()); 351 | } 352 | 353 | void URuntimeArchiverTar::ReportError(ERuntimeArchiverErrorCode ErrorCode, const FString& ErrorString) const 354 | { 355 | Super::ReportError(ErrorCode, ErrorString); 356 | } 357 | 358 | FRuntimeArchiverTarEncapsulator::FRuntimeArchiverTarEncapsulator() 359 | : RemainingDataSize{0} 360 | , LastHeaderPosition{0} 361 | , CachedNumOfHeaders{0} 362 | , bIsFinalized{false} 363 | { 364 | } 365 | 366 | FRuntimeArchiverTarEncapsulator::~FRuntimeArchiverTarEncapsulator() 367 | { 368 | if (Stream.IsValid()) 369 | { 370 | if (Stream->IsWrite()) 371 | { 372 | Finalize(); 373 | } 374 | 375 | Stream.Reset(); 376 | } 377 | } 378 | 379 | bool FRuntimeArchiverTarEncapsulator::IsValid() 380 | { 381 | return Stream.IsValid() && Stream->IsValid(); 382 | } 383 | 384 | bool FRuntimeArchiverTarEncapsulator::TestArchive() 385 | { 386 | if (Stream->IsWrite()) 387 | { 388 | return true; 389 | } 390 | 391 | static FTarHeader Header{}; 392 | 393 | if (!Rewind()) 394 | { 395 | return false; 396 | } 397 | 398 | return ReadHeader(Header); 399 | } 400 | 401 | bool FRuntimeArchiverTarEncapsulator::OpenFile(const FString& ArchivePath, bool bWrite) 402 | { 403 | if (Stream.IsValid()) 404 | { 405 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open tar stream because it has already been opened")); 406 | return false; 407 | } 408 | 409 | Stream.Reset(new FRuntimeArchiverFileStream(ArchivePath, bWrite)); 410 | 411 | if (!Stream->IsValid()) 412 | { 413 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open tar stream because it is not valid")); 414 | return false; 415 | } 416 | 417 | return TestArchive(); 418 | } 419 | 420 | bool FRuntimeArchiverTarEncapsulator::OpenMemory(const TArray64& ArchiveData, int32 InitialAllocationSize, bool bWrite) 421 | { 422 | if (Stream.IsValid()) 423 | { 424 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open tar stream because it has already been opened")); 425 | return false; 426 | } 427 | 428 | Stream.Reset(bWrite ? new FRuntimeArchiverMemoryStream(InitialAllocationSize) : new FRuntimeArchiverMemoryStream(ArchiveData)); 429 | 430 | if (!IsValid()) 431 | { 432 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to open tar stream because it is not valid")); 433 | return false; 434 | } 435 | 436 | return TestArchive(); 437 | } 438 | 439 | bool FRuntimeArchiverTarEncapsulator::FindIf(TFunctionRef ComparePredicate, FTarHeader& Header, int32& Index, bool bRemainPosition) 440 | { 441 | if (!IsValid()) 442 | { 443 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to find tar entry because stream is invalid")); 444 | return false; 445 | } 446 | 447 | const int64 PreviousPosition = Stream->Tell(); 448 | 449 | // Make sure looking from the start 450 | if (!Rewind()) 451 | { 452 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to rewind read/write position of tar archive to find tar entry")); 453 | return false; 454 | } 455 | 456 | FTarHeader TempHeader; 457 | int32 TempIndex = 0; 458 | 459 | bool bFound = false; 460 | 461 | // Iterate all files until we hit an error or find the header 462 | while (ReadHeader(TempHeader)) 463 | { 464 | if (ComparePredicate(TempHeader, TempIndex)) 465 | { 466 | Header = TempHeader; 467 | Index = TempIndex; 468 | 469 | bFound = true; 470 | break; 471 | } 472 | 473 | if (!Next()) 474 | { 475 | break; 476 | } 477 | 478 | TempIndex++; 479 | } 480 | 481 | if (bRemainPosition) 482 | { 483 | Stream->Seek(PreviousPosition); 484 | } 485 | 486 | return bFound; 487 | } 488 | 489 | bool FRuntimeArchiverTarEncapsulator::GetArchiveEntries(int32& NumOfArchiveEntries) 490 | { 491 | if (!IsValid()) 492 | { 493 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to get tar archive entries because stream is invalid")); 494 | return false; 495 | } 496 | 497 | // TODO: verify the validity of the last entry 498 | 499 | // Read-only mode is supposed to have a fixed (immutable) number of entries, so if possible, return the cached number of entries in this mode to improve performance 500 | if (!Stream->IsWrite() && CachedNumOfHeaders > 0) 501 | { 502 | NumOfArchiveEntries = CachedNumOfHeaders; 503 | return true; 504 | } 505 | 506 | const int64 PreviousPosition = Stream->Tell(); 507 | 508 | // Making sure looking from the start 509 | if (!Rewind()) 510 | { 511 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to rewind read/write position of tar archive to get the number of archive entries")); 512 | return false; 513 | } 514 | 515 | FTarHeader TempHeader; 516 | int32 NumOfHeaders{0}; 517 | 518 | // Iterate all files 519 | while (ReadHeader(TempHeader)) 520 | { 521 | NumOfHeaders++; 522 | 523 | if (!Next()) 524 | { 525 | break; 526 | } 527 | } 528 | 529 | Stream->Seek(PreviousPosition); 530 | 531 | // Cache number of entries in read-only mode to improve performance 532 | if (!Stream->IsWrite()) 533 | { 534 | CachedNumOfHeaders = NumOfHeaders; 535 | } 536 | 537 | NumOfArchiveEntries = NumOfHeaders; 538 | return true; 539 | } 540 | 541 | bool FRuntimeArchiverTarEncapsulator::GetArchiveData(TArray64& ArchiveData) 542 | { 543 | if (!IsValid()) 544 | { 545 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to get tar archive data because stream is invalid")); 546 | return false; 547 | } 548 | 549 | if (Stream->IsWrite() && !Finalize()) 550 | { 551 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to get tar archive data because finalization failed")); 552 | return false; 553 | } 554 | 555 | ArchiveData.SetNumUninitialized(Stream->Size()); 556 | 557 | const int64 PrevRemainingDataSize = RemainingDataSize; 558 | const int64 PrevLastHeaderPosition = LastHeaderPosition; 559 | const int64 PrevStreamPosition = Stream->Tell(); 560 | 561 | Rewind(); 562 | 563 | if (!Stream->Read(ArchiveData.GetData(), ArchiveData.Num())) 564 | { 565 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to read tar archive data with size '%lld'"), ArchiveData.Num()); 566 | return false; 567 | } 568 | 569 | RemainingDataSize = PrevRemainingDataSize; 570 | LastHeaderPosition = PrevLastHeaderPosition; 571 | 572 | if (!Stream->Seek(PrevStreamPosition)) 573 | { 574 | return false; 575 | } 576 | 577 | return true; 578 | } 579 | 580 | bool FRuntimeArchiverTarEncapsulator::Rewind() 581 | { 582 | if (!IsValid()) 583 | { 584 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to rewind read/write position of tar archive because stream is invalid")); 585 | return false; 586 | } 587 | 588 | RemainingDataSize = 0; 589 | LastHeaderPosition = 0; 590 | return Stream->Seek(0); 591 | } 592 | 593 | bool FRuntimeArchiverTarEncapsulator::Next() 594 | { 595 | if (!IsValid()) 596 | { 597 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to get next tar archive data because stream is invalid")); 598 | return false; 599 | } 600 | 601 | FTarHeader Header; 602 | if (!ReadHeader(Header)) 603 | { 604 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to read header for getting next tar entry data")); 605 | return false; 606 | } 607 | 608 | const int64 NextEntryOffset = RuntimeArchiverTarOperations::RoundUp(Header.GetSize(), 512) + sizeof(FTarHeader); 609 | 610 | UE_LOG(LogRuntimeArchiver, Verbose, TEXT("Seeking to next tar entry at offset %lld"), NextEntryOffset); 611 | 612 | return Stream->Seek(Stream->Tell() + NextEntryOffset); 613 | } 614 | 615 | bool FRuntimeArchiverTarEncapsulator::ReadHeader(FTarHeader& Header) 616 | { 617 | LastHeaderPosition = Stream->Tell(); 618 | 619 | // Make sure that we will read valid data (the last 2 entries in the archive are not valid) 620 | { 621 | const int64 PotentialMaxPosition = (sizeof(FTarHeader) * 2) + Stream->Tell() + sizeof(Header); 622 | 623 | if (PotentialMaxPosition > Stream->Size()) 624 | { 625 | return false; 626 | } 627 | } 628 | 629 | if (!Stream->Read(&Header, sizeof(Header))) 630 | { 631 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to read header from stream")); 632 | return false; 633 | } 634 | 635 | // Seek back to start of header 636 | if (!Stream->Seek(LastHeaderPosition)) 637 | { 638 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to seek back to start of header")); 639 | return false; 640 | } 641 | 642 | return true; 643 | } 644 | 645 | bool FRuntimeArchiverTarEncapsulator::ReadData(TArray64& Data) 646 | { 647 | // If there is no remaining data, then this is the first reading. Getting the size, setting the remaining data and seeking to the beginning of the data 648 | if (RemainingDataSize == 0) 649 | { 650 | FTarHeader Header; 651 | if (!ReadHeader(Header)) 652 | { 653 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to read header for getting tar entry data")); 654 | return false; 655 | } 656 | 657 | if (!Stream->Seek(Stream->Tell() + sizeof(FTarHeader))) 658 | { 659 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to seek to next header for getting tar entry data")); 660 | return false; 661 | } 662 | 663 | RemainingDataSize = Header.GetSize(); 664 | } 665 | 666 | if (!Stream->Read(Data.GetData(), Data.Num())) 667 | { 668 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to read tar entry data")); 669 | return false; 670 | } 671 | 672 | RemainingDataSize -= Data.Num(); 673 | 674 | // If there is no remaining data, then we have finished reading and seek back to the header 675 | if (RemainingDataSize == 0) 676 | { 677 | return Stream->Seek(LastHeaderPosition); 678 | } 679 | 680 | return true; 681 | } 682 | 683 | bool FRuntimeArchiverTarEncapsulator::WriteHeader(const FTarHeader& Header) 684 | { 685 | RemainingDataSize = Header.GetSize(); 686 | return Stream->Write(&Header, sizeof(Header)); 687 | } 688 | 689 | bool FRuntimeArchiverTarEncapsulator::WriteData(const TArray64& DataToBeArchived) 690 | { 691 | if (!Stream->Write(DataToBeArchived.GetData(), DataToBeArchived.Num())) 692 | { 693 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to write tar entry data")); 694 | return false; 695 | } 696 | 697 | RemainingDataSize -= DataToBeArchived.Num(); 698 | 699 | // Write padding if all data for this entry has already been written 700 | if (RemainingDataSize == 0) 701 | { 702 | const int64 CurrentPosition{Stream->Tell()}; 703 | return WriteNullBytes(RuntimeArchiverTarOperations::RoundUp(CurrentPosition, 512) - CurrentPosition); 704 | } 705 | 706 | return true; 707 | } 708 | 709 | bool FRuntimeArchiverTarEncapsulator::WriteNullBytes(int64 NumOfBytes) const 710 | { 711 | constexpr ANSICHAR NullCharacter = '\0'; 712 | 713 | for (int64 Index = 0; Index < NumOfBytes; ++Index) 714 | { 715 | if (!Stream->Write(&NullCharacter, 1)) 716 | { 717 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to write null bytes to tar archive")); 718 | return false; 719 | } 720 | } 721 | 722 | return true; 723 | } 724 | 725 | bool FRuntimeArchiverTarEncapsulator::Finalize() 726 | { 727 | if (bIsFinalized) 728 | { 729 | return true; 730 | } 731 | 732 | bIsFinalized = true; 733 | 734 | return WriteNullBytes(sizeof(FTarHeader) * 2); 735 | } 736 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Private/ArchiverTar/RuntimeArchiverTarHeader.cpp: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #include "ArchiverTar/RuntimeArchiverTarHeader.h" 4 | 5 | #include "RuntimeArchiverDefines.h" 6 | #include "RuntimeArchiverTypes.h" 7 | #include "HAL/UnrealMemory.h" 8 | #include "Containers/StringConv.h" 9 | #include "ArchiverTar/RuntimeArchiverTarOperations.h" 10 | 11 | /** 12 | * Helper that handles type flags 13 | */ 14 | class FTarTypeFlagHelper 15 | { 16 | /** File type flags */ 17 | static const ANSICHAR FileTypeFlag; 18 | static const ANSICHAR FileTypeFlag1; 19 | 20 | /** Directory type flag */ 21 | static const ANSICHAR DirectoryTypeFlag; 22 | 23 | /** Unsupported type flags */ 24 | static const ANSICHAR HardLinkTypeFlag; 25 | static const ANSICHAR SymbolicLinkTypeFlag; 26 | static const ANSICHAR CharacterDeviceTypeFlag; 27 | static const ANSICHAR BlockDeviceTypeFlag; 28 | static const ANSICHAR FIFOTypeFlag; 29 | 30 | /** Array of string representation of type flags */ 31 | static const TMap Strings; 32 | 33 | public: 34 | /** 35 | * Get a string representation of the specified type flag 36 | */ 37 | static FString ToString(ANSICHAR TypeFlag) 38 | { 39 | FString const* String = Strings.Find(TypeFlag); 40 | 41 | if (!String) 42 | { 43 | return TEXT("Unrecognized type flag"); 44 | } 45 | 46 | return *String; 47 | } 48 | 49 | /** 50 | * Check if the specified type flag applies to a directory 51 | */ 52 | static bool IsDirectory(ANSICHAR TypeFlag) 53 | { 54 | if (TypeFlag == FileTypeFlag || TypeFlag == FileTypeFlag1) 55 | { 56 | return false; 57 | } 58 | 59 | if (TypeFlag == DirectoryTypeFlag) 60 | { 61 | return true; 62 | } 63 | 64 | UE_LOG(LogRuntimeArchiver, Error, TEXT("The type flag %c (%s) is not supported. Supported type flags are %c/%c (%s) and %c (%s)"), 65 | TCHAR(TypeFlag), *ToString(TypeFlag), 66 | TCHAR(FileTypeFlag), TCHAR(FileTypeFlag1), *ToString(FileTypeFlag), 67 | TCHAR(DirectoryTypeFlag), *ToString(DirectoryTypeFlag)); 68 | return false; 69 | } 70 | 71 | /** 72 | * Get type flag 73 | */ 74 | static ANSICHAR GetTypeFlag(bool bIsDirectory) 75 | { 76 | return bIsDirectory ? DirectoryTypeFlag : FileTypeFlag; 77 | } 78 | }; 79 | 80 | const ANSICHAR FTarTypeFlagHelper::FileTypeFlag{'0'}; 81 | const ANSICHAR FTarTypeFlagHelper::FileTypeFlag1{'\0'}; 82 | const ANSICHAR FTarTypeFlagHelper::DirectoryTypeFlag{'5'}; 83 | const ANSICHAR FTarTypeFlagHelper::HardLinkTypeFlag{'1'}; 84 | const ANSICHAR FTarTypeFlagHelper::SymbolicLinkTypeFlag{'2'}; 85 | const ANSICHAR FTarTypeFlagHelper::CharacterDeviceTypeFlag{'3'}; 86 | const ANSICHAR FTarTypeFlagHelper::BlockDeviceTypeFlag{'4'}; 87 | const ANSICHAR FTarTypeFlagHelper::FIFOTypeFlag{'6'}; 88 | 89 | const TMap FTarTypeFlagHelper::Strings{ 90 | {FileTypeFlag, TEXT("File")}, {FileTypeFlag1, TEXT("File")}, 91 | {DirectoryTypeFlag, TEXT("Directory")}, 92 | {HardLinkTypeFlag, TEXT("Hard link")}, 93 | {SymbolicLinkTypeFlag, TEXT("Symbolic link")}, 94 | {CharacterDeviceTypeFlag, TEXT("Character device")}, 95 | {BlockDeviceTypeFlag, TEXT("Block device")}, 96 | {FIFOTypeFlag, TEXT("FIFO")} 97 | }; 98 | 99 | /** 100 | * Helper that handles checksum 101 | */ 102 | class FTarChecksumHelper 103 | { 104 | public: 105 | /** 106 | * Check if the specified tar header has a valid checksum 107 | * 108 | * @param Header Header to check 109 | * @return Whether the checksum is valid or not 110 | */ 111 | static bool IsValid(const FTarHeader& Header) 112 | { 113 | if (*Header.Checksum == '\0') 114 | { 115 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Checksum in tar header '%s' is NULL"), StringCast(Header.GetName()).Get()); 116 | return false; 117 | } 118 | 119 | if (BuildChecksum(Header) != Header.GetChecksum()) 120 | { 121 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Checksum in tar header %s' is invalid"), StringCast(Header.GetName()).Get()) 122 | return false; 123 | } 124 | 125 | return true; 126 | } 127 | 128 | /** 129 | * Build a checksum based primarily on tar header offsets 130 | * 131 | * @param Header Tar header based on which to build the checksum 132 | * @return Built checksum 133 | */ 134 | static uint32 BuildChecksum(const FTarHeader& Header) 135 | { 136 | const uint8* HeaderPtr = reinterpret_cast(&Header); 137 | 138 | uint32 Checksum = 256; 139 | 140 | for (size_t Index = 0; Index < STRUCT_OFFSET(FTarHeader, Checksum); ++Index) 141 | { 142 | Checksum += HeaderPtr[Index]; 143 | } 144 | 145 | for (size_t Index = STRUCT_OFFSET(FTarHeader, TypeFlag); Index < sizeof(Header); ++Index) 146 | { 147 | Checksum += HeaderPtr[Index]; 148 | } 149 | 150 | return Checksum; 151 | } 152 | }; 153 | 154 | FTarHeader::FTarHeader() 155 | { 156 | FMemory::Memset(this, 0, sizeof(FTarHeader)); 157 | } 158 | 159 | bool FTarHeader::ToEntry(const FTarHeader& Header, int32 Index, FRuntimeArchiveEntry& Entry) 160 | { 161 | if (!FTarChecksumHelper::IsValid(Header)) 162 | { 163 | return false; 164 | } 165 | 166 | Entry.Index = Index; 167 | Entry.Name = StringCast(Header.GetName()).Get(); 168 | Entry.bIsDirectory = FTarTypeFlagHelper::IsDirectory(Header.GetTypeFlag()); 169 | Entry.UncompressedSize = Header.GetSize(); 170 | Entry.CompressedSize = RuntimeArchiverTarOperations::RoundUp(Entry.UncompressedSize, 512); 171 | Entry.CreationTime = FDateTime::FromUnixTimestamp(Header.GetTime()); 172 | 173 | return true; 174 | } 175 | 176 | bool FTarHeader::FromEntry(const FRuntimeArchiveEntry& Entry, FTarHeader& Header) 177 | { 178 | return GenerateHeader(Entry.Name, Entry.UncompressedSize, Entry.CreationTime, Entry.bIsDirectory, Header); 179 | } 180 | 181 | bool FTarHeader::GenerateHeader(const FString& Name, int64 Size, const FDateTime& CreationTime, bool bIsDirectory, FTarHeader& Header) 182 | { 183 | Header = FTarHeader(); 184 | 185 | if (!Header.SetName(StringCast(*Name).Get())) 186 | { 187 | return false; 188 | } 189 | 190 | // Setting all possible permissions for Unix 191 | Header.SetMode(0777); 192 | 193 | Header.SetSize(Size); 194 | Header.SetTime(CreationTime.ToUnixTimestamp()); 195 | Header.SetTypeFlag(FTarTypeFlagHelper::GetTypeFlag(bIsDirectory)); 196 | Header.SetChecksum(FTarChecksumHelper::BuildChecksum(Header)); 197 | 198 | return true; 199 | } 200 | 201 | const RA_UTF8CHAR* FTarHeader::GetName() const 202 | { 203 | return Name; 204 | } 205 | 206 | bool FTarHeader::SetName(const RA_UTF8CHAR* InName) 207 | { 208 | // Check if length is within bounds 209 | if (TCString::Strlen(InName) > UE_ARRAY_COUNT(Name)) 210 | { 211 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Name '%s' is too long for tar header. Maximum length is %d"), StringCast(InName).Get(), static_cast(UE_ARRAY_COUNT(Name))); 212 | return false; 213 | } 214 | TCString::Strncpy(Name, InName, UE_ARRAY_COUNT(Name)); 215 | return true; 216 | } 217 | 218 | uint32 FTarHeader::GetMode() const 219 | { 220 | return RuntimeArchiverTarOperations::OctalToDecimal(Mode); 221 | } 222 | 223 | void FTarHeader::SetMode(uint32 InMode) 224 | { 225 | RuntimeArchiverTarOperations::DecimalToOctal(InMode, Mode, UE_ARRAY_COUNT(Mode)); 226 | } 227 | 228 | uint32 FTarHeader::GetOwner() const 229 | { 230 | return RuntimeArchiverTarOperations::OctalToDecimal(Owner); 231 | } 232 | 233 | void FTarHeader::SetOwner(uint32 InOwner) 234 | { 235 | RuntimeArchiverTarOperations::DecimalToOctal(InOwner, Owner, UE_ARRAY_COUNT(Owner)); 236 | } 237 | 238 | const ANSICHAR* FTarHeader::GetGroup() const 239 | { 240 | return Group; 241 | } 242 | 243 | bool FTarHeader::SetGroup(const ANSICHAR* InGroup) 244 | { 245 | // Check if length is within bounds 246 | if (FCStringAnsi::Strlen(InGroup) > UE_ARRAY_COUNT(Group)) 247 | { 248 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Group '%s' is too long for tar header. Maximum length is %d"), StringCast(InGroup).Get(), static_cast(UE_ARRAY_COUNT(Group))); 249 | return false; 250 | } 251 | FCStringAnsi::Strncpy(Group, InGroup, UE_ARRAY_COUNT(Group)); 252 | return true; 253 | } 254 | 255 | int64 FTarHeader::GetSize() const 256 | { 257 | return RuntimeArchiverTarOperations::OctalToDecimal(Size); 258 | } 259 | 260 | void FTarHeader::SetSize(int64 InSize) 261 | { 262 | RuntimeArchiverTarOperations::DecimalToOctal(InSize, Size, UE_ARRAY_COUNT(Size)); 263 | } 264 | 265 | int64 FTarHeader::GetTime() const 266 | { 267 | return RuntimeArchiverTarOperations::OctalToDecimal(Time); 268 | } 269 | 270 | void FTarHeader::SetTime(int64 InTime) 271 | { 272 | RuntimeArchiverTarOperations::DecimalToOctal(InTime, Time, UE_ARRAY_COUNT(Time)); 273 | } 274 | 275 | uint32 FTarHeader::GetChecksum() const 276 | { 277 | return RuntimeArchiverTarOperations::OctalToDecimal(Checksum); 278 | } 279 | 280 | void FTarHeader::SetChecksum(uint32 InChecksum) 281 | { 282 | RuntimeArchiverTarOperations::DecimalToOctal(InChecksum, Checksum, UE_ARRAY_COUNT(Checksum)); 283 | } 284 | 285 | ANSICHAR FTarHeader::GetTypeFlag() const 286 | { 287 | return TypeFlag; 288 | } 289 | 290 | void FTarHeader::SetTypeFlag(ANSICHAR InTypeFlag) 291 | { 292 | TypeFlag = InTypeFlag; 293 | } 294 | 295 | const ANSICHAR* FTarHeader::GetLinkName() const 296 | { 297 | return LinkName; 298 | } 299 | 300 | bool FTarHeader::SetLinkName(const ANSICHAR* InLinkName) 301 | { 302 | if (FCStringAnsi::Strlen(InLinkName) > UE_ARRAY_COUNT(LinkName)) 303 | { 304 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Link name '%s' is too long for tar header. Maximum length is %d"), StringCast(InLinkName).Get(), static_cast(UE_ARRAY_COUNT(LinkName))); 305 | return false; 306 | } 307 | FCStringAnsi::Strncpy(LinkName, InLinkName, UE_ARRAY_COUNT(LinkName)); 308 | return true; 309 | } 310 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Private/ArchiverTar/RuntimeArchiverTarHeader.h: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #pragma once 4 | 5 | #include "Engine/EngineBaseTypes.h" 6 | #include "Misc/EngineVersionComparison.h" 7 | #include "Misc/CString.h" 8 | 9 | // On UE < 5.0, UTF-8 characters are not supported, so we should at least use ANSI characters 10 | #if UE_VERSION_OLDER_THAN(5, 0, 0) 11 | using RA_UTF8CHAR = ANSICHAR; 12 | #else 13 | using RA_UTF8CHAR = UTF8CHAR; 14 | #endif 15 | 16 | class FTarChecksumHelper; 17 | struct FRuntimeArchiveEntry; 18 | 19 | /** 20 | * Tar header used to denote archived data 21 | */ 22 | struct FTarHeader 23 | { 24 | friend FTarChecksumHelper; 25 | 26 | FTarHeader(); 27 | 28 | private: 29 | /** Entry name */ 30 | RA_UTF8CHAR Name[100 / sizeof(RA_UTF8CHAR)]; 31 | 32 | /** Permission bits */ 33 | ANSICHAR Mode[8]; 34 | 35 | /** Owner user ID (UID) */ 36 | ANSICHAR Owner[8]; 37 | 38 | /** Group ID (GID) */ 39 | ANSICHAR Group[8]; 40 | 41 | /** Size in bytes */ 42 | ANSICHAR Size[12]; 43 | 44 | /** Time of last modification */ 45 | ANSICHAR Time[12]; 46 | 47 | /** Sum of all bytes in the header block */ 48 | ANSICHAR Checksum[8]; 49 | 50 | /** Entry type (e.g. directory or file) */ 51 | ANSICHAR TypeFlag; 52 | 53 | /** Name of target file name. Not used, but denotes some rarely used entry types */ 54 | ANSICHAR LinkName[100]; 55 | 56 | /** Unused padding */ 57 | ANSICHAR Padding[255]; 58 | 59 | // TODO: Replace Padding with additional fields (Magic, Version, UName, GName, DevMajor, DevMinor, Prefix) for better compatibility 60 | 61 | public: 62 | /** 63 | * Convert a tar header to an archive entry 64 | * 65 | * @param Header Tar header to convert from 66 | * @param Index Tar header index 67 | * @param Entry Filled archive entry 68 | * @return Whether the conversion was successful or not 69 | */ 70 | static bool ToEntry(const FTarHeader& Header, int32 Index, FRuntimeArchiveEntry& Entry); 71 | 72 | /** 73 | * Convert an archive entry to a tar header 74 | * 75 | * @param Entry Archive entry to convert from 76 | * @param Header Filled tar header 77 | * @return Whether the conversion was successful or not 78 | */ 79 | static bool FromEntry(const FRuntimeArchiveEntry& Entry, FTarHeader& Header); 80 | 81 | /** 82 | * Generate tar header based on input 83 | * 84 | * @param Name Entry name 85 | * @param Size Data size 86 | * @param CreationTime Entry creation time 87 | * @param bIsDirectory Whether the entry is a directory or a file 88 | * @param Header Filled tar header 89 | * @return Whether the conversion was successful or not 90 | */ 91 | static bool GenerateHeader(const FString& Name, int64 Size, const FDateTime& CreationTime, bool bIsDirectory, FTarHeader& Header); 92 | 93 | //~ Writing and reading tar header data 94 | 95 | const RA_UTF8CHAR* GetName() const; 96 | bool SetName(const RA_UTF8CHAR* InName); 97 | 98 | uint32 GetMode() const; 99 | void SetMode(uint32 InMode); 100 | 101 | uint32 GetOwner() const; 102 | void SetOwner(uint32 InOwner); 103 | 104 | const ANSICHAR* GetGroup() const; 105 | bool SetGroup(const ANSICHAR* InGroup); 106 | 107 | int64 GetSize() const; 108 | void SetSize(int64 InSize); 109 | 110 | int64 GetTime() const; 111 | void SetTime(int64 InTime); 112 | 113 | uint32 GetChecksum() const; 114 | void SetChecksum(uint32 InChecksum); 115 | 116 | ANSICHAR GetTypeFlag() const; 117 | void SetTypeFlag(ANSICHAR InTypeFlag); 118 | 119 | const ANSICHAR* GetLinkName() const; 120 | bool SetLinkName(const ANSICHAR* InLinkName); 121 | }; 122 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Private/ArchiverTar/RuntimeArchiverTarOperations.h: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #pragma once 4 | 5 | #include 6 | #include "Misc/CString.h" 7 | 8 | /** 9 | * Common functions that are used in tar operations 10 | */ 11 | namespace RuntimeArchiverTarOperations 12 | { 13 | 14 | /** 15 | * Round a number up by the specified increment 16 | */ 17 | template 18 | FORCEINLINE typename TEnableIf::Value, IntegralType>::Type 19 | RoundUp(IntegralType Number, IntegralType Increment) 20 | { 21 | return Number + (Increment - Number % Increment) % Increment; 22 | } 23 | 24 | /** 25 | * Convert string containing octal number to decimal number 26 | * This overload is for int32 27 | */ 28 | template 29 | typename TEnableIf::Value, DecimalType>::Type 30 | OctalToDecimal(const CharType* OctalString) 31 | { 32 | return std::stoi(OctalString, 0, 8); 33 | } 34 | 35 | /** 36 | * Convert string containing octal number to decimal number 37 | * This overload is for int64 38 | */ 39 | template 40 | typename TEnableIf::Value, DecimalType>::Type 41 | OctalToDecimal(const CharType* OctalString) 42 | { 43 | return std::stoll(OctalString, 0, 8); 44 | } 45 | 46 | /** 47 | * Convert string containing octal number to decimal number 48 | * This overload is for uint32 49 | */ 50 | template 51 | typename TEnableIf::Value, DecimalType>::Type 52 | OctalToDecimal(const CharType* OctalString) 53 | { 54 | return std::stoul(OctalString, 0, 8); 55 | } 56 | 57 | /** 58 | * Convert decimal number to string containing octal number 59 | * This overload is for int32 60 | */ 61 | template 62 | typename TEnableIf::Value>::Type 63 | DecimalToOctal(DecimalType Decimal, CharType* Octal, int32 MaxLength) 64 | { 65 | TCString::Sprintf(Octal, "%0*llo", MaxLength-1, Decimal); 66 | } 67 | 68 | /** 69 | * Convert decimal number to string containing octal number 70 | * This overload is for int32 71 | */ 72 | template 73 | typename TEnableIf::Value>::Type 74 | DecimalToOctal(DecimalType Decimal, CharType* Octal, int32 MaxLength) 75 | { 76 | TCString::Sprintf(Octal, "%0*o", MaxLength-1, Decimal); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Private/ArchiverZip/RuntimeArchiverZip.cpp: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #include "ArchiverZip/RuntimeArchiverZip.h" 4 | 5 | #include "RuntimeArchiverSubsystem.h" 6 | #include "RuntimeArchiverDefines.h" 7 | #include "RuntimeArchiverZipIncludes.h" 8 | #include "Misc/Paths.h" 9 | 10 | URuntimeArchiverZip::URuntimeArchiverZip() 11 | : Super::URuntimeArchiverBase() 12 | , bAppendMode(false) 13 | , MinizArchiver(nullptr) 14 | { 15 | } 16 | 17 | bool URuntimeArchiverZip::CreateArchiveInStorage(FString ArchivePath) 18 | { 19 | if (!Super::CreateArchiveInStorage(ArchivePath)) 20 | { 21 | return false; 22 | } 23 | 24 | FPaths::NormalizeFilename(ArchivePath); 25 | 26 | // Creating an archive in storage 27 | if (!mz_zip_writer_init_file_v2(static_cast(MinizArchiver), TCHAR_TO_UTF8(*ArchivePath), 0, MZ_ZIP_FLAG_WRITE_ZIP64)) 28 | { 29 | ReportError(ERuntimeArchiverErrorCode::NotInitialized, FString::Printf(TEXT("An error occurred while initializing zip archive '%s'"), *ArchivePath)); 30 | Reset(); 31 | return false; 32 | } 33 | 34 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully created zip archive '%s' in '%s'"), *GetName(), *ArchivePath); 35 | 36 | return true; 37 | } 38 | 39 | bool URuntimeArchiverZip::CreateArchiveInMemory(int32 InitialAllocationSize) 40 | { 41 | if (!Super::CreateArchiveInMemory(InitialAllocationSize)) 42 | { 43 | return false; 44 | } 45 | 46 | // Creating an archive in memory 47 | if (!mz_zip_writer_init_heap(static_cast(MinizArchiver), 0, static_cast(InitialAllocationSize))) 48 | { 49 | ReportError(ERuntimeArchiverErrorCode::NotInitialized, TEXT("Unable to initialize archive in memory")); 50 | Reset(); 51 | return false; 52 | } 53 | 54 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully created zip archive '%s' in memory"), *GetName()); 55 | 56 | return true; 57 | } 58 | 59 | bool URuntimeArchiverZip::OpenArchiveFromStorage(FString ArchivePath) 60 | { 61 | if (!Super::OpenArchiveFromStorage(ArchivePath)) 62 | { 63 | return false; 64 | } 65 | 66 | FPaths::NormalizeFilename(ArchivePath); 67 | 68 | // Reading the archive from the file path 69 | if (!mz_zip_reader_init_file(static_cast(MinizArchiver), TCHAR_TO_UTF8(*ArchivePath), 0)) 70 | { 71 | ReportError(ERuntimeArchiverErrorCode::NotInitialized, FString::Printf(TEXT("An error occurred while opening zip archive '%s' to read"), *ArchivePath)); 72 | Reset(); 73 | return false; 74 | } 75 | 76 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully opened zip archive '%s' in '%s' to read"), *GetName(), *ArchivePath); 77 | 78 | return true; 79 | } 80 | 81 | bool URuntimeArchiverZip::OpenArchiveFromMemory(const TArray64& ArchiveData) 82 | { 83 | if (!Super::OpenArchiveFromMemory(ArchiveData)) 84 | { 85 | return false; 86 | } 87 | 88 | if (!mz_zip_reader_init_mem(static_cast(MinizArchiver), ArchiveData.GetData(), ArchiveData.Num(), MZ_ZIP_FLAG_WRITE_ZIP64)) 89 | { 90 | ReportError(ERuntimeArchiverErrorCode::NotInitialized,TEXT("Unable to open in-memory zip archive to read")); 91 | Reset(); 92 | return false; 93 | } 94 | 95 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully opened in-memory zip archive '%s' to read"), *GetName()); 96 | 97 | return true; 98 | } 99 | 100 | bool URuntimeArchiverZip::CloseArchive() 101 | { 102 | if (!Super::CloseArchive()) 103 | { 104 | return false; 105 | } 106 | 107 | bool bResult{true}; 108 | 109 | switch (Mode) 110 | { 111 | case ERuntimeArchiverMode::Read: 112 | { 113 | bResult = static_cast(mz_zip_reader_end(static_cast(MinizArchiver))); 114 | break; 115 | } 116 | case ERuntimeArchiverMode::Write: 117 | { 118 | if (Location == ERuntimeArchiverLocation::Storage) 119 | { 120 | bResult = static_cast(mz_zip_writer_finalize_archive(static_cast(MinizArchiver))); 121 | } 122 | 123 | if (!mz_zip_writer_end(static_cast(MinizArchiver))) 124 | { 125 | bResult = false; 126 | } 127 | 128 | break; 129 | } 130 | default: 131 | { 132 | bResult = false; 133 | break; 134 | } 135 | } 136 | 137 | if (!bResult) 138 | { 139 | ReportError(ERuntimeArchiverErrorCode::CloseError, TEXT("Failed to close the archive")); 140 | return false; 141 | } 142 | 143 | Reset(); 144 | 145 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully closed zip archive '%s'"), *GetName()); 146 | 147 | return true; 148 | } 149 | 150 | bool URuntimeArchiverZip::GetArchiveData(TArray64& ArchiveData) 151 | { 152 | if (!Super::GetArchiveData(ArchiveData)) 153 | { 154 | return false; 155 | } 156 | 157 | if (Mode != ERuntimeArchiverMode::Write || Location != ERuntimeArchiverLocation::Memory) 158 | { 159 | ReportError(ERuntimeArchiverErrorCode::UnsupportedMode, FString::Printf(TEXT("Only '%s' mode and '%s' location are supported to get zip archive data (using mode: '%s', using location: '%s')"), 160 | *UEnum::GetValueAsName(ERuntimeArchiverMode::Write).ToString(), *UEnum::GetValueAsName(ERuntimeArchiverLocation::Memory).ToString(), 161 | *UEnum::GetValueAsName(Mode).ToString(), *UEnum::GetValueAsName(Location).ToString())); 162 | return false; 163 | } 164 | 165 | mz_zip_archive* MinizArchiverReal = static_cast(MinizArchiver); 166 | 167 | if (!mz_zip_writer_finalize_archive(MinizArchiverReal)) 168 | { 169 | ReportError(ERuntimeArchiverErrorCode::GetError, FString::Printf(TEXT("Unable to get zip archive data from memory"))); 170 | return false; 171 | } 172 | 173 | const int64 ArchiveSize = static_cast(MinizArchiverReal->m_archive_size); 174 | 175 | ArchiveData = TArray64(static_cast(MinizArchiverReal->m_pState->m_pMem), ArchiveSize); 176 | 177 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully retrieved zip archive data from memory with size %lld bytes"), ArchiveSize); 178 | 179 | return true; 180 | } 181 | 182 | bool URuntimeArchiverZip::GetArchiveEntries(int32& NumOfArchiveEntries) 183 | { 184 | if (!Super::GetArchiveEntries(NumOfArchiveEntries)) 185 | { 186 | return false; 187 | } 188 | 189 | NumOfArchiveEntries = mz_zip_reader_get_num_files(static_cast(MinizArchiver)); 190 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully retrieved %d zip entries"), NumOfArchiveEntries); 191 | return true; 192 | } 193 | 194 | bool URuntimeArchiverZip::GetArchiveEntryInfoByName(FString EntryName, FRuntimeArchiveEntry& EntryInfo) 195 | { 196 | if (!Super::GetArchiveEntryInfoByName(EntryName, EntryInfo)) 197 | { 198 | return false; 199 | } 200 | 201 | FPaths::NormalizeFilename(EntryName); 202 | 203 | // Looking for the entry index depending on the entry name 204 | const int32 EntryIndex = mz_zip_reader_locate_file(static_cast(MinizArchiver), TCHAR_TO_UTF8(*EntryName), nullptr, 0); 205 | 206 | if (EntryIndex == -1) 207 | { 208 | ReportError(ERuntimeArchiverErrorCode::GetError, FString::Printf(TEXT("Unable to find zip entry index under the entry name '%s'"), *EntryName)); 209 | return false; 210 | } 211 | 212 | return GetArchiveEntryInfoByIndex(EntryIndex, EntryInfo); 213 | } 214 | 215 | bool URuntimeArchiverZip::GetArchiveEntryInfoByIndex(int32 EntryIndex, FRuntimeArchiveEntry& EntryInfo) 216 | { 217 | if (!Super::GetArchiveEntryInfoByIndex(EntryIndex, EntryInfo)) 218 | { 219 | return false; 220 | } 221 | 222 | // Get file information 223 | mz_zip_archive_file_stat ArchiveFileStat; 224 | const bool bResult = static_cast(mz_zip_reader_file_stat(static_cast(MinizArchiver), static_cast(EntryIndex), &ArchiveFileStat)); 225 | 226 | if (!bResult) 227 | { 228 | ReportError(ERuntimeArchiverErrorCode::GetError, FString::Printf(TEXT("Unable to get zip entry information by index %d"), EntryIndex)); 229 | return false; 230 | } 231 | 232 | // Filling in the entry data 233 | { 234 | EntryInfo.Index = static_cast(ArchiveFileStat.m_file_index); 235 | EntryInfo.CompressedSize = static_cast(ArchiveFileStat.m_comp_size); 236 | EntryInfo.UncompressedSize = static_cast(ArchiveFileStat.m_uncomp_size); 237 | #ifndef MINIZ_NO_TIME 238 | EntryInfo.CreationTime = FDateTime::FromUnixTimestamp(ArchiveFileStat.m_time); 239 | #endif 240 | EntryInfo.Name = UTF8_TO_TCHAR(ArchiveFileStat.m_filename); 241 | EntryInfo.bIsDirectory = static_cast(mz_zip_reader_is_file_a_directory(static_cast(MinizArchiver), static_cast(EntryIndex))); 242 | } 243 | 244 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully retrieved zip entry '%s'"), *EntryInfo.Name); 245 | 246 | return true; 247 | } 248 | 249 | bool URuntimeArchiverZip::AddEntryFromMemory(FString EntryName, const TArray64& DataToBeArchived, ERuntimeArchiverCompressionLevel CompressionLevel) 250 | { 251 | if (!Super::AddEntryFromMemory(EntryName, DataToBeArchived, CompressionLevel)) 252 | { 253 | return false; 254 | } 255 | 256 | FPaths::NormalizeFilename(EntryName); 257 | 258 | // Writing data to the entry from memory 259 | const bool bResult = static_cast(mz_zip_writer_add_mem_ex(static_cast(MinizArchiver), TCHAR_TO_UTF8(*EntryName), 260 | DataToBeArchived.GetData(), static_cast(DataToBeArchived.Num()), 261 | nullptr, 0, 262 | static_cast(CompressionLevel), 0, 0)); 263 | 264 | if (!bResult) 265 | { 266 | ReportError(ERuntimeArchiverErrorCode::AddError, FString::Printf(TEXT("Unable to add zip entry '%s' from memory"), *EntryName)); 267 | return false; 268 | } 269 | 270 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully added zip entry '%s' from memory with size '%lld'"), *EntryName, static_cast(DataToBeArchived.Num())); 271 | 272 | return true; 273 | } 274 | 275 | bool URuntimeArchiverZip::ExtractEntryToMemory(const FRuntimeArchiveEntry& EntryInfo, TArray64& UnarchivedData) 276 | { 277 | if (!Super::ExtractEntryToMemory(EntryInfo, UnarchivedData)) 278 | { 279 | return false; 280 | } 281 | 282 | if (Mode != ERuntimeArchiverMode::Read) 283 | { 284 | ReportError(ERuntimeArchiverErrorCode::UnsupportedMode, FString::Printf(TEXT("Only '%s' mode is supported for extracting zip entries (using mode: '%s')"), *UEnum::GetValueAsName(ERuntimeArchiverMode::Read).ToString(), *UEnum::GetValueAsName(Mode).ToString())); 285 | return false; 286 | } 287 | 288 | int32 NumOfArchiveEntries; 289 | if (!GetArchiveEntries(NumOfArchiveEntries) || EntryInfo.Index > (NumOfArchiveEntries - 1)) 290 | { 291 | ReportError(ERuntimeArchiverErrorCode::InvalidArgument, FString::Printf(TEXT("Zip entry index %d is invalid. Min index: 0, Max index: %d"), EntryInfo.Index, (NumOfArchiveEntries - 1))); 292 | return false; 293 | } 294 | 295 | // Extracting to memory 296 | size_t EntryInMemorySize; 297 | void* EntryInMemoryPtr = mz_zip_reader_extract_to_heap(static_cast(MinizArchiver), static_cast(EntryInfo.Index), &EntryInMemorySize, 0); 298 | 299 | if (!EntryInMemoryPtr) 300 | { 301 | ReportError(ERuntimeArchiverErrorCode::ExtractError, FString::Printf(TEXT("Unable to extract zip entry '%s' into memory"), *EntryInfo.Name)); 302 | return false; 303 | } 304 | 305 | UnarchivedData = TArray64(static_cast(EntryInMemoryPtr), EntryInMemorySize); 306 | 307 | FMemory::Free(EntryInMemoryPtr); 308 | 309 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully extracted zip entry '%s' into memory"), *EntryInfo.Name); 310 | 311 | return true; 312 | } 313 | 314 | bool URuntimeArchiverZip::Initialize() 315 | { 316 | if (!Super::Initialize()) 317 | { 318 | return false; 319 | } 320 | 321 | // Creating Miniz archiver 322 | MinizArchiver = static_cast(FMemory::Memzero(FMemory::Malloc(sizeof(mz_zip_archive)), sizeof(mz_zip_archive))); 323 | 324 | if (!MinizArchiver) 325 | { 326 | ReportError(ERuntimeArchiverErrorCode::NotInitialized, TEXT("Unable to allocate memory for zip archiver")); 327 | return false; 328 | } 329 | 330 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully initialized zip archiver '%s'"), *GetName()); 331 | 332 | return true; 333 | } 334 | 335 | bool URuntimeArchiverZip::IsInitialized() const 336 | { 337 | return Super::IsInitialized() && MinizArchiver; 338 | } 339 | 340 | void URuntimeArchiverZip::Reset() 341 | { 342 | if (MinizArchiver) 343 | { 344 | FMemory::Free(MinizArchiver); 345 | MinizArchiver = nullptr; 346 | } 347 | 348 | Super::Reset(); 349 | 350 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully uninitialized zip archiver '%s'"), *GetName()); 351 | } 352 | 353 | void URuntimeArchiverZip::ReportError(ERuntimeArchiverErrorCode ErrorCode, const FString& ErrorString) const 354 | { 355 | FString ReadyErrorString = FString::Printf(TEXT("%s: %s."), *UEnum::GetValueAsName(ErrorCode).ToString(), *ErrorString); 356 | 357 | if (IsInitialized()) 358 | { 359 | mz_zip_archive* MinizArchiverReal = static_cast(MinizArchiver); 360 | 361 | const mz_zip_error LastMinizError = mz_zip_get_last_error(MinizArchiverReal); 362 | const FString LastMinizErrorStr = UTF8_TO_TCHAR(mz_zip_get_error_string(LastMinizError)); 363 | 364 | // Cleaning last miniz error to avoid getting the same error next time 365 | if (LastMinizError != MZ_ZIP_NO_ERROR) 366 | { 367 | mz_zip_set_error(MinizArchiverReal, MZ_ZIP_NO_ERROR); 368 | ReadyErrorString += FString::Printf(TEXT("\nMiniz error details: '%s'."), *LastMinizErrorStr); 369 | } 370 | } 371 | 372 | Super::ReportError(ErrorCode, ReadyErrorString); 373 | } 374 | 375 | bool URuntimeArchiverZip::OpenArchiveFromStorageToAppend(FString ArchivePath) 376 | { 377 | if (!Initialize()) 378 | { 379 | return false; 380 | } 381 | 382 | FPaths::NormalizeFilename(ArchivePath); 383 | 384 | if (ArchivePath.IsEmpty() || !FPaths::FileExists(ArchivePath)) 385 | { 386 | ReportError(ERuntimeArchiverErrorCode::InvalidArgument, FString::Printf(TEXT("Archive '%s' does not exist"), *ArchivePath)); 387 | return false; 388 | } 389 | 390 | bAppendMode = true; 391 | 392 | Mode = ERuntimeArchiverMode::Write; 393 | Location = ERuntimeArchiverLocation::Storage; 394 | 395 | // Reading the archive from the file path 396 | if (!mz_zip_reader_init_file(static_cast(MinizArchiver), TCHAR_TO_UTF8(*ArchivePath), 0)) 397 | { 398 | ReportError(ERuntimeArchiverErrorCode::NotInitialized, FString::Printf(TEXT("An error occurred while opening zip archive '%s'"), *ArchivePath)); 399 | Reset(); 400 | return false; 401 | } 402 | 403 | // Initializing the archive for adding entries 404 | if (!mz_zip_writer_init_from_reader_v2(static_cast(MinizArchiver), TCHAR_TO_UTF8(*ArchivePath), 0)) 405 | { 406 | mz_zip_reader_end(static_cast(MinizArchiver)); 407 | ReportError(ERuntimeArchiverErrorCode::NotInitialized, FString::Printf(TEXT("An error occurred while opening zip archive '%s' to append"), *ArchivePath)); 408 | Reset(); 409 | return false; 410 | } 411 | 412 | UE_LOG(LogRuntimeArchiver, Log, TEXT("Successfully opened zip archive '%s' in '%s' to append"), *GetName(), *ArchivePath); 413 | 414 | return true; 415 | } 416 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Private/ArchiverZip/RuntimeArchiverZipIncludes.h: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #pragma once 4 | 5 | #include "CoreTypes.h" 6 | 7 | #ifndef __ORDER_LITTLE_ENDIAN__ 8 | #define __ORDER_LITTLE_ENDIAN__ PLATFORM_LITTLE_ENDIAN 9 | #endif 10 | 11 | #ifndef MINIZ_USE_UNALIGNED_LOADS_AND_STORES 12 | #define MINIZ_USE_UNALIGNED_LOADS_AND_STORES PLATFORM_SUPPORTS_UNALIGNED_LOADS 13 | #endif 14 | 15 | #ifndef MINIZ_LITTLE_ENDIAN 16 | #define MINIZ_LITTLE_ENDIAN PLATFORM_LITTLE_ENDIAN 17 | #endif 18 | 19 | #ifndef MINIZ_HAS_64BIT_REGISTERS 20 | #define MINIZ_HAS_64BIT_REGISTERS PLATFORM_64BITS 21 | #endif 22 | 23 | #ifndef _LARGEFILE64_SOURCE 24 | #define _LARGEFILE64_SOURCE 25 | #endif 26 | 27 | #pragma warning( push ) 28 | #pragma warning( disable : 4334) 29 | 30 | #if PLATFORM_WINDOWS 31 | #define WIN32_LEAN_AND_MEAN 32 | #include "Windows/AllowWindowsPlatformTypes.h" 33 | #endif 34 | 35 | #undef malloc 36 | #undef free 37 | #undef realloc 38 | #undef memset 39 | #undef memcpy 40 | 41 | #define malloc(Count) FMemory::Malloc(Count) 42 | #define free(Original) FMemory::Free(Original) 43 | #define realloc(Original, Count) FMemory::Realloc(Original, Count) 44 | #define memset(Dest, Char, Count) FMemory::Memset(Dest, Char, Count) 45 | #define memcpy(Dest, Source, Count) FMemory::Memcpy(Dest, Source, Count) 46 | 47 | THIRD_PARTY_INCLUDES_START 48 | #include "miniz_export.h" 49 | #include "miniz.c" 50 | #include "miniz_zip.c" 51 | #include "miniz_tinfl.c" 52 | #include "miniz_tdef.c" 53 | THIRD_PARTY_INCLUDES_END 54 | 55 | #undef malloc 56 | #undef free 57 | #undef realloc 58 | #undef memset 59 | #undef memcpy 60 | 61 | #if PLATFORM_WINDOWS 62 | #include "Windows/HideWindowsPlatformTypes.h" 63 | #endif 64 | 65 | #pragma warning( pop ) -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Private/AsyncTasks/RuntimeArchiverArchiveAsyncTask.cpp: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #include "AsyncTasks/RuntimeArchiverArchiveAsyncTask.h" 4 | 5 | URuntimeArchiverArchiveAsyncTask* URuntimeArchiverArchiveAsyncTask::ArchiveDirectory(TSubclassOf ArchiverClass, FString ArchivePath, FString DirectoryPath, bool bAddParentDirectory, ERuntimeArchiverCompressionLevel CompressionLevel) 6 | { 7 | URuntimeArchiverArchiveAsyncTask* ArchiveTask = NewObject(); 8 | 9 | ArchiveTask->Archiver = URuntimeArchiverBase::CreateRuntimeArchiver(ArchiveTask, ArchiverClass); 10 | ArchiveTask->Archiver->SetInternalFlags(EInternalObjectFlags::Async); 11 | 12 | { 13 | ArchiveTask->OperationType = EOperationType::Directory; 14 | ArchiveTask->DirectoryInfo = {MoveTemp(ArchivePath), MoveTemp(DirectoryPath), bAddParentDirectory, CompressionLevel}; 15 | } 16 | 17 | return ArchiveTask; 18 | } 19 | 20 | URuntimeArchiverArchiveAsyncTask* URuntimeArchiverArchiveAsyncTask::ArchiveFiles(TSubclassOf ArchiverClass, FString ArchivePath, TArray FilePaths, ERuntimeArchiverCompressionLevel CompressionLevel) 21 | { 22 | URuntimeArchiverArchiveAsyncTask* ArchiveTask = NewObject(); 23 | 24 | ArchiveTask->Archiver = URuntimeArchiverBase::CreateRuntimeArchiver(ArchiveTask, ArchiverClass); 25 | ArchiveTask->Archiver->SetInternalFlags(EInternalObjectFlags::Async); 26 | 27 | { 28 | ArchiveTask->OperationType = EOperationType::Files; 29 | ArchiveTask->FilesInfo = {MoveTemp(ArchivePath), MoveTemp(FilePaths), CompressionLevel}; 30 | } 31 | 32 | return ArchiveTask; 33 | } 34 | 35 | void URuntimeArchiverArchiveAsyncTask::Activate() 36 | { 37 | Super::Activate(); 38 | 39 | switch (OperationType) 40 | { 41 | case EOperationType::Directory: 42 | { 43 | StartDirectory(); 44 | break; 45 | } 46 | case EOperationType::Files: 47 | { 48 | StartFiles(); 49 | break; 50 | } 51 | } 52 | } 53 | 54 | void URuntimeArchiverArchiveAsyncTask::StartDirectory() 55 | { 56 | if (!Archiver->CreateArchiveInStorage(DirectoryInfo.ArchivePath)) 57 | { 58 | OnResult_Callback(false); 59 | return; 60 | } 61 | 62 | OperationResult.BindDynamic(this, &URuntimeArchiverArchiveAsyncTask::OnResult_Callback); 63 | 64 | Archiver->AddEntriesFromStorage_Directory(OperationResult, MoveTemp(DirectoryInfo.DirectoryPath), DirectoryInfo.bAddParentDirectory, DirectoryInfo.CompressionLevel); 65 | } 66 | 67 | void URuntimeArchiverArchiveAsyncTask::StartFiles() 68 | { 69 | if (!Archiver->CreateArchiveInStorage(FilesInfo.ArchivePath)) 70 | { 71 | OnResult_Callback(false); 72 | return; 73 | } 74 | 75 | OperationResult.BindDynamic(this, &URuntimeArchiverArchiveAsyncTask::OnResult_Callback); 76 | OperationProgress.BindDynamic(this, &URuntimeArchiverArchiveAsyncTask::OnProgress_Callback); 77 | 78 | Archiver->AddEntriesFromStorage(OperationResult, OperationProgress, MoveTemp(FilesInfo.FilePaths), FilesInfo.CompressionLevel); 79 | } 80 | 81 | void URuntimeArchiverArchiveAsyncTask::OnResult_Callback(bool bSuccess) 82 | { 83 | OperationResult.Clear(); 84 | 85 | if (!bSuccess || !Archiver->CloseArchive()) 86 | { 87 | OnFail.Broadcast(0); 88 | return; 89 | } 90 | 91 | OnSuccess.Broadcast(100); 92 | 93 | SetReadyToDestroy(); 94 | Archiver->ClearInternalFlags(EInternalObjectFlags::Async); 95 | } 96 | 97 | void URuntimeArchiverArchiveAsyncTask::OnProgress_Callback(int32 Percentage) 98 | { 99 | OnProgress.Broadcast(Percentage); 100 | } 101 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Private/AsyncTasks/RuntimeArchiverUnarchiveAsyncTask.cpp: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #include "AsyncTasks/RuntimeArchiverUnarchiveAsyncTask.h" 4 | 5 | #include "RuntimeArchiverDefines.h" 6 | 7 | URuntimeArchiverUnarchiveAsyncTask* URuntimeArchiverUnarchiveAsyncTask::UnarchiveDirectory(TSubclassOf ArchiverClass, FString ArchivePath, FString EntryName, FString DirectoryPath, bool bAddParentDirectory, bool bForceOverwrite) 8 | { 9 | URuntimeArchiverUnarchiveAsyncTask* ArchiveTask = NewObject(); 10 | 11 | ArchiveTask->Archiver = URuntimeArchiverBase::CreateRuntimeArchiver(ArchiveTask, ArchiverClass); 12 | ArchiveTask->Archiver->SetInternalFlags(EInternalObjectFlags::Async); 13 | 14 | { 15 | ArchiveTask->OperationType = EOperationType::Directory; 16 | ArchiveTask->DirectoryInfo = {MoveTemp(ArchivePath), MoveTemp(EntryName), MoveTemp(DirectoryPath), bAddParentDirectory, bForceOverwrite}; 17 | } 18 | 19 | return ArchiveTask; 20 | } 21 | 22 | URuntimeArchiverUnarchiveAsyncTask* URuntimeArchiverUnarchiveAsyncTask::UnarchiveFiles(TSubclassOf ArchiverClass, FString ArchivePath, TArray EntryNames, FString DirectoryPath, bool bForceOverwrite) 23 | { 24 | URuntimeArchiverUnarchiveAsyncTask* ArchiveTask = NewObject(); 25 | 26 | ArchiveTask->Archiver = URuntimeArchiverBase::CreateRuntimeArchiver(ArchiveTask, ArchiverClass); 27 | ArchiveTask->Archiver->SetInternalFlags(EInternalObjectFlags::Async); 28 | 29 | { 30 | ArchiveTask->OperationType = EOperationType::Files; 31 | ArchiveTask->FilesInfo = {MoveTemp(ArchivePath), MoveTemp(EntryNames), MoveTemp(DirectoryPath), bForceOverwrite}; 32 | } 33 | 34 | return ArchiveTask; 35 | } 36 | 37 | void URuntimeArchiverUnarchiveAsyncTask::Activate() 38 | { 39 | Super::Activate(); 40 | 41 | switch (OperationType) 42 | { 43 | case EOperationType::Directory: 44 | { 45 | StartDirectory(); 46 | break; 47 | } 48 | case EOperationType::Files: 49 | { 50 | StartFiles(); 51 | break; 52 | } 53 | } 54 | } 55 | 56 | void URuntimeArchiverUnarchiveAsyncTask::StartDirectory() 57 | { 58 | if (!Archiver->OpenArchiveFromStorage(DirectoryInfo.ArchivePath)) 59 | { 60 | OnFail.Broadcast(0); 61 | return; 62 | } 63 | 64 | OperationResult.BindDynamic(this, &URuntimeArchiverUnarchiveAsyncTask::OnResult_Callback); 65 | 66 | Archiver->ExtractEntriesToStorage_Directory(OperationResult, MoveTemp(DirectoryInfo.EntryName), MoveTemp(DirectoryInfo.DirectoryPath), DirectoryInfo.bAddParentDirectory, DirectoryInfo.bForceOverwrite); 67 | } 68 | 69 | void URuntimeArchiverUnarchiveAsyncTask::StartFiles() 70 | { 71 | if (!Archiver->OpenArchiveFromStorage(FilesInfo.ArchivePath)) 72 | { 73 | OnFail.Broadcast(0); 74 | return; 75 | } 76 | 77 | OperationResult.BindDynamic(this, &URuntimeArchiverUnarchiveAsyncTask::OnResult_Callback); 78 | OperationProgress.BindDynamic(this, &URuntimeArchiverUnarchiveAsyncTask::OnProgress_Callback); 79 | 80 | TArray Entries; 81 | 82 | // Getting archive entries by names 83 | { 84 | for (const FString& EntryName : FilesInfo.EntryNames) 85 | { 86 | FRuntimeArchiveEntry Entry; 87 | if (!Archiver->GetArchiveEntryInfoByName(EntryName, Entry)) 88 | { 89 | UE_LOG(LogRuntimeArchiver, Error, TEXT("Unable to find '%s' archive entry"), *EntryName); 90 | OnResult_Callback(false); 91 | return; 92 | } 93 | 94 | Entries.Add(Entry); 95 | } 96 | } 97 | 98 | Archiver->ExtractEntriesToStorage(OperationResult, OperationProgress, MoveTemp(Entries), MoveTemp(FilesInfo.DirectoryPath), MoveTemp(FilesInfo.bForceOverwrite)); 99 | } 100 | 101 | void URuntimeArchiverUnarchiveAsyncTask::OnResult_Callback(bool bSuccess) 102 | { 103 | OperationResult.Clear(); 104 | 105 | bSuccess &= Archiver->CloseArchive(); 106 | 107 | if (!bSuccess) 108 | { 109 | OnFail.Broadcast(0); 110 | return; 111 | } 112 | 113 | OnSuccess.Broadcast(100); 114 | 115 | SetReadyToDestroy(); 116 | Archiver->ClearInternalFlags(EInternalObjectFlags::Async); 117 | } 118 | 119 | void URuntimeArchiverUnarchiveAsyncTask::OnProgress_Callback(int32 Percentage) 120 | { 121 | OnProgress.Broadcast(Percentage); 122 | } 123 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Private/RuntimeArchiver.cpp: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #include "RuntimeArchiver.h" 4 | #include "RuntimeArchiverDefines.h" 5 | 6 | #define LOCTEXT_NAMESPACE "FRuntimeArchiverModule" 7 | 8 | void FRuntimeArchiverModule::StartupModule() 9 | { 10 | } 11 | 12 | void FRuntimeArchiverModule::ShutdownModule() 13 | { 14 | } 15 | 16 | #undef LOCTEXT_NAMESPACE 17 | 18 | IMPLEMENT_MODULE(FRuntimeArchiverModule, RuntimeArchiver) 19 | 20 | DEFINE_LOG_CATEGORY(LogRuntimeArchiver); 21 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Private/RuntimeArchiverSubsystem.cpp: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #include "RuntimeArchiverSubsystem.h" 4 | #include "Engine.h" 5 | 6 | URuntimeArchiverSubsystem* URuntimeArchiverSubsystem::GetArchiveSubsystem() 7 | { 8 | return GEngine->GetEngineSubsystem(); 9 | } 10 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Private/RuntimeArchiverUtilities.cpp: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #include "RuntimeArchiverUtilities.h" 4 | 5 | TArray URuntimeArchiverUtilities::ParseDirectories(const FString& FilePath) 6 | { 7 | TArray Directories; 8 | 9 | // All parts of directories on the left 10 | FString LeftDirectories; 11 | 12 | // Directory used to split 13 | FString DirectoryToSplit = FilePath; 14 | 15 | // Left side of split directory 16 | FString LeftDirectorySide; 17 | 18 | // Right side of split directory. It defines remaining directories 19 | FString RightDirectorySide; 20 | 21 | // Splitting the string by the forward slash (/) from left to right 22 | while (DirectoryToSplit.Split(TEXT("/"), &LeftDirectorySide, &RightDirectorySide, ESearchCase::IgnoreCase, ESearchDir::FromStart)) 23 | { 24 | // Adding the left side of a directory and a forward slash at the end 25 | LeftDirectories += LeftDirectorySide + TEXT("/"); 26 | 27 | // Adding the resulting string to the array 28 | Directories.Add(LeftDirectories); 29 | 30 | // Changing DirectoryToSplit to scan the right side of directories on the next loop 31 | DirectoryToSplit = RightDirectorySide; 32 | } 33 | 34 | return MoveTemp(Directories); 35 | } -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Private/Streams/RuntimeArchiverFileStream.cpp: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #include "Streams/RuntimeArchiverFileStream.h" 4 | 5 | #include "RuntimeArchiverDefines.h" 6 | #include "GenericPlatform/GenericPlatformFile.h" 7 | #include "HAL/PlatformFileManager.h" 8 | 9 | FRuntimeArchiverFileStream::FRuntimeArchiverFileStream(const FString& ArchivePath, bool bWrite) 10 | : FRuntimeArchiverBaseStream(bWrite) 11 | { 12 | IPlatformFile& PlatformFile{FPlatformFileManager::Get().GetPlatformFile()}; 13 | FileHandle = bWrite ? PlatformFile.OpenWrite(*ArchivePath, false, true) : PlatformFile.OpenRead(*ArchivePath, false); 14 | 15 | UE_LOG(LogRuntimeArchiver, Log, TEXT("File opened at '%s', bWrite: %s. Validity: %s"), 16 | *ArchivePath, bWrite ? TEXT("true") : TEXT("false"), FRuntimeArchiverFileStream::IsValid() ? TEXT("true") : TEXT("false")); 17 | } 18 | 19 | FRuntimeArchiverFileStream::~FRuntimeArchiverFileStream() 20 | { 21 | if (FRuntimeArchiverFileStream::IsValid()) 22 | { 23 | delete FileHandle; 24 | FileHandle = nullptr; 25 | } 26 | } 27 | 28 | bool FRuntimeArchiverFileStream::IsValid() const 29 | { 30 | return FileHandle != nullptr; 31 | } 32 | 33 | bool FRuntimeArchiverFileStream::Read(void* Data, int64 Size) 34 | { 35 | if (!IsValid()) 36 | { 37 | return false; 38 | } 39 | 40 | const bool bSuccess{FileHandle->Read(static_cast(Data), Size)}; 41 | Position = FileHandle->Tell(); 42 | return bSuccess; 43 | } 44 | 45 | bool FRuntimeArchiverFileStream::Write(const void* Data, int64 Size) 46 | { 47 | ensureMsgf(bWrite, TEXT("Cannot write data to the stream because it is in read-only mode")); 48 | 49 | if (!IsValid()) 50 | { 51 | return false; 52 | } 53 | 54 | const bool bSuccess{FileHandle->Write(static_cast(Data), Size)}; 55 | Position = FileHandle->Tell(); 56 | return bSuccess; 57 | } 58 | 59 | bool FRuntimeArchiverFileStream::Seek(int64 NewPosition) 60 | { 61 | if (!IsValid()) 62 | { 63 | return false; 64 | } 65 | 66 | const bool bSuccess{FileHandle->Seek(NewPosition)}; 67 | Position = FileHandle->Tell(); 68 | return bSuccess; 69 | } 70 | 71 | int64 FRuntimeArchiverFileStream::Size() 72 | { 73 | if (!IsValid()) 74 | { 75 | return -1; 76 | } 77 | 78 | return FileHandle->Size(); 79 | } 80 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Private/Streams/RuntimeArchiverMemoryStream.cpp: -------------------------------------------------------------------------------- 1 | #include "Streams/RuntimeArchiverMemoryStream.h" 2 | 3 | FRuntimeArchiverMemoryStream::FRuntimeArchiverMemoryStream(const TArray64& ArchiveData) 4 | : FRuntimeArchiverBaseStream(false) 5 | , ArchiveData(ArchiveData) 6 | { 7 | } 8 | 9 | FRuntimeArchiverMemoryStream::FRuntimeArchiverMemoryStream(int32 InitialAllocationSize) 10 | : FRuntimeArchiverBaseStream(true) 11 | { 12 | ArchiveData.SetNum(InitialAllocationSize); 13 | } 14 | 15 | bool FRuntimeArchiverMemoryStream::IsValid() const 16 | { 17 | return true; 18 | } 19 | 20 | bool FRuntimeArchiverMemoryStream::Read(void* Data, int64 Size) 21 | { 22 | if (!IsValid()) 23 | { 24 | return false; 25 | } 26 | 27 | if (Position + Size > ArchiveData.Num()) 28 | { 29 | return false; 30 | } 31 | 32 | const bool bSuccess{FMemory::Memcpy(Data, ArchiveData.GetData() + Position, Size) != nullptr}; 33 | Position += Size; 34 | 35 | return bSuccess; 36 | } 37 | 38 | bool FRuntimeArchiverMemoryStream::Write(const void* Data, int64 Size) 39 | { 40 | ensureMsgf(bWrite, TEXT("Cannot write data to the stream because it is in read-only mode")); 41 | 42 | if (!IsValid()) 43 | { 44 | return false; 45 | } 46 | 47 | const int64 NewPosition = Position + Size; 48 | 49 | if (NewPosition > ArchiveData.Num()) 50 | { 51 | ArchiveData.SetNumUninitialized(NewPosition); 52 | } 53 | 54 | const bool bSuccess{FMemory::Memcpy(ArchiveData.GetData() + Position, Data, Size) != nullptr}; 55 | Position = NewPosition; 56 | 57 | return bSuccess; 58 | } 59 | 60 | bool FRuntimeArchiverMemoryStream::Seek(int64 NewPosition) 61 | { 62 | if (!IsValid()) 63 | { 64 | return false; 65 | } 66 | 67 | if (NewPosition == Position) 68 | { 69 | return true; 70 | } 71 | 72 | if (NewPosition > ArchiveData.Num()) 73 | { 74 | if (NewPosition != 0) 75 | { 76 | return false; 77 | } 78 | } 79 | 80 | Position = NewPosition; 81 | 82 | return true; 83 | } 84 | 85 | int64 FRuntimeArchiverMemoryStream::Size() 86 | { 87 | if (!IsValid()) 88 | { 89 | return -1; 90 | } 91 | 92 | return ArchiveData.Num(); 93 | } 94 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Public/ArchiverGZip/RuntimeArchiverGZip.h: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "RuntimeArchiverBase.h" 7 | #include "ArchiverTar/RuntimeArchiverTar.h" 8 | #include "UObject/StrongObjectPtr.h" 9 | #include "RuntimeArchiverGZip.generated.h" 10 | 11 | /** 12 | * GZip archiver class. Works with tar.gz (tgz) archives 13 | * Archiving of data occurs through the Tar archiver and their subsequent compression through GZip raw archiver (the same applies for unarchiving) 14 | */ 15 | UCLASS(BlueprintType, Category = "Runtime Archiver") 16 | class RUNTIMEARCHIVER_API URuntimeArchiverGZip : public URuntimeArchiverBase 17 | { 18 | GENERATED_BODY() 19 | 20 | public: 21 | URuntimeArchiverGZip(); 22 | 23 | //~ Begin URuntimeArchiverBase Interface 24 | virtual bool CreateArchiveInStorage(FString ArchivePath) override; 25 | virtual bool CreateArchiveInMemory(int32 InitialAllocationSize = 0) override; 26 | 27 | virtual bool OpenArchiveFromStorage(FString ArchivePath) override; 28 | virtual bool OpenArchiveFromMemory(const TArray64& ArchiveData) override; 29 | 30 | virtual bool CloseArchive() override; 31 | 32 | virtual bool GetArchiveData(TArray64& ArchiveData) override; 33 | 34 | virtual bool GetArchiveEntries(int32& NumOfArchiveEntries) override; 35 | 36 | virtual bool GetArchiveEntryInfoByName(FString EntryName, FRuntimeArchiveEntry& EntryInfo) override; 37 | virtual bool GetArchiveEntryInfoByIndex(int32 EntryIndex, FRuntimeArchiveEntry& EntryInfo) override; 38 | 39 | virtual bool AddEntryFromMemory(FString EntryName, const TArray64& DataToBeArchived, ERuntimeArchiverCompressionLevel CompressionLevel) override; 40 | 41 | virtual bool ExtractEntryToMemory(const FRuntimeArchiveEntry& EntryInfo, TArray64& UnarchivedData) override; 42 | 43 | virtual bool Initialize() override; 44 | virtual bool IsInitialized() const override; 45 | virtual void Reset() override; 46 | 47 | virtual void ReportError(ERuntimeArchiverErrorCode ErrorCode, const FString& ErrorString) const override; 48 | //~ End URuntimeArchiverBase Interface 49 | 50 | private: 51 | /** Tar archiver used for internal operations */ 52 | TStrongObjectPtr TarArchiver; 53 | 54 | /** Stream containing LZ4 compressed data */ 55 | TUniquePtr CompressedStream; 56 | 57 | /** Last saved compression level */ 58 | ERuntimeArchiverCompressionLevel LastCompressionLevel; 59 | }; 60 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Public/ArchiverLZ4/RuntimeArchiverLZ4.h: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "RuntimeArchiverBase.h" 7 | #include "ArchiverTar/RuntimeArchiverTar.h" 8 | #include "UObject/StrongObjectPtr.h" 9 | #include "RuntimeArchiverLZ4.generated.h" 10 | 11 | /** 12 | * LZ4 archiver class. Works with tar.lz4 (tlz4) archives 13 | * Archiving of data occurs through the Tar archiver and their subsequent compression through LZ4 raw archiver (the same applies for unarchiving) 14 | * Note that it currently has some bugs when unarchiving data from a tar archive 15 | */ 16 | UCLASS(BlueprintType, Category = "Runtime Archiver") 17 | class RUNTIMEARCHIVER_API URuntimeArchiverLZ4 : public URuntimeArchiverBase 18 | { 19 | GENERATED_BODY() 20 | 21 | public: 22 | URuntimeArchiverLZ4(); 23 | 24 | //~ Begin URuntimeArchiverBase Interface 25 | virtual bool CreateArchiveInStorage(FString ArchivePath) override; 26 | virtual bool CreateArchiveInMemory(int32 InitialAllocationSize = 0) override; 27 | 28 | virtual bool OpenArchiveFromStorage(FString ArchivePath) override; 29 | virtual bool OpenArchiveFromMemory(const TArray64& ArchiveData) override; 30 | 31 | virtual bool CloseArchive() override; 32 | 33 | virtual bool GetArchiveData(TArray64& ArchiveData) override; 34 | 35 | virtual bool GetArchiveEntries(int32& NumOfArchiveEntries) override; 36 | 37 | virtual bool GetArchiveEntryInfoByName(FString EntryName, FRuntimeArchiveEntry& EntryInfo) override; 38 | virtual bool GetArchiveEntryInfoByIndex(int32 EntryIndex, FRuntimeArchiveEntry& EntryInfo) override; 39 | 40 | virtual bool AddEntryFromMemory(FString EntryName, const TArray64& DataToBeArchived, ERuntimeArchiverCompressionLevel CompressionLevel) override; 41 | 42 | virtual bool ExtractEntryToMemory(const FRuntimeArchiveEntry& EntryInfo, TArray64& UnarchivedData) override; 43 | 44 | virtual bool Initialize() override; 45 | virtual bool IsInitialized() const override; 46 | virtual void Reset() override; 47 | 48 | virtual void ReportError(ERuntimeArchiverErrorCode ErrorCode, const FString& ErrorString) const override; 49 | //~ End URuntimeArchiverBase Interface 50 | 51 | private: 52 | /** Tar archiver used for internal operations */ 53 | TStrongObjectPtr TarArchiver; 54 | 55 | /** Stream containing LZ4 compressed data */ 56 | TUniquePtr CompressedStream; 57 | 58 | /** Last saved compression level */ 59 | ERuntimeArchiverCompressionLevel LastCompressionLevel; 60 | }; 61 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Public/ArchiverOodle/RuntimeArchiverOodle.h: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "RuntimeArchiverBase.h" 7 | #include "ArchiverTar/RuntimeArchiverTar.h" 8 | #include "UObject/StrongObjectPtr.h" 9 | #include "RuntimeArchiverOodle.generated.h" 10 | 11 | /** 12 | * Oodle archiver class. Works with tar.ood (tood) archives. This doesn't have any specs, but works similarly to tar.gz 13 | * Archiving of data occurs through the Tar archiver and their subsequent compression through Oodle raw archiver (the same applies for unarchiving) 14 | */ 15 | UCLASS(BlueprintType, Category = "Runtime Archiver") 16 | class RUNTIMEARCHIVER_API URuntimeArchiverOodle : public URuntimeArchiverBase 17 | { 18 | GENERATED_BODY() 19 | 20 | public: 21 | URuntimeArchiverOodle(); 22 | 23 | //~ Begin URuntimeArchiverBase Interface 24 | virtual bool CreateArchiveInStorage(FString ArchivePath) override; 25 | virtual bool CreateArchiveInMemory(int32 InitialAllocationSize = 0) override; 26 | 27 | virtual bool OpenArchiveFromStorage(FString ArchivePath) override; 28 | virtual bool OpenArchiveFromMemory(const TArray64& ArchiveData) override; 29 | 30 | virtual bool CloseArchive() override; 31 | 32 | virtual bool GetArchiveData(TArray64& ArchiveData) override; 33 | 34 | virtual bool GetArchiveEntries(int32& NumOfArchiveEntries) override; 35 | 36 | virtual bool GetArchiveEntryInfoByName(FString EntryName, FRuntimeArchiveEntry& EntryInfo) override; 37 | virtual bool GetArchiveEntryInfoByIndex(int32 EntryIndex, FRuntimeArchiveEntry& EntryInfo) override; 38 | 39 | virtual bool AddEntryFromMemory(FString EntryName, const TArray64& DataToBeArchived, ERuntimeArchiverCompressionLevel CompressionLevel) override; 40 | 41 | virtual bool ExtractEntryToMemory(const FRuntimeArchiveEntry& EntryInfo, TArray64& UnarchivedData) override; 42 | 43 | virtual bool Initialize() override; 44 | virtual bool IsInitialized() const override; 45 | virtual void Reset() override; 46 | 47 | virtual void ReportError(ERuntimeArchiverErrorCode ErrorCode, const FString& ErrorString) const override; 48 | //~ End URuntimeArchiverBase Interface 49 | 50 | private: 51 | /** Tar archiver used for internal operations */ 52 | TStrongObjectPtr TarArchiver; 53 | 54 | /** Stream containing LZ4 compressed data */ 55 | TUniquePtr CompressedStream; 56 | 57 | /** Last saved compression level */ 58 | ERuntimeArchiverCompressionLevel LastCompressionLevel; 59 | }; 60 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Public/ArchiverRaw/RuntimeArchiverRaw.h: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "RuntimeArchiverTypes.h" 7 | #include "UObject/Object.h" 8 | #include "RuntimeArchiverRaw.generated.h" 9 | 10 | DECLARE_DELEGATE_OneParam(FRuntimeArchiverRawMemoryResultNative, const TArray64&); 11 | DECLARE_DYNAMIC_DELEGATE_OneParam(FRuntimeArchiverRawMemoryResult, const TArray&, Data); 12 | 13 | /** 14 | * Raw archiver class. Works with various archives, especially those specific to the engine, such as Oodle, LZ4, GZip, etc. 15 | */ 16 | UCLASS() 17 | class RUNTIMEARCHIVER_API URuntimeArchiverRaw : public UObject 18 | { 19 | GENERATED_BODY() 20 | 21 | public: 22 | /** 23 | * Asynchronously compress raw data 24 | * 25 | * @param RawFormat Raw format 26 | * @param CompressionLevel Compression level. The higher the level, the more compression 27 | * @param UncompressedData Uncompressed data 28 | * @param OnResult Delegate broadcasting the result 29 | * @return Whether the compression was successful or not 30 | */ 31 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver|Raw") 32 | static void CompressRawDataAsync(ERuntimeArchiverRawFormat RawFormat, ERuntimeArchiverCompressionLevel CompressionLevel, TArray UncompressedData, const FRuntimeArchiverRawMemoryResult& OnResult); 33 | 34 | /** 35 | * Asynchronously compress raw data. Prefer to use this function if possible 36 | * 37 | * @param RawFormat Raw format 38 | * @param CompressionLevel Compression level. The higher the level, the more compression 39 | * @param UncompressedData Uncompressed data 40 | * @param OnResult Delegate broadcasting the result 41 | * @return Whether the compression was successful or not 42 | */ 43 | static void CompressRawDataAsync(ERuntimeArchiverRawFormat RawFormat, ERuntimeArchiverCompressionLevel CompressionLevel, TArray64 UncompressedData, const FRuntimeArchiverRawMemoryResultNative& OnResult); 44 | 45 | /** 46 | * Compress raw data 47 | * 48 | * @param RawFormat Raw format 49 | * @param CompressionLevel Compression level. The higher the level, the more compression 50 | * @param UncompressedData Uncompressed data 51 | * @param CompressedData Out compressed data 52 | * @return Whether the compression was successful or not 53 | */ 54 | static bool CompressRawData(ERuntimeArchiverRawFormat RawFormat, ERuntimeArchiverCompressionLevel CompressionLevel, const TArray64& UncompressedData, TArray64& CompressedData); 55 | 56 | /** 57 | * Asynchronously uncompress raw data 58 | * 59 | * @param RawFormat Raw format 60 | * @param CompressedData Compressed data 61 | * @param OnResult Delegate broadcasting the result 62 | * @return Whether the compression was successful or not 63 | */ 64 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver|Raw") 65 | static void UncompressRawDataAsync(ERuntimeArchiverRawFormat RawFormat, TArray CompressedData, const FRuntimeArchiverRawMemoryResult& OnResult); 66 | 67 | /** 68 | * Asynchronously uncompress raw data. Prefer to use this function if possible 69 | * 70 | * @param RawFormat Raw format 71 | * @param CompressedData Compressed data 72 | * @param OnResult Delegate broadcasting the result 73 | * @return Whether the compression was successful or not 74 | */ 75 | static void UncompressRawDataAsync(ERuntimeArchiverRawFormat RawFormat, TArray64 CompressedData, const FRuntimeArchiverRawMemoryResultNative& OnResult); 76 | 77 | /** 78 | * Uncompress raw data 79 | * 80 | * @param RawFormat Raw format 81 | * @param CompressedData Compressed data 82 | * @param UncompressedData Out uncompressed data 83 | * @return Whether the compression was successful or not 84 | */ 85 | static bool UncompressRawData(ERuntimeArchiverRawFormat RawFormat, TArray64 CompressedData, TArray64& UncompressedData); 86 | 87 | /** 88 | * Guess the size of the compressed raw data given the format. Returns the maximum possible size 89 | * 90 | * @param RawFormat Raw format 91 | * @param UncompressedData Uncompressed data 92 | * @return Guessed size 93 | */ 94 | static int64 GuessCompressedSize(ERuntimeArchiverRawFormat RawFormat, const TArray64& UncompressedData); 95 | 96 | /** 97 | * Guess the size of the uncompressed raw data given the format. Returns the maximum possible size 98 | * 99 | * @param RawFormat Raw format 100 | * @param CompressedData Compressed data 101 | * @return Guessed size 102 | */ 103 | static int64 GuessUncompressedSize(ERuntimeArchiverRawFormat RawFormat, const TArray64& CompressedData); 104 | }; 105 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Public/ArchiverTar/RuntimeArchiverTar.h: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "RuntimeArchiverBase.h" 7 | #include "Misc/EngineVersionComparison.h" 8 | #include "Streams/RuntimeArchiverBaseStream.h" 9 | #include "RuntimeArchiverTar.generated.h" 10 | 11 | struct FTarHeader; 12 | class FRuntimeArchiverTarEncapsulator; 13 | 14 | /** 15 | * Tar archiver class. Works with tar archives. Inspired by Microtar 16 | */ 17 | UCLASS(BlueprintType, Category = "Runtime Archiver") 18 | class RUNTIMEARCHIVER_API URuntimeArchiverTar : public URuntimeArchiverBase 19 | { 20 | GENERATED_BODY() 21 | 22 | public: 23 | //~ Begin URuntimeArchiverBase Interface 24 | virtual bool CreateArchiveInStorage(FString ArchivePath) override; 25 | virtual bool CreateArchiveInMemory(int32 InitialAllocationSize = 0) override; 26 | 27 | virtual bool OpenArchiveFromStorage(FString ArchivePath) override; 28 | virtual bool OpenArchiveFromMemory(const TArray64& ArchiveData) override; 29 | 30 | virtual bool CloseArchive() override; 31 | 32 | virtual bool GetArchiveData(TArray64& ArchiveData) override; 33 | 34 | virtual bool GetArchiveEntries(int32& NumOfArchiveEntries) override; 35 | 36 | virtual bool GetArchiveEntryInfoByName(FString EntryName, FRuntimeArchiveEntry& EntryInfo) override; 37 | virtual bool GetArchiveEntryInfoByIndex(int32 EntryIndex, FRuntimeArchiveEntry& EntryInfo) override; 38 | 39 | virtual bool AddEntryFromMemory(FString EntryName, const TArray64& DataToBeArchived, ERuntimeArchiverCompressionLevel CompressionLevel) override; 40 | 41 | virtual bool ExtractEntryToMemory(const FRuntimeArchiveEntry& EntryInfo, TArray64& UnarchivedData) override; 42 | 43 | virtual bool Initialize() override; 44 | virtual bool IsInitialized() const override; 45 | virtual void Reset() override; 46 | 47 | virtual void ReportError(ERuntimeArchiverErrorCode ErrorCode, const FString& ErrorString) const override; 48 | //~ End URuntimeArchiverBase Interface 49 | 50 | private: 51 | /** Tar encapsulator */ 52 | TUniquePtr TarEncapsulator; 53 | }; 54 | 55 | /** 56 | * Encapsulator between archiver and stream that implements intermediate operations 57 | */ 58 | class FRuntimeArchiverTarEncapsulator 59 | { 60 | public: 61 | FRuntimeArchiverTarEncapsulator(); 62 | virtual ~FRuntimeArchiverTarEncapsulator(); 63 | 64 | /** 65 | * Check if the archive is successfully opened or not 66 | * 67 | * @return Whether the archive is successfully opened or not 68 | */ 69 | bool IsValid(); 70 | 71 | /** 72 | * Test if tar archive contains valid data or not. Only the first header is checked for performance 73 | * 74 | * @return Whether the archive contains valid data or not 75 | */ 76 | bool TestArchive(); 77 | 78 | /** 79 | * Open a tar archive from a file as a stream for reading or writing 80 | * 81 | * @param ArchivePath Path to archive to open 82 | * @param bWrite Whether to open for writing or for reading 83 | * @return Whether the archive was successfully opened or not 84 | */ 85 | bool OpenFile(const FString& ArchivePath, bool bWrite); 86 | 87 | /** 88 | * Open a tar archive from memory as a stream for reading or writing 89 | * 90 | * @param ArchiveData Tar archive data. Must be empty if read-only mode is used 91 | * @param InitialAllocationSize Estimated archive size if known. Avoids unnecessary memory allocation. Must be empty if read-only mode is used 92 | * @param bWrite Whether to open for writing or for reading 93 | * @return Whether the archive was successfully opened or not 94 | */ 95 | bool OpenMemory(const TArray64& ArchiveData, int32 InitialAllocationSize, bool bWrite); 96 | 97 | /** 98 | * Find header from the tar archive. Optionally updates the reading position of the found header. Works similar to the std::find_if algorithm 99 | * 100 | * @param ComparePredicate Compare predicate. Should return true when header is found, false otherwise. The first parameter is the header that can be the target, the second is its index in the archive 101 | * @param Header Found header 102 | * @param Index Found entry index 103 | * @param bRemainPosition Whether to keep the previous read/write position, or update 104 | * @return Whether the header was found or not 105 | */ 106 | bool FindIf(TFunctionRef ComparePredicate, FTarHeader& Header, int32& Index, bool bRemainPosition); 107 | 108 | /** 109 | * Get the number of tar archive entries 110 | * 111 | * @param NumOfArchiveEntries The number of entries in the archive 112 | * @return Whether the operation was successful or not 113 | */ 114 | bool GetArchiveEntries(int32& NumOfArchiveEntries); 115 | 116 | /** 117 | * Get tar archive data 118 | * 119 | * @param ArchiveData Binary archive data 120 | * @return Whether the operation was successful or not 121 | */ 122 | bool GetArchiveData(TArray64& ArchiveData); 123 | 124 | /** 125 | * Reset read/write position 126 | * 127 | * @return Whether the operation was successful or not 128 | */ 129 | bool Rewind(); 130 | 131 | /** 132 | * Seek to the next header 133 | * 134 | * @return Whether the operation was successful or not 135 | */ 136 | bool Next(); 137 | 138 | /** 139 | * Read the header from the current position 140 | * 141 | * @param Header Read header 142 | * @return Whether the operation was successful or not 143 | */ 144 | bool ReadHeader(FTarHeader& Header); 145 | 146 | /** 147 | * Read the archived data from the current position 148 | * 149 | * @param UnarchivedData Unarchived data 150 | * @return Whether the operation was successful or not 151 | */ 152 | bool ReadData(TArray64& UnarchivedData); 153 | 154 | /** 155 | * Write the header from the current position 156 | * 157 | * @param Header Header to write 158 | * @return Whether the operation was successful or not 159 | */ 160 | bool WriteHeader(const FTarHeader& Header); 161 | 162 | /** 163 | * Write the archived data from the current position 164 | * 165 | * @param DataToBeArchived Binary data to be archived. Leave it blank if it is a directory 166 | * @return Whether the operation was successful or not 167 | */ 168 | bool WriteData(const TArray64& DataToBeArchived); 169 | 170 | /** 171 | * Write null bytes represented as null character '\0' 172 | * 173 | * @param NumOfBytes Number of null bytes to write 174 | * @return Whether the operation was successful or not 175 | */ 176 | bool WriteNullBytes(int64 NumOfBytes) const; 177 | 178 | /** 179 | * Write additional null bytes at the end to finalize the archive data 180 | * 181 | * @return Whether the operation was successful or not 182 | */ 183 | bool Finalize(); 184 | 185 | private: 186 | /** Used stream */ 187 | TUniquePtr Stream; 188 | 189 | /** Remaining read or write data size */ 190 | int64 RemainingDataSize; 191 | 192 | /** Last header position */ 193 | int64 LastHeaderPosition; 194 | 195 | /** Cached number of headers for a small optimization */ 196 | int32 CachedNumOfHeaders; 197 | 198 | /** Whether the tar archive was finalized or not */ 199 | bool bIsFinalized; 200 | }; 201 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Public/ArchiverZip/RuntimeArchiverZip.h: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "RuntimeArchiverBase.h" 7 | #include "RuntimeArchiverZip.generated.h" 8 | 9 | /** 10 | * Zip archiver class. Works with zip archives 11 | */ 12 | UCLASS(BlueprintType, Category = "Runtime Archiver") 13 | class RUNTIMEARCHIVER_API URuntimeArchiverZip : public URuntimeArchiverBase 14 | { 15 | GENERATED_BODY() 16 | 17 | public: 18 | /** Default constructor */ 19 | URuntimeArchiverZip(); 20 | 21 | //~ Begin URuntimeArchiverBase Interface 22 | virtual bool CreateArchiveInStorage(FString ArchivePath) override; 23 | virtual bool CreateArchiveInMemory(int32 InitialAllocationSize = 0) override; 24 | 25 | virtual bool OpenArchiveFromStorage(FString ArchivePath) override; 26 | virtual bool OpenArchiveFromMemory(const TArray64& ArchiveData) override; 27 | 28 | virtual bool CloseArchive() override; 29 | 30 | virtual bool GetArchiveData(TArray64& ArchiveData) override; 31 | 32 | virtual bool GetArchiveEntries(int32& NumOfArchiveEntries) override; 33 | 34 | virtual bool GetArchiveEntryInfoByName(FString EntryName, FRuntimeArchiveEntry& EntryInfo) override; 35 | virtual bool GetArchiveEntryInfoByIndex(int32 EntryIndex, FRuntimeArchiveEntry& EntryInfo) override; 36 | 37 | virtual bool AddEntryFromMemory(FString EntryName, const TArray64& DataToBeArchived, ERuntimeArchiverCompressionLevel CompressionLevel) override; 38 | 39 | virtual bool ExtractEntryToMemory(const FRuntimeArchiveEntry& EntryInfo, TArray64& UnarchivedData) override; 40 | 41 | virtual bool Initialize() override; 42 | virtual bool IsInitialized() const override; 43 | virtual void Reset() override; 44 | 45 | virtual void ReportError(ERuntimeArchiverErrorCode ErrorCode, const FString& ErrorString) const override; 46 | //~ End URuntimeArchiverBase Interface 47 | 48 | public: 49 | /** 50 | * Open an archive from storage to append 51 | * 52 | * @param ArchivePath Path to open an archive 53 | * @return Whether the operation was successful or not 54 | */ 55 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver|Open") 56 | bool OpenArchiveFromStorageToAppend(FString ArchivePath); 57 | 58 | private: 59 | /** Whether to use append mode or not */ 60 | bool bAppendMode; 61 | 62 | /** Miniz archiver */ 63 | void* MinizArchiver; 64 | }; 65 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Public/AsyncTasks/RuntimeArchiverArchiveAsyncTask.h: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "RuntimeArchiverBase.h" 7 | #include "Kismet/BlueprintAsyncActionBase.h" 8 | #include "Misc/EngineVersionComparison.h" 9 | #include "RuntimeArchiverArchiveAsyncTask.generated.h" 10 | 11 | /** 12 | * Async task which simplifies archiving from an archive 13 | */ 14 | UCLASS() 15 | class URuntimeArchiverArchiveAsyncTask : public UBlueprintAsyncActionBase 16 | { 17 | GENERATED_BODY() 18 | 19 | public: 20 | /** 21 | * Asynchronously archive entries from a directory 22 | * 23 | * @param ArchiverClass Archiver class 24 | * @param ArchivePath Path to open an archive 25 | * @param DirectoryPath Directory to be archived 26 | * @param bAddParentDirectory Whether to add the specified directory as a parent 27 | * @param CompressionLevel Compression level. The higher the level, the more compression 28 | */ 29 | UFUNCTION(BlueprintCallable, meta = (BlueprintInternalUseOnly = "true"), Category = "Runtime Archiver|Async") 30 | static URuntimeArchiverArchiveAsyncTask* ArchiveDirectory(TSubclassOf ArchiverClass, FString ArchivePath, FString DirectoryPath, bool bAddParentDirectory, ERuntimeArchiverCompressionLevel CompressionLevel = ERuntimeArchiverCompressionLevel::Compression6); 31 | 32 | /** 33 | * Asynchronously archive entries from file paths 34 | * 35 | * @param ArchiverClass Archiver class 36 | * @param ArchivePath Path to open an archive 37 | * @param FilePaths File paths to be archived 38 | * @param CompressionLevel Compression level. The higher the level, the more compression 39 | */ 40 | UFUNCTION(BlueprintCallable, meta = (BlueprintInternalUseOnly = "true"), Category = "Runtime Archiver|Async") 41 | static URuntimeArchiverArchiveAsyncTask* ArchiveFiles(TSubclassOf ArchiverClass, FString ArchivePath, TArray FilePaths, ERuntimeArchiverCompressionLevel CompressionLevel = ERuntimeArchiverCompressionLevel::Compression6); 42 | 43 | /** Archiving completed successfully */ 44 | UPROPERTY(BlueprintAssignable) 45 | FRuntimeArchiverAsyncActionResult OnSuccess; 46 | 47 | /** Archiving in progress */ 48 | UPROPERTY(BlueprintAssignable) 49 | FRuntimeArchiverAsyncActionResult OnProgress; 50 | 51 | /** Archiving failed */ 52 | UPROPERTY(BlueprintAssignable) 53 | FRuntimeArchiverAsyncActionResult OnFail; 54 | 55 | protected: 56 | //~ Begin UBlueprintAsyncActionBase Interface 57 | virtual void Activate() override; 58 | //~ End UBlueprintAsyncActionBase Interface 59 | 60 | private: 61 | /** Information about the operation to archive the directory */ 62 | struct 63 | { 64 | FString ArchivePath; 65 | FString DirectoryPath; 66 | bool bAddParentDirectory; 67 | ERuntimeArchiverCompressionLevel CompressionLevel; 68 | } DirectoryInfo; 69 | 70 | /** Information about the operation to archive files */ 71 | struct 72 | { 73 | FString ArchivePath; 74 | TArray FilePaths; 75 | ERuntimeArchiverCompressionLevel CompressionLevel; 76 | } FilesInfo; 77 | 78 | /** Specific archiving operation */ 79 | enum class EOperationType : uint8 80 | { 81 | Directory, 82 | Files 83 | } OperationType; 84 | 85 | /** Used archiver */ 86 | UPROPERTY() 87 | #if UE_VERSION_NEWER_THAN(5, 0, 0) 88 | TObjectPtr Archiver; 89 | #else 90 | URuntimeArchiverBase* Archiver; 91 | #endif 92 | 93 | /** Operation result delegate */ 94 | FRuntimeArchiverAsyncOperationResult OperationResult; 95 | 96 | /** Operation in progress delegate */ 97 | FRuntimeArchiverAsyncOperationProgress OperationProgress; 98 | 99 | /** Start archiving directory operation */ 100 | void StartDirectory(); 101 | 102 | /** Start archiving files operation */ 103 | void StartFiles(); 104 | 105 | /** 106 | * Execute the result of the operation 107 | * 108 | * @param bSuccess Whether the result is successful or not 109 | */ 110 | UFUNCTION() 111 | void OnResult_Callback(bool bSuccess); 112 | 113 | /** 114 | * Execute the progress of the operation 115 | * 116 | * @param Percentage Current operation percentage 117 | */ 118 | UFUNCTION() 119 | void OnProgress_Callback(int32 Percentage); 120 | }; 121 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Public/AsyncTasks/RuntimeArchiverUnarchiveAsyncTask.h: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "RuntimeArchiverBase.h" 7 | #include "Kismet/BlueprintAsyncActionBase.h" 8 | #include "Misc/EngineVersionComparison.h" 9 | #include "RuntimeArchiverUnarchiveAsyncTask.generated.h" 10 | 11 | /** 12 | * Async task which simplifies unarchiving from an archive 13 | */ 14 | UCLASS() 15 | class RUNTIMEARCHIVER_API URuntimeArchiverUnarchiveAsyncTask : public UBlueprintAsyncActionBase 16 | { 17 | GENERATED_BODY() 18 | 19 | public: 20 | /** 21 | * Asynchronously unarchive entries in a directory to storage 22 | * 23 | * @param ArchiverClass Archiver class 24 | * @param ArchivePath Path to open an archive 25 | * @param EntryName Path to the entry to extract to. Must be a directory only. Leave the field empty to use all entries 26 | * @param DirectoryPath Path to the directory for exporting entries 27 | * @param bAddParentDirectory Whether to add the specified directory as a parent 28 | * @param bForceOverwrite Whether to force a file to be overwritten if it exists or not 29 | */ 30 | UFUNCTION(BlueprintCallable, meta = (BlueprintInternalUseOnly = "true"), Category = "Runtime Archiver|Async") 31 | static URuntimeArchiverUnarchiveAsyncTask* UnarchiveDirectory(TSubclassOf ArchiverClass, FString ArchivePath, FString EntryName, FString DirectoryPath, bool bAddParentDirectory, bool bForceOverwrite = true); 32 | 33 | /** 34 | * Asynchronously unarchive entries to storage 35 | * 36 | * @param ArchiverClass Archiver class 37 | * @param ArchivePath Path to open an archive 38 | * @param EntryNames Array of all entry names to extract 39 | * @param DirectoryPath Path to the directory for exporting entries 40 | * @param bForceOverwrite Whether to force a file to be overwritten if it exists or not 41 | */ 42 | UFUNCTION(BlueprintCallable, meta = (BlueprintInternalUseOnly = "true"), Category = "Runtime Archiver|Async") 43 | static URuntimeArchiverUnarchiveAsyncTask* UnarchiveFiles(TSubclassOf ArchiverClass, FString ArchivePath, TArray EntryNames, FString DirectoryPath, bool bForceOverwrite = true); 44 | 45 | /** Unarchiving completed successfully */ 46 | UPROPERTY(BlueprintAssignable) 47 | FRuntimeArchiverAsyncActionResult OnSuccess; 48 | 49 | /** Archiving in progress */ 50 | UPROPERTY(BlueprintAssignable) 51 | FRuntimeArchiverAsyncActionResult OnProgress; 52 | 53 | /** Unarchiving failed */ 54 | UPROPERTY(BlueprintAssignable) 55 | FRuntimeArchiverAsyncActionResult OnFail; 56 | 57 | protected: 58 | //~ Begin UBlueprintAsyncActionBase Interface 59 | virtual void Activate() override; 60 | //~ End UBlueprintAsyncActionBase Interface 61 | 62 | private: 63 | /** Information about the operation to unarchive the directory */ 64 | struct 65 | { 66 | FString ArchivePath; 67 | FString EntryName; 68 | FString DirectoryPath; 69 | bool bAddParentDirectory; 70 | bool bForceOverwrite; 71 | } DirectoryInfo; 72 | 73 | /** Information about the operation to unarchive files */ 74 | struct 75 | { 76 | FString ArchivePath; 77 | TArray EntryNames; 78 | FString DirectoryPath; 79 | bool bForceOverwrite; 80 | } FilesInfo; 81 | 82 | /** Specific unarchiving operation */ 83 | enum class EOperationType : uint8 84 | { 85 | Directory, 86 | Files 87 | } OperationType; 88 | 89 | /** Used archiver */ 90 | UPROPERTY() 91 | #if UE_VERSION_NEWER_THAN(5, 0, 0) 92 | TObjectPtr Archiver; 93 | #else 94 | URuntimeArchiverBase* Archiver; 95 | #endif 96 | 97 | /** Operation result delegate */ 98 | FRuntimeArchiverAsyncOperationResult OperationResult; 99 | 100 | /** Operation in progress delegate */ 101 | FRuntimeArchiverAsyncOperationProgress OperationProgress; 102 | 103 | /** Start unarchiving directory operation */ 104 | void StartDirectory(); 105 | 106 | /** Start unarchiving files operation */ 107 | void StartFiles(); 108 | 109 | /** 110 | * Execute the result of the operation 111 | * 112 | * @param bSuccess Whether the result is successful or not 113 | */ 114 | UFUNCTION() 115 | void OnResult_Callback(bool bSuccess); 116 | 117 | /** 118 | * Execute the progress of the operation 119 | * 120 | * @param Percentage Current operation percentage 121 | */ 122 | UFUNCTION() 123 | void OnProgress_Callback(int32 Percentage); 124 | }; 125 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Public/RuntimeArchiver.h: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "Modules/ModuleManager.h" 7 | 8 | class FRuntimeArchiverModule : public IModuleInterface 9 | { 10 | public: 11 | 12 | /** IModuleInterface implementation */ 13 | virtual void StartupModule() override; 14 | virtual void ShutdownModule() override; 15 | }; 16 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Public/RuntimeArchiverBase.h: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "RuntimeArchiverTypes.h" 7 | #include "UObject/Object.h" 8 | #include "Templates/SubclassOf.h" 9 | #include "RuntimeArchiverBase.generated.h" 10 | 11 | /** 12 | * The base class for the archiver. Do not create it manually! 13 | */ 14 | UCLASS(Abstract, HideDropdown, BlueprintType, Category = "Runtime Archiver") 15 | class RUNTIMEARCHIVER_API URuntimeArchiverBase : public UObject 16 | { 17 | GENERATED_BODY() 18 | 19 | public: 20 | /** 21 | * Default constructor 22 | */ 23 | URuntimeArchiverBase(); 24 | 25 | /** 26 | * Create an archiver to archive/unarchive files and directories 27 | */ 28 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver", meta = (WorldContext="WorldContextObject", DeterminesOutputType = "ArchiverClass")) 29 | static URuntimeArchiverBase* CreateRuntimeArchiver(UObject* WorldContextObject, UPARAM(meta = (AllowAbstract = "false")) TSubclassOf ArchiverClass); 30 | 31 | //~ Begin UObject Interface 32 | virtual void BeginDestroy() override; 33 | //~ End UObject Interface 34 | 35 | public: 36 | /** 37 | * Create an archive in the specified path. The file will be created after calling the "CloseArchive" function 38 | * 39 | * @param ArchivePath Path to create an archive 40 | * @return Whether the operation was successful or not 41 | */ 42 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver|Create") 43 | virtual bool CreateArchiveInStorage(FString ArchivePath); 44 | 45 | /** 46 | * Create an archive in memory 47 | * 48 | * @param InitialAllocationSize Estimated archive size if known. Avoids unnecessary memory allocation 49 | * @return Whether the operation was successful or not 50 | * @note Call GetArchiveDataFromMemory to get the archive data 51 | */ 52 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver|Create") 53 | virtual bool CreateArchiveInMemory(int32 InitialAllocationSize = 0); 54 | 55 | /** 56 | * Open an archive from storage 57 | * 58 | * @param ArchivePath Path to open an archive 59 | * @return Whether the operation was successful or not 60 | */ 61 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver|Open") 62 | virtual bool OpenArchiveFromStorage(FString ArchivePath); 63 | 64 | /** 65 | * Open an archive from memory 66 | * 67 | * @param ArchiveData Binary archive data 68 | * @return Whether the operation was successful or not 69 | */ 70 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver|Open") 71 | bool OpenArchiveFromMemory(TArray ArchiveData); 72 | 73 | /** 74 | * Open an archive from memory. Prefer to use this function if possible 75 | * 76 | * @param ArchiveData Binary archive data 77 | * @return Whether the operation was successful or not 78 | */ 79 | virtual bool OpenArchiveFromMemory(const TArray64& ArchiveData); 80 | 81 | /** 82 | * Close previously created/opened archive 83 | * 84 | * @return Whether the operation was successful or not 85 | */ 86 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver|Close") 87 | virtual bool CloseArchive(); 88 | 89 | /** 90 | * Get archive data created in memory 91 | * 92 | * @param ArchiveData Binary archive data 93 | * @return Whether the operation was successful or not 94 | */ 95 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver|Get") 96 | bool GetArchiveData(TArray& ArchiveData); 97 | 98 | /** 99 | * Get archive data created in memory. Prefer to use this function if possible 100 | * 101 | * @param ArchiveData Binary archive data 102 | * @return Whether the operation was successful or not 103 | */ 104 | virtual bool GetArchiveData(TArray64& ArchiveData); 105 | 106 | /** 107 | * Get the number of the archive entries 108 | * 109 | * @param NumOfArchiveEntries The number of entries in the archive 110 | * @return Whether the operation was successful or not 111 | */ 112 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver|Get") 113 | virtual bool GetArchiveEntries(int32& NumOfArchiveEntries); 114 | 115 | /** 116 | * Get information about the archive entry by name 117 | * 118 | * @param EntryName Archive entry name 119 | * @param EntryInfo Retrieved information about the entry 120 | * @return Whether the operation was successful or not 121 | */ 122 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver|Get") 123 | virtual bool GetArchiveEntryInfoByName(FString EntryName, FRuntimeArchiveEntry& EntryInfo); 124 | 125 | /** 126 | * Get information about the archive entry by index 127 | * 128 | * @param EntryIndex Archive entry index. Must be >= 0 129 | * @param EntryInfo Retrieved information about the entry 130 | * @return Whether the operation was successful or not 131 | */ 132 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver|Get") 133 | virtual bool GetArchiveEntryInfoByIndex(int32 EntryIndex, FRuntimeArchiveEntry& EntryInfo); 134 | 135 | /** 136 | * Add entry from storage. In other words, import the file into the archive 137 | * 138 | * @param EntryName Entry name. In other words, the name of the file in the archive 139 | * @param FilePath Path to the file to be archived 140 | * @param CompressionLevel Compression level. The higher the level, the more compression 141 | * @return Whether the operation was successful or not 142 | */ 143 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver|Add") 144 | bool AddEntryFromStorage(FString EntryName, FString FilePath, ERuntimeArchiverCompressionLevel CompressionLevel = ERuntimeArchiverCompressionLevel::Compression6); 145 | 146 | /** 147 | * Add entries from storage. In other words, import files into the archive 148 | * 149 | * @param OnResult Delegate broadcasting the result 150 | * @param OnProgress Delegate broadcasting the progress 151 | * @param FilePaths File paths to be archived 152 | * @param CompressionLevel Compression level. The higher the level, the more compression 153 | */ 154 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver|Add") 155 | void AddEntriesFromStorage(const FRuntimeArchiverAsyncOperationResult& OnResult, const FRuntimeArchiverAsyncOperationProgress& OnProgress, TArray FilePaths, ERuntimeArchiverCompressionLevel CompressionLevel = ERuntimeArchiverCompressionLevel::Compression6); 156 | 157 | /** 158 | * Add entries from storage. Must be used for directories only 159 | * 160 | * @param OnResult Delegate broadcasting the result 161 | * @param DirectoryPath Directory to be archived 162 | * @param bAddParentDirectory Whether to add the specified directory as a parent 163 | * @param CompressionLevel Compression level. The higher the level, the more compression 164 | */ 165 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver|Add") 166 | void AddEntriesFromStorage_Directory(const FRuntimeArchiverAsyncOperationResult& OnResult, FString DirectoryPath, bool bAddParentDirectory, ERuntimeArchiverCompressionLevel CompressionLevel = ERuntimeArchiverCompressionLevel::Compression6); 167 | 168 | private: 169 | /** 170 | * Internal function to recursively add entries from storage 171 | * 172 | * @param BaseDirectoryPathToExclude The base directory path to be used to exclude from the absolute file path 173 | * @param DirectoryPath Directory to scan 174 | * @param CompressionLevel Compression level. The higher the level, the more compression 175 | * @return Whether the operation was successful or not 176 | */ 177 | bool AddEntriesFromStorage_Directory_Internal(FString BaseDirectoryPathToExclude, FString DirectoryPath, ERuntimeArchiverCompressionLevel CompressionLevel = ERuntimeArchiverCompressionLevel::Compression6); 178 | 179 | public: 180 | /** 181 | * Add entry from memory. In other words, import the data in-memory into the archive 182 | * 183 | * @param EntryName Entry name. In other words, the name of the file in the archive 184 | * @param DataToBeArchived Binary data to be archived 185 | * @param CompressionLevel Compression level. The higher the level, the more compression 186 | * @return Whether the operation was successful or not 187 | */ 188 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver|Add") 189 | bool AddEntryFromMemory(FString EntryName, TArray DataToBeArchived, ERuntimeArchiverCompressionLevel CompressionLevel = ERuntimeArchiverCompressionLevel::Compression6); 190 | 191 | /** 192 | * Add entry from memory. In other words, import the data in-memory into the archive. Prefer to use this function if possible 193 | * 194 | * @param EntryName Entry name. In other words, the name of the file in the archive 195 | * @param DataToBeArchived Binary data to be archived 196 | * @param CompressionLevel Compression level. The higher the level, the more compression 197 | * @return Whether the operation was successful or not 198 | */ 199 | virtual bool AddEntryFromMemory(FString EntryName, const TArray64& DataToBeArchived, ERuntimeArchiverCompressionLevel CompressionLevel = ERuntimeArchiverCompressionLevel::Compression6); 200 | 201 | /** 202 | * Extract entry to storage. In other words, extract the file from the archive to storage 203 | * 204 | * @param EntryInfo Information about the entry 205 | * @param FilePath Path to the file to extract to 206 | * @param bForceOverwrite Whether to force a file to be overwritten if it exists or not 207 | * @return Whether the operation was successful or not 208 | */ 209 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver|Extract") 210 | bool ExtractEntryToStorage(const FRuntimeArchiveEntry& EntryInfo, FString FilePath, bool bForceOverwrite = true); 211 | 212 | /** 213 | * Extract entries to storage. In other words, extract the file from the archive to storage 214 | * 215 | * @param OnResult Delegate broadcasting the result 216 | * @param OnProgress Delegate broadcasting the progress 217 | * @param EntryInfo Array of all entries to extract 218 | * @param DirectoryPath Path to the directory for exporting entries 219 | * @param bForceOverwrite Whether to force a file to be overwritten if it exists or not 220 | */ 221 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver|Extract") 222 | void ExtractEntriesToStorage(const FRuntimeArchiverAsyncOperationResult& OnResult, const FRuntimeArchiverAsyncOperationProgress& OnProgress, TArray EntryInfo, FString DirectoryPath, bool bForceOverwrite = true); 223 | 224 | /** 225 | * Extract entries to storage. Must be used for directories only 226 | * 227 | * @param OnResult Delegate broadcasting the result 228 | * @param EntryName Path to the entry to extract to. Must be a directory only. Leave the field empty to use all entries 229 | * @param DirectoryPath Path to the directory for exporting entries 230 | * @param bAddParentDirectory Whether to add the specified directory as a parent 231 | * @param bForceOverwrite Whether to force a file to be overwritten if it exists or not 232 | */ 233 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver|Extract") 234 | void ExtractEntriesToStorage_Directory(const FRuntimeArchiverAsyncOperationResult& OnResult, FString EntryName, FString DirectoryPath, bool bAddParentDirectory, bool bForceOverwrite = true); 235 | 236 | /** 237 | * Extract entry into memory. In other words, extract the file from the archive into memory 238 | * 239 | * @param EntryInfo Information about the entry 240 | * @param UnarchivedData Unarchived entry data 241 | * @return Whether the operation was successful or not 242 | */ 243 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver|Extract") 244 | bool ExtractEntryToMemory(const FRuntimeArchiveEntry& EntryInfo, TArray& UnarchivedData); 245 | 246 | /** 247 | * Extract entry into memory. In other words, extract the file from the archive into memory. Prefer to use this function if possible 248 | * 249 | * @param EntryInfo Information about the entry 250 | * @param UnarchivedData Unarchived entry data 251 | * @return Whether the operation was successful or not 252 | */ 253 | virtual bool ExtractEntryToMemory(const FRuntimeArchiveEntry& EntryInfo, TArray64& UnarchivedData); 254 | 255 | /** 256 | * Initialize the archiver 257 | */ 258 | virtual bool Initialize(); 259 | 260 | /** 261 | * Check whether the archiver is initialized 262 | */ 263 | virtual bool IsInitialized() const; 264 | 265 | /** 266 | * Reset the archiver. Here it is supposed to clear all the data 267 | */ 268 | virtual void Reset(); 269 | 270 | protected: 271 | /** 272 | * Report an error in the archiver 273 | * 274 | * @param ErrorCode Archiver error code 275 | * @param ErrorString Error details 276 | */ 277 | virtual void ReportError(ERuntimeArchiverErrorCode ErrorCode, const FString& ErrorString) const; 278 | 279 | /** Archive mode */ 280 | ERuntimeArchiverMode Mode; 281 | 282 | /** Archive location */ 283 | ERuntimeArchiverLocation Location; 284 | }; 285 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Public/RuntimeArchiverDefines.h: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | 7 | #include "Logging/LogCategory.h" 8 | #include "Logging/LogMacros.h" 9 | #include "Logging/LogVerbosity.h" 10 | 11 | DECLARE_LOG_CATEGORY_EXTERN(LogRuntimeArchiver, Log, All); 12 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Public/RuntimeArchiverSubsystem.h: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #pragma once 4 | 5 | #include "RuntimeArchiver.h" 6 | #include "Subsystems/EngineSubsystem.h" 7 | #include "RuntimeArchiverTypes.h" 8 | #include "RuntimeArchiverSubsystem.generated.h" 9 | 10 | 11 | /** Delegate broadcast of any archiver errors */ 12 | DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FRuntimeArchiverError, ERuntimeArchiverErrorCode, ErrorCode, const FString&, ErrorString); 13 | 14 | /** 15 | * Archiver subsystem. Used for singleton access of generic stuff 16 | */ 17 | UCLASS() 18 | class RUNTIMEARCHIVER_API URuntimeArchiverSubsystem : public UEngineSubsystem 19 | { 20 | GENERATED_BODY() 21 | 22 | public: 23 | 24 | /** Bind to know when an error occurs while running the archiver */ 25 | UPROPERTY(BlueprintAssignable, Category = "Runtime Archiver Subsystem|Delegates") 26 | FRuntimeArchiverError OnError; 27 | 28 | /** A little helper for getting a subsystem */ 29 | static URuntimeArchiverSubsystem* GetArchiveSubsystem(); 30 | }; 31 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Public/RuntimeArchiverTypes.h: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #pragma once 4 | 5 | #include "RuntimeArchiverTypes.generated.h" 6 | 7 | /** Possible archiver errors */ 8 | UENUM(Blueprintable, Category = "Runtime Archiver") 9 | enum class ERuntimeArchiverErrorCode : uint8 10 | { 11 | NotInitialized, 12 | UnsupportedMode, 13 | UnsupportedLocation, 14 | ExtractError, 15 | AddError, 16 | CloseError, 17 | GetError, 18 | InvalidArgument 19 | }; 20 | 21 | /** Archive entry compression level. The higher the level, the more compression */ 22 | UENUM(Blueprintable, Category = "Runtime Archiver") 23 | enum class ERuntimeArchiverCompressionLevel : uint8 24 | { 25 | Compression0 = 0 UMETA(DisplayName = "0", ToolTip = "No compression"), 26 | Compression1 = 1 UMETA(DisplayName = "1", ToolTip = "Best speed compression"), 27 | Compression2 = 2 UMETA(DisplayName = "2", ToolTip = "Compression 2"), 28 | Compression3 = 3 UMETA(DisplayName = "3", ToolTip = "Compression 3"), 29 | Compression4 = 4 UMETA(DisplayName = "4", ToolTip = "Compression 4"), 30 | Compression5 = 5 UMETA(DisplayName = "5", ToolTip = "Compression 5"), 31 | Compression6 = 6 UMETA(DisplayName = "6", ToolTip = "Compression 6"), 32 | Compression7 = 7 UMETA(DisplayName = "7", ToolTip = "Compression 7"), 33 | Compression8 = 8 UMETA(DisplayName = "8", ToolTip = "Compression 8"), 34 | Compression9 = 9 UMETA(DisplayName = "9", ToolTip = "Compression 9"), 35 | Compression10 = 10 UMETA(DisplayName = "10", ToolTip = "Best compression") 36 | }; 37 | 38 | /** Archive mode */ 39 | UENUM() 40 | enum class ERuntimeArchiverMode : uint8 41 | { 42 | Undefined, 43 | Read, 44 | Write 45 | }; 46 | 47 | /** Archive location */ 48 | UENUM() 49 | enum class ERuntimeArchiverLocation : uint8 50 | { 51 | Undefined, 52 | Storage, 53 | Memory 54 | }; 55 | 56 | /** RAW archive format */ 57 | UENUM() 58 | enum class ERuntimeArchiverRawFormat : uint8 59 | { 60 | Oodle, 61 | GZip, 62 | LZ4 63 | }; 64 | 65 | /** Information about archive entry. Used to search for files/directories in an archive to extract data. Do not fill it in manually */ 66 | USTRUCT(BlueprintType, Category = "Runtime Archiver") 67 | struct FRuntimeArchiveEntry 68 | { 69 | GENERATED_BODY() 70 | 71 | /** Entry index. Used to identify entry */ 72 | UPROPERTY(BlueprintReadOnly, Category = "Runtime Archiver") 73 | int32 Index; 74 | 75 | /** Entry name */ 76 | UPROPERTY(BlueprintReadOnly, Category = "Runtime Archiver") 77 | FString Name; 78 | 79 | /** Whether this entry is a directory (folder) or not */ 80 | UPROPERTY(BlueprintReadOnly, Category = "Runtime Archiver") 81 | bool bIsDirectory; 82 | 83 | /** Uncompressed entry size */ 84 | UPROPERTY(BlueprintReadOnly, Category = "Runtime Archiver") 85 | int64 UncompressedSize; 86 | 87 | /** Compressed entry size */ 88 | UPROPERTY(BlueprintReadOnly, Category = "Runtime Archiver") 89 | int64 CompressedSize; 90 | 91 | /** Entry creation time */ 92 | UPROPERTY(BlueprintReadOnly, Category = "Runtime Archiver") 93 | FDateTime CreationTime; 94 | 95 | /** Default constructor */ 96 | FRuntimeArchiveEntry() 97 | : Index(0) 98 | , Name(FString()) 99 | , bIsDirectory(false) 100 | , UncompressedSize(0) 101 | , CompressedSize(0) 102 | , CreationTime(FDateTime()) 103 | { 104 | } 105 | 106 | /** Custom constructor */ 107 | FRuntimeArchiveEntry(int32 Index) 108 | : Index(Index) 109 | , Name(FString()) 110 | , bIsDirectory(false) 111 | , UncompressedSize(0) 112 | , CompressedSize(0) 113 | , CreationTime(FDateTime()) 114 | { 115 | } 116 | }; 117 | 118 | /** Delegate broadcasting the result of asynchronous archive operations */ 119 | DECLARE_DYNAMIC_DELEGATE_OneParam(FRuntimeArchiverAsyncOperationResult, bool, bSuccess); 120 | 121 | /** Delegate broadcasting the progress of asynchronous archive operations */ 122 | DECLARE_DYNAMIC_DELEGATE_OneParam(FRuntimeArchiverAsyncOperationProgress, int32, Percentage); 123 | 124 | /** Dynamic delegate broadcasting the result of asynchronous archive actions */ 125 | DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FRuntimeArchiverAsyncActionResult, int32, Percentage); 126 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Public/RuntimeArchiverUtilities.h: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #pragma once 4 | 5 | #include "CoreMinimal.h" 6 | #include "RuntimeArchiverTypes.h" 7 | #include "UObject/Object.h" 8 | #include "RuntimeArchiverUtilities.generated.h" 9 | 10 | /** 11 | * Provides general purpose utility functions for archivers 12 | */ 13 | UCLASS() 14 | class RUNTIMEARCHIVER_API URuntimeArchiverUtilities : public UObject 15 | { 16 | GENERATED_BODY() 17 | public: 18 | /** 19 | * Parse directories from path. It is assumed that the path is given to the file. 20 | * For example the path "Folder/SubFolder/File.txt" will return an array of two elements, "Folder/" and "Folder/SubFolder/" 21 | * 22 | * @param FilePath The path to parse directories from 23 | * @return Parsed directories 24 | */ 25 | UFUNCTION(BlueprintCallable, Category = "Runtime Archiver|Utilities") 26 | static TArray ParseDirectories(const FString& FilePath); 27 | }; 28 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Public/Streams/RuntimeArchiverBaseStream.h: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #pragma once 4 | 5 | /** 6 | * Base tar archive stream. Do not create it directly 7 | */ 8 | class FRuntimeArchiverBaseStream 9 | { 10 | protected: 11 | /** 12 | * Base constructor 13 | * 14 | * @param bWrite Whether to have write permission or not 15 | */ 16 | explicit FRuntimeArchiverBaseStream(bool bWrite) 17 | : Position(0) 18 | , bWrite(bWrite) 19 | { 20 | } 21 | 22 | public: 23 | /** It should be impossible to create this class directly */ 24 | FRuntimeArchiverBaseStream() = delete; 25 | 26 | /** Virtual destructor to properly destroy child classes */ 27 | virtual ~FRuntimeArchiverBaseStream() = default; 28 | 29 | /** 30 | * Check if the the stream seems to be valid or not 31 | * 32 | * @return Whether the stream is valid or not 33 | */ 34 | virtual bool IsValid() const 35 | { 36 | ensureMsgf(false, TEXT("IsValid cannot be called from runtime archiver base stream")); 37 | return false; 38 | } 39 | 40 | bool IsWrite() const { return bWrite; } 41 | 42 | /** 43 | * Get the current write or read position 44 | */ 45 | int64 Tell() const { return Position; } 46 | 47 | /** 48 | * Seek archived data at a specified position. In other words, change the current write or read position 49 | * 50 | * @param NewPosition Position for seeking the archived data 51 | */ 52 | virtual bool Seek(int64 NewPosition) 53 | { 54 | ensureMsgf(false, TEXT("Seek cannot be called from runtime archiver base stream")); 55 | return false; 56 | } 57 | 58 | /** 59 | * Read archived data from the current position 60 | * 61 | * @param Data In-memory data pointer to fill 62 | * @param Size Data size 63 | * @return Whether the operation was successful or not 64 | */ 65 | virtual bool Read(void* Data, int64 Size) 66 | { 67 | ensureMsgf(false, TEXT("Read cannot be called from runtime archiver base stream")); 68 | return false; 69 | } 70 | 71 | /** 72 | * Write archived data from the current position 73 | * 74 | * @param Data In-memory data pointer to retrieve 75 | * @param Size Data size 76 | * @return Whether the operation was successful or not 77 | */ 78 | virtual bool Write(const void* Data, int64 Size) 79 | { 80 | ensureMsgf(false, TEXT("Write cannot be called from runtime archiver base stream")); 81 | return false; 82 | } 83 | 84 | /** 85 | * Get the total size 86 | */ 87 | virtual int64 Size() 88 | { 89 | ensureMsgf(false, TEXT("Size cannot be called from runtime archiver base stream")); 90 | return 0; 91 | } 92 | 93 | protected: 94 | /** Current read or write position */ 95 | int64 Position; 96 | 97 | /** Whether there is write permission or not */ 98 | bool bWrite; 99 | }; 100 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Public/Streams/RuntimeArchiverFileStream.h: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #pragma once 4 | 5 | #include "RuntimeArchiverBaseStream.h" 6 | 7 | /** 8 | * File tar stream. Manages data at the file system level 9 | */ 10 | class RUNTIMEARCHIVER_API FRuntimeArchiverFileStream : public FRuntimeArchiverBaseStream 11 | { 12 | public: 13 | /** It should be impossible to create this object by the default constructor */ 14 | FRuntimeArchiverFileStream() = delete; 15 | 16 | /** 17 | * Open a tar archive file stream 18 | * 19 | * @param ArchivePath Path to open an archive 20 | * @param bWrite Whether to open for writing or not 21 | */ 22 | explicit FRuntimeArchiverFileStream(const FString& ArchivePath, bool bWrite); 23 | 24 | virtual ~FRuntimeArchiverFileStream() override; 25 | 26 | //~ Begin FRuntimeArchiverBaseStream Interface 27 | virtual bool IsValid() const override; 28 | virtual bool Read(void* Data, int64 Size) override; 29 | virtual bool Write(const void* Data, int64 Size) override; 30 | virtual bool Seek(int64 NewPosition) override; 31 | virtual int64 Size() override; 32 | //~ End FRuntimeArchiverBaseStream Interface 33 | 34 | private: 35 | /** The file handle used to read or write */ 36 | IFileHandle* FileHandle; 37 | }; 38 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/Public/Streams/RuntimeArchiverMemoryStream.h: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | #pragma once 4 | 5 | #include "RuntimeArchiverBaseStream.h" 6 | 7 | /** 8 | * Memory tar stream. Manages data at the memory level 9 | */ 10 | class RUNTIMEARCHIVER_API FRuntimeArchiverMemoryStream : public FRuntimeArchiverBaseStream 11 | { 12 | public: 13 | /** It should be impossible to create this object by the default constructor */ 14 | FRuntimeArchiverMemoryStream() = delete; 15 | 16 | virtual ~FRuntimeArchiverMemoryStream() override = default; 17 | 18 | /** 19 | * Read-only constructor 20 | * 21 | * @param ArchiveData Binary archive data 22 | */ 23 | explicit FRuntimeArchiverMemoryStream(const TArray64& ArchiveData); 24 | 25 | /** 26 | * Write constructor 27 | * 28 | * @param InitialAllocationSize Estimated archive size if known. Avoids unnecessary memory allocation 29 | */ 30 | explicit FRuntimeArchiverMemoryStream(int32 InitialAllocationSize); 31 | 32 | //~ Begin FArchiverTarBaseStream Interface 33 | virtual bool IsValid() const override; 34 | virtual bool Read(void* Data, int64 Size) override; 35 | virtual bool Write(const void* Data, int64 Size) override; 36 | virtual bool Seek(int64 NewPosition) override; 37 | virtual int64 Size() override; 38 | //~ End FArchiverTarBaseStream Interface 39 | 40 | protected: 41 | /** Binary archive data */ 42 | TArray64 ArchiveData; 43 | }; 44 | -------------------------------------------------------------------------------- /Source/RuntimeArchiver/RuntimeArchiver.Build.cs: -------------------------------------------------------------------------------- 1 | // Georgy Treshchev 2024. 2 | 3 | using System; 4 | using System.IO; 5 | using UnrealBuildTool; 6 | 7 | public class RuntimeArchiver : ModuleRules 8 | { 9 | 10 | private string ThirdPartPath 11 | { 12 | get { return Path.GetFullPath(Path.Combine(ModuleDirectory, "../ThirdParty/")); } 13 | } 14 | private string MinizPath 15 | { 16 | get { return Path.GetFullPath(Path.Combine(ModuleDirectory, ThirdPartPath, "miniz")); } 17 | } 18 | 19 | public RuntimeArchiver(ReadOnlyTargetRules Target) : base(Target) 20 | { 21 | PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; 22 | 23 | PrivateIncludePaths.AddRange( 24 | new string[] 25 | { 26 | MinizPath 27 | } 28 | ); 29 | 30 | PublicDependencyModuleNames.AddRange( 31 | new string[] 32 | { 33 | "Core" 34 | } 35 | ); 36 | 37 | PrivateDependencyModuleNames.AddRange( 38 | new string[] 39 | { 40 | "CoreUObject", 41 | "Engine" 42 | } 43 | ); 44 | } 45 | } -------------------------------------------------------------------------------- /Source/ThirdParty/miniz/miniz.h: -------------------------------------------------------------------------------- 1 | /* miniz.c 3.0.2 - public domain deflate/inflate, zlib-subset, ZIP reading/writing/appending, PNG writing 2 | See "unlicense" statement at the end of this file. 3 | Rich Geldreich , last updated Oct. 13, 2013 4 | Implements RFC 1950: http://www.ietf.org/rfc/rfc1950.txt and RFC 1951: http://www.ietf.org/rfc/rfc1951.txt 5 | 6 | Most API's defined in miniz.c are optional. For example, to disable the archive related functions just define 7 | MINIZ_NO_ARCHIVE_APIS, or to get rid of all stdio usage define MINIZ_NO_STDIO (see the list below for more macros). 8 | 9 | * Low-level Deflate/Inflate implementation notes: 10 | 11 | Compression: Use the "tdefl" API's. The compressor supports raw, static, and dynamic blocks, lazy or 12 | greedy parsing, match length filtering, RLE-only, and Huffman-only streams. It performs and compresses 13 | approximately as well as zlib. 14 | 15 | Decompression: Use the "tinfl" API's. The entire decompressor is implemented as a single function 16 | coroutine: see tinfl_decompress(). It supports decompression into a 32KB (or larger power of 2) wrapping buffer, or into a memory 17 | block large enough to hold the entire file. 18 | 19 | The low-level tdefl/tinfl API's do not make any use of dynamic memory allocation. 20 | 21 | * zlib-style API notes: 22 | 23 | miniz.c implements a fairly large subset of zlib. There's enough functionality present for it to be a drop-in 24 | zlib replacement in many apps: 25 | The z_stream struct, optional memory allocation callbacks 26 | deflateInit/deflateInit2/deflate/deflateReset/deflateEnd/deflateBound 27 | inflateInit/inflateInit2/inflate/inflateReset/inflateEnd 28 | compress, compress2, compressBound, uncompress 29 | CRC-32, Adler-32 - Using modern, minimal code size, CPU cache friendly routines. 30 | Supports raw deflate streams or standard zlib streams with adler-32 checking. 31 | 32 | Limitations: 33 | The callback API's are not implemented yet. No support for gzip headers or zlib static dictionaries. 34 | I've tried to closely emulate zlib's various flavors of stream flushing and return status codes, but 35 | there are no guarantees that miniz.c pulls this off perfectly. 36 | 37 | * PNG writing: See the tdefl_write_image_to_png_file_in_memory() function, originally written by 38 | Alex Evans. Supports 1-4 bytes/pixel images. 39 | 40 | * ZIP archive API notes: 41 | 42 | The ZIP archive API's where designed with simplicity and efficiency in mind, with just enough abstraction to 43 | get the job done with minimal fuss. There are simple API's to retrieve file information, read files from 44 | existing archives, create new archives, append new files to existing archives, or clone archive data from 45 | one archive to another. It supports archives located in memory or the heap, on disk (using stdio.h), 46 | or you can specify custom file read/write callbacks. 47 | 48 | - Archive reading: Just call this function to read a single file from a disk archive: 49 | 50 | void *mz_zip_extract_archive_file_to_heap(const char *pZip_filename, const char *pArchive_name, 51 | size_t *pSize, mz_uint zip_flags); 52 | 53 | For more complex cases, use the "mz_zip_reader" functions. Upon opening an archive, the entire central 54 | directory is located and read as-is into memory, and subsequent file access only occurs when reading individual files. 55 | 56 | - Archives file scanning: The simple way is to use this function to scan a loaded archive for a specific file: 57 | 58 | int mz_zip_reader_locate_file(mz_zip_archive *pZip, const char *pName, const char *pComment, mz_uint flags); 59 | 60 | The locate operation can optionally check file comments too, which (as one example) can be used to identify 61 | multiple versions of the same file in an archive. This function uses a simple linear search through the central 62 | directory, so it's not very fast. 63 | 64 | Alternately, you can iterate through all the files in an archive (using mz_zip_reader_get_num_files()) and 65 | retrieve detailed info on each file by calling mz_zip_reader_file_stat(). 66 | 67 | - Archive creation: Use the "mz_zip_writer" functions. The ZIP writer immediately writes compressed file data 68 | to disk and builds an exact image of the central directory in memory. The central directory image is written 69 | all at once at the end of the archive file when the archive is finalized. 70 | 71 | The archive writer can optionally align each file's local header and file data to any power of 2 alignment, 72 | which can be useful when the archive will be read from optical media. Also, the writer supports placing 73 | arbitrary data blobs at the very beginning of ZIP archives. Archives written using either feature are still 74 | readable by any ZIP tool. 75 | 76 | - Archive appending: The simple way to add a single file to an archive is to call this function: 77 | 78 | mz_bool mz_zip_add_mem_to_archive_file_in_place(const char *pZip_filename, const char *pArchive_name, 79 | const void *pBuf, size_t buf_size, const void *pComment, mz_uint16 comment_size, mz_uint level_and_flags); 80 | 81 | The archive will be created if it doesn't already exist, otherwise it'll be appended to. 82 | Note the appending is done in-place and is not an atomic operation, so if something goes wrong 83 | during the operation it's possible the archive could be left without a central directory (although the local 84 | file headers and file data will be fine, so the archive will be recoverable). 85 | 86 | For more complex archive modification scenarios: 87 | 1. The safest way is to use a mz_zip_reader to read the existing archive, cloning only those bits you want to 88 | preserve into a new archive using using the mz_zip_writer_add_from_zip_reader() function (which compiles the 89 | compressed file data as-is). When you're done, delete the old archive and rename the newly written archive, and 90 | you're done. This is safe but requires a bunch of temporary disk space or heap memory. 91 | 92 | 2. Or, you can convert an mz_zip_reader in-place to an mz_zip_writer using mz_zip_writer_init_from_reader(), 93 | append new files as needed, then finalize the archive which will write an updated central directory to the 94 | original archive. (This is basically what mz_zip_add_mem_to_archive_file_in_place() does.) There's a 95 | possibility that the archive's central directory could be lost with this method if anything goes wrong, though. 96 | 97 | - ZIP archive support limitations: 98 | No spanning support. Extraction functions can only handle unencrypted, stored or deflated files. 99 | Requires streams capable of seeking. 100 | 101 | * This is a header file library, like stb_image.c. To get only a header file, either cut and paste the 102 | below header, or create miniz.h, #define MINIZ_HEADER_FILE_ONLY, and then include miniz.c from it. 103 | 104 | * Important: For best perf. be sure to customize the below macros for your target platform: 105 | #define MINIZ_USE_UNALIGNED_LOADS_AND_STORES 1 106 | #define MINIZ_LITTLE_ENDIAN 1 107 | #define MINIZ_HAS_64BIT_REGISTERS 1 108 | 109 | * On platforms using glibc, Be sure to "#define _LARGEFILE64_SOURCE 1" before including miniz.c to ensure miniz 110 | uses the 64-bit variants: fopen64(), stat64(), etc. Otherwise you won't be able to process large files 111 | (i.e. 32-bit stat() fails for me on files > 0x7FFFFFFF bytes). 112 | */ 113 | #pragma once 114 | 115 | #include "miniz_export.h" 116 | 117 | /* Defines to completely disable specific portions of miniz.c: 118 | If all macros here are defined the only functionality remaining will be CRC-32 and adler-32. */ 119 | 120 | /* Define MINIZ_NO_STDIO to disable all usage and any functions which rely on stdio for file I/O. */ 121 | /*#define MINIZ_NO_STDIO */ 122 | 123 | /* If MINIZ_NO_TIME is specified then the ZIP archive functions will not be able to get the current time, or */ 124 | /* get/set file times, and the C run-time funcs that get/set times won't be called. */ 125 | /* The current downside is the times written to your archives will be from 1979. */ 126 | /*#define MINIZ_NO_TIME */ 127 | 128 | /* Define MINIZ_NO_DEFLATE_APIS to disable all compression API's. */ 129 | /*#define MINIZ_NO_DEFLATE_APIS */ 130 | 131 | /* Define MINIZ_NO_INFLATE_APIS to disable all decompression API's. */ 132 | /*#define MINIZ_NO_INFLATE_APIS */ 133 | 134 | /* Define MINIZ_NO_ARCHIVE_APIS to disable all ZIP archive API's. */ 135 | /*#define MINIZ_NO_ARCHIVE_APIS */ 136 | 137 | /* Define MINIZ_NO_ARCHIVE_WRITING_APIS to disable all writing related ZIP archive API's. */ 138 | /*#define MINIZ_NO_ARCHIVE_WRITING_APIS */ 139 | 140 | /* Define MINIZ_NO_ZLIB_APIS to remove all ZLIB-style compression/decompression API's. */ 141 | /*#define MINIZ_NO_ZLIB_APIS */ 142 | 143 | /* Define MINIZ_NO_ZLIB_COMPATIBLE_NAME to disable zlib names, to prevent conflicts against stock zlib. */ 144 | /*#define MINIZ_NO_ZLIB_COMPATIBLE_NAMES */ 145 | 146 | /* Define MINIZ_NO_MALLOC to disable all calls to malloc, free, and realloc. 147 | Note if MINIZ_NO_MALLOC is defined then the user must always provide custom user alloc/free/realloc 148 | callbacks to the zlib and archive API's, and a few stand-alone helper API's which don't provide custom user 149 | functions (such as tdefl_compress_mem_to_heap() and tinfl_decompress_mem_to_heap()) won't work. */ 150 | /*#define MINIZ_NO_MALLOC */ 151 | 152 | #ifdef MINIZ_NO_INFLATE_APIS 153 | #define MINIZ_NO_ARCHIVE_APIS 154 | #endif 155 | 156 | #ifdef MINIZ_NO_DEFLATE_APIS 157 | #define MINIZ_NO_ARCHIVE_WRITING_APIS 158 | #endif 159 | 160 | #if defined(__TINYC__) && (defined(__linux) || defined(__linux__)) 161 | /* TODO: Work around "error: include file 'sys\utime.h' when compiling with tcc on Linux */ 162 | #define MINIZ_NO_TIME 163 | #endif 164 | 165 | #include 166 | 167 | #if !defined(MINIZ_NO_TIME) && !defined(MINIZ_NO_ARCHIVE_APIS) 168 | #include 169 | #endif 170 | 171 | #if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__i386) || defined(__i486__) || defined(__i486) || defined(i386) || defined(__ia64__) || defined(__x86_64__) 172 | /* MINIZ_X86_OR_X64_CPU is only used to help set the below macros. */ 173 | #define MINIZ_X86_OR_X64_CPU 1 174 | #else 175 | #define MINIZ_X86_OR_X64_CPU 0 176 | #endif 177 | 178 | /* Set MINIZ_LITTLE_ENDIAN only if not set */ 179 | #if !defined(MINIZ_LITTLE_ENDIAN) 180 | #if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) 181 | 182 | #if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) 183 | /* Set MINIZ_LITTLE_ENDIAN to 1 if the processor is little endian. */ 184 | #define MINIZ_LITTLE_ENDIAN 1 185 | #else 186 | #define MINIZ_LITTLE_ENDIAN 0 187 | #endif 188 | 189 | #else 190 | 191 | #if MINIZ_X86_OR_X64_CPU 192 | #define MINIZ_LITTLE_ENDIAN 1 193 | #else 194 | #define MINIZ_LITTLE_ENDIAN 0 195 | #endif 196 | 197 | #endif 198 | #endif 199 | 200 | /* Using unaligned loads and stores causes errors when using UBSan */ 201 | #if defined(__has_feature) 202 | #if __has_feature(undefined_behavior_sanitizer) 203 | #define MINIZ_USE_UNALIGNED_LOADS_AND_STORES 0 204 | #endif 205 | #endif 206 | 207 | /* Set MINIZ_USE_UNALIGNED_LOADS_AND_STORES only if not set */ 208 | #if !defined(MINIZ_USE_UNALIGNED_LOADS_AND_STORES) 209 | #if MINIZ_X86_OR_X64_CPU 210 | /* Set MINIZ_USE_UNALIGNED_LOADS_AND_STORES to 1 on CPU's that permit efficient integer loads and stores from unaligned addresses. */ 211 | #define MINIZ_USE_UNALIGNED_LOADS_AND_STORES 0 212 | #define MINIZ_UNALIGNED_USE_MEMCPY 213 | #else 214 | #define MINIZ_USE_UNALIGNED_LOADS_AND_STORES 0 215 | #endif 216 | #endif 217 | 218 | #if defined(_M_X64) || defined(_WIN64) || defined(__MINGW64__) || defined(_LP64) || defined(__LP64__) || defined(__ia64__) || defined(__x86_64__) 219 | /* Set MINIZ_HAS_64BIT_REGISTERS to 1 if operations on 64-bit integers are reasonably fast (and don't involve compiler generated calls to helper functions). */ 220 | //#define MINIZ_HAS_64BIT_REGISTERS 1 221 | #else 222 | //#define MINIZ_HAS_64BIT_REGISTERS 0 223 | #endif 224 | 225 | #ifdef __cplusplus 226 | extern "C" 227 | { 228 | #endif 229 | 230 | /* ------------------- zlib-style API Definitions. */ 231 | 232 | /* For more compatibility with zlib, miniz.c uses unsigned long for some parameters/struct members. Beware: mz_ulong can be either 32 or 64-bits! */ 233 | typedef unsigned long mz_ulong; 234 | 235 | /* mz_free() internally uses the MZ_FREE() macro (which by default calls free() unless you've modified the MZ_MALLOC macro) to release a block allocated from the heap. */ 236 | MINIZ_EXPORT void mz_free(void *p); 237 | 238 | #define MZ_ADLER32_INIT (1) 239 | /* mz_adler32() returns the initial adler-32 value to use when called with ptr==NULL. */ 240 | MINIZ_EXPORT mz_ulong mz_adler32(mz_ulong adler, const unsigned char *ptr, size_t buf_len); 241 | 242 | #define MZ_CRC32_INIT (0) 243 | /* mz_crc32() returns the initial CRC-32 value to use when called with ptr==NULL. */ 244 | MINIZ_EXPORT mz_ulong mz_crc32(mz_ulong crc, const unsigned char *ptr, size_t buf_len); 245 | 246 | /* Compression strategies. */ 247 | enum 248 | { 249 | MZ_DEFAULT_STRATEGY = 0, 250 | MZ_FILTERED = 1, 251 | MZ_HUFFMAN_ONLY = 2, 252 | MZ_RLE = 3, 253 | MZ_FIXED = 4 254 | }; 255 | 256 | /* Method */ 257 | #define MZ_DEFLATED 8 258 | 259 | /* Heap allocation callbacks. 260 | Note that mz_alloc_func parameter types purposely differ from zlib's: items/size is size_t, not unsigned long. */ 261 | typedef void *(*mz_alloc_func)(void *opaque, size_t items, size_t size); 262 | typedef void (*mz_free_func)(void *opaque, void *address); 263 | typedef void *(*mz_realloc_func)(void *opaque, void *address, size_t items, size_t size); 264 | 265 | /* Compression levels: 0-9 are the standard zlib-style levels, 10 is best possible compression (not zlib compatible, and may be very slow), MZ_DEFAULT_COMPRESSION=MZ_DEFAULT_LEVEL. */ 266 | enum 267 | { 268 | MZ_NO_COMPRESSION = 0, 269 | MZ_BEST_SPEED = 1, 270 | MZ_BEST_COMPRESSION = 9, 271 | MZ_UBER_COMPRESSION = 10, 272 | MZ_DEFAULT_LEVEL = 6, 273 | MZ_DEFAULT_COMPRESSION = -1 274 | }; 275 | 276 | #define MZ_VERSION "11.0.2" 277 | #define MZ_VERNUM 0xB002 278 | #define MZ_VER_MAJOR 11 279 | #define MZ_VER_MINOR 2 280 | #define MZ_VER_REVISION 0 281 | #define MZ_VER_SUBREVISION 0 282 | 283 | #ifndef MINIZ_NO_ZLIB_APIS 284 | 285 | /* Flush values. For typical usage you only need MZ_NO_FLUSH and MZ_FINISH. The other values are for advanced use (refer to the zlib docs). */ 286 | enum 287 | { 288 | MZ_NO_FLUSH = 0, 289 | MZ_PARTIAL_FLUSH = 1, 290 | MZ_SYNC_FLUSH = 2, 291 | MZ_FULL_FLUSH = 3, 292 | MZ_FINISH = 4, 293 | MZ_BLOCK = 5 294 | }; 295 | 296 | /* Return status codes. MZ_PARAM_ERROR is non-standard. */ 297 | enum 298 | { 299 | MZ_OK = 0, 300 | MZ_STREAM_END = 1, 301 | MZ_NEED_DICT = 2, 302 | MZ_ERRNO = -1, 303 | MZ_STREAM_ERROR = -2, 304 | MZ_DATA_ERROR = -3, 305 | MZ_MEM_ERROR = -4, 306 | MZ_BUF_ERROR = -5, 307 | MZ_VERSION_ERROR = -6, 308 | MZ_PARAM_ERROR = -10000 309 | }; 310 | 311 | /* Window bits */ 312 | #define MZ_DEFAULT_WINDOW_BITS 15 313 | 314 | struct mz_internal_state; 315 | 316 | /* Compression/decompression stream struct. */ 317 | typedef struct mz_stream_s 318 | { 319 | const unsigned char *next_in; /* pointer to next byte to read */ 320 | unsigned int avail_in; /* number of bytes available at next_in */ 321 | mz_ulong total_in; /* total number of bytes consumed so far */ 322 | 323 | unsigned char *next_out; /* pointer to next byte to write */ 324 | unsigned int avail_out; /* number of bytes that can be written to next_out */ 325 | mz_ulong total_out; /* total number of bytes produced so far */ 326 | 327 | char *msg; /* error msg (unused) */ 328 | struct mz_internal_state *state; /* internal state, allocated by zalloc/zfree */ 329 | 330 | mz_alloc_func zalloc; /* optional heap allocation function (defaults to malloc) */ 331 | mz_free_func zfree; /* optional heap free function (defaults to free) */ 332 | void *opaque; /* heap alloc function user pointer */ 333 | 334 | int data_type; /* data_type (unused) */ 335 | mz_ulong adler; /* adler32 of the source or uncompressed data */ 336 | mz_ulong reserved; /* not used */ 337 | } mz_stream; 338 | 339 | typedef mz_stream *mz_streamp; 340 | 341 | /* Returns the version string of miniz.c. */ 342 | MINIZ_EXPORT const char *mz_version(void); 343 | 344 | #ifndef MINIZ_NO_DEFLATE_APIS 345 | 346 | /* mz_deflateInit() initializes a compressor with default options: */ 347 | /* Parameters: */ 348 | /* pStream must point to an initialized mz_stream struct. */ 349 | /* level must be between [MZ_NO_COMPRESSION, MZ_BEST_COMPRESSION]. */ 350 | /* level 1 enables a specially optimized compression function that's been optimized purely for performance, not ratio. */ 351 | /* (This special func. is currently only enabled when MINIZ_USE_UNALIGNED_LOADS_AND_STORES and MINIZ_LITTLE_ENDIAN are defined.) */ 352 | /* Return values: */ 353 | /* MZ_OK on success. */ 354 | /* MZ_STREAM_ERROR if the stream is bogus. */ 355 | /* MZ_PARAM_ERROR if the input parameters are bogus. */ 356 | /* MZ_MEM_ERROR on out of memory. */ 357 | MINIZ_EXPORT int mz_deflateInit(mz_streamp pStream, int level); 358 | 359 | /* mz_deflateInit2() is like mz_deflate(), except with more control: */ 360 | /* Additional parameters: */ 361 | /* method must be MZ_DEFLATED */ 362 | /* window_bits must be MZ_DEFAULT_WINDOW_BITS (to wrap the deflate stream with zlib header/adler-32 footer) or -MZ_DEFAULT_WINDOW_BITS (raw deflate/no header or footer) */ 363 | /* mem_level must be between [1, 9] (it's checked but ignored by miniz.c) */ 364 | MINIZ_EXPORT int mz_deflateInit2(mz_streamp pStream, int level, int method, int window_bits, int mem_level, int strategy); 365 | 366 | /* Quickly resets a compressor without having to reallocate anything. Same as calling mz_deflateEnd() followed by mz_deflateInit()/mz_deflateInit2(). */ 367 | MINIZ_EXPORT int mz_deflateReset(mz_streamp pStream); 368 | 369 | /* mz_deflate() compresses the input to output, consuming as much of the input and producing as much output as possible. */ 370 | /* Parameters: */ 371 | /* pStream is the stream to read from and write to. You must initialize/update the next_in, avail_in, next_out, and avail_out members. */ 372 | /* flush may be MZ_NO_FLUSH, MZ_PARTIAL_FLUSH/MZ_SYNC_FLUSH, MZ_FULL_FLUSH, or MZ_FINISH. */ 373 | /* Return values: */ 374 | /* MZ_OK on success (when flushing, or if more input is needed but not available, and/or there's more output to be written but the output buffer is full). */ 375 | /* MZ_STREAM_END if all input has been consumed and all output bytes have been written. Don't call mz_deflate() on the stream anymore. */ 376 | /* MZ_STREAM_ERROR if the stream is bogus. */ 377 | /* MZ_PARAM_ERROR if one of the parameters is invalid. */ 378 | /* MZ_BUF_ERROR if no forward progress is possible because the input and/or output buffers are empty. (Fill up the input buffer or free up some output space and try again.) */ 379 | MINIZ_EXPORT int mz_deflate(mz_streamp pStream, int flush); 380 | 381 | /* mz_deflateEnd() deinitializes a compressor: */ 382 | /* Return values: */ 383 | /* MZ_OK on success. */ 384 | /* MZ_STREAM_ERROR if the stream is bogus. */ 385 | MINIZ_EXPORT int mz_deflateEnd(mz_streamp pStream); 386 | 387 | /* mz_deflateBound() returns a (very) conservative upper bound on the amount of data that could be generated by deflate(), assuming flush is set to only MZ_NO_FLUSH or MZ_FINISH. */ 388 | MINIZ_EXPORT mz_ulong mz_deflateBound(mz_streamp pStream, mz_ulong source_len); 389 | 390 | /* Single-call compression functions mz_compress() and mz_compress2(): */ 391 | /* Returns MZ_OK on success, or one of the error codes from mz_deflate() on failure. */ 392 | MINIZ_EXPORT int mz_compress(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len); 393 | MINIZ_EXPORT int mz_compress2(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len, int level); 394 | 395 | /* mz_compressBound() returns a (very) conservative upper bound on the amount of data that could be generated by calling mz_compress(). */ 396 | MINIZ_EXPORT mz_ulong mz_compressBound(mz_ulong source_len); 397 | 398 | #endif /*#ifndef MINIZ_NO_DEFLATE_APIS*/ 399 | 400 | #ifndef MINIZ_NO_INFLATE_APIS 401 | 402 | /* Initializes a decompressor. */ 403 | MINIZ_EXPORT int mz_inflateInit(mz_streamp pStream); 404 | 405 | /* mz_inflateInit2() is like mz_inflateInit() with an additional option that controls the window size and whether or not the stream has been wrapped with a zlib header/footer: */ 406 | /* window_bits must be MZ_DEFAULT_WINDOW_BITS (to parse zlib header/footer) or -MZ_DEFAULT_WINDOW_BITS (raw deflate). */ 407 | MINIZ_EXPORT int mz_inflateInit2(mz_streamp pStream, int window_bits); 408 | 409 | /* Quickly resets a compressor without having to reallocate anything. Same as calling mz_inflateEnd() followed by mz_inflateInit()/mz_inflateInit2(). */ 410 | MINIZ_EXPORT int mz_inflateReset(mz_streamp pStream); 411 | 412 | /* Decompresses the input stream to the output, consuming only as much of the input as needed, and writing as much to the output as possible. */ 413 | /* Parameters: */ 414 | /* pStream is the stream to read from and write to. You must initialize/update the next_in, avail_in, next_out, and avail_out members. */ 415 | /* flush may be MZ_NO_FLUSH, MZ_SYNC_FLUSH, or MZ_FINISH. */ 416 | /* On the first call, if flush is MZ_FINISH it's assumed the input and output buffers are both sized large enough to decompress the entire stream in a single call (this is slightly faster). */ 417 | /* MZ_FINISH implies that there are no more source bytes available beside what's already in the input buffer, and that the output buffer is large enough to hold the rest of the decompressed data. */ 418 | /* Return values: */ 419 | /* MZ_OK on success. Either more input is needed but not available, and/or there's more output to be written but the output buffer is full. */ 420 | /* MZ_STREAM_END if all needed input has been consumed and all output bytes have been written. For zlib streams, the adler-32 of the decompressed data has also been verified. */ 421 | /* MZ_STREAM_ERROR if the stream is bogus. */ 422 | /* MZ_DATA_ERROR if the deflate stream is invalid. */ 423 | /* MZ_PARAM_ERROR if one of the parameters is invalid. */ 424 | /* MZ_BUF_ERROR if no forward progress is possible because the input buffer is empty but the inflater needs more input to continue, or if the output buffer is not large enough. Call mz_inflate() again */ 425 | /* with more input data, or with more room in the output buffer (except when using single call decompression, described above). */ 426 | MINIZ_EXPORT int mz_inflate(mz_streamp pStream, int flush); 427 | 428 | /* Deinitializes a decompressor. */ 429 | MINIZ_EXPORT int mz_inflateEnd(mz_streamp pStream); 430 | 431 | /* Single-call decompression. */ 432 | /* Returns MZ_OK on success, or one of the error codes from mz_inflate() on failure. */ 433 | MINIZ_EXPORT int mz_uncompress(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong source_len); 434 | MINIZ_EXPORT int mz_uncompress2(unsigned char *pDest, mz_ulong *pDest_len, const unsigned char *pSource, mz_ulong *pSource_len); 435 | #endif /*#ifndef MINIZ_NO_INFLATE_APIS*/ 436 | 437 | /* Returns a string description of the specified error code, or NULL if the error code is invalid. */ 438 | MINIZ_EXPORT const char *mz_error(int err); 439 | 440 | /* Redefine zlib-compatible names to miniz equivalents, so miniz.c can be used as a drop-in replacement for the subset of zlib that miniz.c supports. */ 441 | /* Define MINIZ_NO_ZLIB_COMPATIBLE_NAMES to disable zlib-compatibility if you use zlib in the same project. */ 442 | #ifndef MINIZ_NO_ZLIB_COMPATIBLE_NAMES 443 | typedef unsigned char Byte; 444 | typedef unsigned int uInt; 445 | typedef mz_ulong uLong; 446 | typedef Byte Bytef; 447 | typedef uInt uIntf; 448 | typedef char charf; 449 | typedef int intf; 450 | typedef void *voidpf; 451 | typedef uLong uLongf; 452 | typedef void *voidp; 453 | typedef void *const voidpc; 454 | #define Z_NULL 0 455 | #define Z_NO_FLUSH MZ_NO_FLUSH 456 | #define Z_PARTIAL_FLUSH MZ_PARTIAL_FLUSH 457 | #define Z_SYNC_FLUSH MZ_SYNC_FLUSH 458 | #define Z_FULL_FLUSH MZ_FULL_FLUSH 459 | #define Z_FINISH MZ_FINISH 460 | #define Z_BLOCK MZ_BLOCK 461 | #define Z_OK MZ_OK 462 | #define Z_STREAM_END MZ_STREAM_END 463 | #define Z_NEED_DICT MZ_NEED_DICT 464 | #define Z_ERRNO MZ_ERRNO 465 | #define Z_STREAM_ERROR MZ_STREAM_ERROR 466 | #define Z_DATA_ERROR MZ_DATA_ERROR 467 | #define Z_MEM_ERROR MZ_MEM_ERROR 468 | #define Z_BUF_ERROR MZ_BUF_ERROR 469 | #define Z_VERSION_ERROR MZ_VERSION_ERROR 470 | #define Z_PARAM_ERROR MZ_PARAM_ERROR 471 | #define Z_NO_COMPRESSION MZ_NO_COMPRESSION 472 | #define Z_BEST_SPEED MZ_BEST_SPEED 473 | #define Z_BEST_COMPRESSION MZ_BEST_COMPRESSION 474 | #define Z_DEFAULT_COMPRESSION MZ_DEFAULT_COMPRESSION 475 | #define Z_DEFAULT_STRATEGY MZ_DEFAULT_STRATEGY 476 | #define Z_FILTERED MZ_FILTERED 477 | #define Z_HUFFMAN_ONLY MZ_HUFFMAN_ONLY 478 | #define Z_RLE MZ_RLE 479 | #define Z_FIXED MZ_FIXED 480 | #define Z_DEFLATED MZ_DEFLATED 481 | #define Z_DEFAULT_WINDOW_BITS MZ_DEFAULT_WINDOW_BITS 482 | #define alloc_func mz_alloc_func 483 | #define free_func mz_free_func 484 | #define internal_state mz_internal_state 485 | #define z_stream mz_stream 486 | 487 | #ifndef MINIZ_NO_DEFLATE_APIS 488 | #define deflateInit mz_deflateInit 489 | #define deflateInit2 mz_deflateInit2 490 | #define deflateReset mz_deflateReset 491 | #define deflate mz_deflate 492 | #define deflateEnd mz_deflateEnd 493 | #define deflateBound mz_deflateBound 494 | #define compress mz_compress 495 | #define compress2 mz_compress2 496 | #define compressBound mz_compressBound 497 | #endif /*#ifndef MINIZ_NO_DEFLATE_APIS*/ 498 | 499 | #ifndef MINIZ_NO_INFLATE_APIS 500 | #define inflateInit mz_inflateInit 501 | #define inflateInit2 mz_inflateInit2 502 | #define inflateReset mz_inflateReset 503 | #define inflate mz_inflate 504 | #define inflateEnd mz_inflateEnd 505 | #define uncompress mz_uncompress 506 | #define uncompress2 mz_uncompress2 507 | #endif /*#ifndef MINIZ_NO_INFLATE_APIS*/ 508 | 509 | #define crc32 mz_crc32 510 | #define adler32 mz_adler32 511 | #define MAX_WBITS 15 512 | #define MAX_MEM_LEVEL 9 513 | #define zError mz_error 514 | #define ZLIB_VERSION MZ_VERSION 515 | #define ZLIB_VERNUM MZ_VERNUM 516 | #define ZLIB_VER_MAJOR MZ_VER_MAJOR 517 | #define ZLIB_VER_MINOR MZ_VER_MINOR 518 | #define ZLIB_VER_REVISION MZ_VER_REVISION 519 | #define ZLIB_VER_SUBREVISION MZ_VER_SUBREVISION 520 | #define zlibVersion mz_version 521 | #define zlib_version mz_version() 522 | #endif /* #ifndef MINIZ_NO_ZLIB_COMPATIBLE_NAMES */ 523 | 524 | #endif /* MINIZ_NO_ZLIB_APIS */ 525 | 526 | #ifdef __cplusplus 527 | } 528 | #endif 529 | 530 | #include "miniz_common.h" 531 | #include "miniz_tdef.h" 532 | #include "miniz_tinfl.h" 533 | #include "miniz_zip.h" 534 | -------------------------------------------------------------------------------- /Source/ThirdParty/miniz/miniz_common.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "miniz_export.h" 8 | 9 | /* ------------------- Types and macros */ 10 | typedef unsigned char mz_uint8; 11 | typedef int16_t mz_int16; 12 | typedef uint16_t mz_uint16; 13 | typedef uint32_t mz_uint32; 14 | typedef uint32_t mz_uint; 15 | typedef int64_t mz_int64; 16 | typedef uint64_t mz_uint64; 17 | typedef int mz_bool; 18 | 19 | #define MZ_FALSE (0) 20 | #define MZ_TRUE (1) 21 | 22 | /* Works around MSVC's spammy "warning C4127: conditional expression is constant" message. */ 23 | #ifdef _MSC_VER 24 | #define MZ_MACRO_END while (0, 0) 25 | #else 26 | #define MZ_MACRO_END while (0) 27 | #endif 28 | 29 | #ifdef MINIZ_NO_STDIO 30 | #define MZ_FILE void * 31 | #else 32 | #include 33 | #define MZ_FILE FILE 34 | #endif /* #ifdef MINIZ_NO_STDIO */ 35 | 36 | #ifdef MINIZ_NO_TIME 37 | typedef struct mz_dummy_time_t_tag 38 | { 39 | mz_uint32 m_dummy1; 40 | mz_uint32 m_dummy2; 41 | } mz_dummy_time_t; 42 | #define MZ_TIME_T mz_dummy_time_t 43 | #else 44 | #define MZ_TIME_T time_t 45 | #endif 46 | 47 | #define MZ_ASSERT(x) assert(x) 48 | 49 | #ifdef MINIZ_NO_MALLOC 50 | #define MZ_MALLOC(x) NULL 51 | #define MZ_FREE(x) (void)x, ((void)0) 52 | #define MZ_REALLOC(p, x) NULL 53 | #else 54 | #define MZ_MALLOC(x) malloc(x) 55 | #define MZ_FREE(x) free(x) 56 | #define MZ_REALLOC(p, x) realloc(p, x) 57 | #endif 58 | 59 | #define MZ_MAX(a, b) (((a) > (b)) ? (a) : (b)) 60 | #define MZ_MIN(a, b) (((a) < (b)) ? (a) : (b)) 61 | #define MZ_CLEAR_OBJ(obj) memset(&(obj), 0, sizeof(obj)) 62 | #define MZ_CLEAR_ARR(obj) memset((obj), 0, sizeof(obj)) 63 | #define MZ_CLEAR_PTR(obj) memset((obj), 0, sizeof(*obj)) 64 | 65 | #if MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN 66 | #define MZ_READ_LE16(p) *((const mz_uint16 *)(p)) 67 | #define MZ_READ_LE32(p) *((const mz_uint32 *)(p)) 68 | #else 69 | #define MZ_READ_LE16(p) ((mz_uint32)(((const mz_uint8 *)(p))[0]) | ((mz_uint32)(((const mz_uint8 *)(p))[1]) << 8U)) 70 | #define MZ_READ_LE32(p) ((mz_uint32)(((const mz_uint8 *)(p))[0]) | ((mz_uint32)(((const mz_uint8 *)(p))[1]) << 8U) | ((mz_uint32)(((const mz_uint8 *)(p))[2]) << 16U) | ((mz_uint32)(((const mz_uint8 *)(p))[3]) << 24U)) 71 | #endif 72 | 73 | #define MZ_READ_LE64(p) (((mz_uint64)MZ_READ_LE32(p)) | (((mz_uint64)MZ_READ_LE32((const mz_uint8 *)(p) + sizeof(mz_uint32))) << 32U)) 74 | 75 | #ifdef _MSC_VER 76 | #define MZ_FORCEINLINE __forceinline 77 | #elif defined(__GNUC__) 78 | #define MZ_FORCEINLINE __inline__ __attribute__((__always_inline__)) 79 | #else 80 | #define MZ_FORCEINLINE inline 81 | #endif 82 | 83 | #ifdef __cplusplus 84 | extern "C" 85 | { 86 | #endif 87 | 88 | extern MINIZ_EXPORT void *miniz_def_alloc_func(void *opaque, size_t items, size_t size); 89 | extern MINIZ_EXPORT void miniz_def_free_func(void *opaque, void *address); 90 | extern MINIZ_EXPORT void *miniz_def_realloc_func(void *opaque, void *address, size_t items, size_t size); 91 | 92 | #define MZ_UINT16_MAX (0xFFFFU) 93 | #define MZ_UINT32_MAX (0xFFFFFFFFU) 94 | 95 | #ifdef __cplusplus 96 | } 97 | #endif 98 | -------------------------------------------------------------------------------- /Source/ThirdParty/miniz/miniz_export.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #define MINIZ_EXPORT -------------------------------------------------------------------------------- /Source/ThirdParty/miniz/miniz_tdef.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "miniz_common.h" 3 | 4 | #ifndef MINIZ_NO_DEFLATE_APIS 5 | 6 | #ifdef __cplusplus 7 | extern "C" 8 | { 9 | #endif 10 | /* ------------------- Low-level Compression API Definitions */ 11 | 12 | /* Set TDEFL_LESS_MEMORY to 1 to use less memory (compression will be slightly slower, and raw/dynamic blocks will be output more frequently). */ 13 | #ifndef TDEFL_LESS_MEMORY 14 | #define TDEFL_LESS_MEMORY 0 15 | #endif 16 | 17 | /* tdefl_init() compression flags logically OR'd together (low 12 bits contain the max. number of probes per dictionary search): */ 18 | /* TDEFL_DEFAULT_MAX_PROBES: The compressor defaults to 128 dictionary probes per dictionary search. 0=Huffman only, 1=Huffman+LZ (fastest/crap compression), 4095=Huffman+LZ (slowest/best compression). */ 19 | enum 20 | { 21 | TDEFL_HUFFMAN_ONLY = 0, 22 | TDEFL_DEFAULT_MAX_PROBES = 128, 23 | TDEFL_MAX_PROBES_MASK = 0xFFF 24 | }; 25 | 26 | /* TDEFL_WRITE_ZLIB_HEADER: If set, the compressor outputs a zlib header before the deflate data, and the Adler-32 of the source data at the end. Otherwise, you'll get raw deflate data. */ 27 | /* TDEFL_COMPUTE_ADLER32: Always compute the adler-32 of the input data (even when not writing zlib headers). */ 28 | /* TDEFL_GREEDY_PARSING_FLAG: Set to use faster greedy parsing, instead of more efficient lazy parsing. */ 29 | /* TDEFL_NONDETERMINISTIC_PARSING_FLAG: Enable to decrease the compressor's initialization time to the minimum, but the output may vary from run to run given the same input (depending on the contents of memory). */ 30 | /* TDEFL_RLE_MATCHES: Only look for RLE matches (matches with a distance of 1) */ 31 | /* TDEFL_FILTER_MATCHES: Discards matches <= 5 chars if enabled. */ 32 | /* TDEFL_FORCE_ALL_STATIC_BLOCKS: Disable usage of optimized Huffman tables. */ 33 | /* TDEFL_FORCE_ALL_RAW_BLOCKS: Only use raw (uncompressed) deflate blocks. */ 34 | /* The low 12 bits are reserved to control the max # of hash probes per dictionary lookup (see TDEFL_MAX_PROBES_MASK). */ 35 | enum 36 | { 37 | TDEFL_WRITE_ZLIB_HEADER = 0x01000, 38 | TDEFL_COMPUTE_ADLER32 = 0x02000, 39 | TDEFL_GREEDY_PARSING_FLAG = 0x04000, 40 | TDEFL_NONDETERMINISTIC_PARSING_FLAG = 0x08000, 41 | TDEFL_RLE_MATCHES = 0x10000, 42 | TDEFL_FILTER_MATCHES = 0x20000, 43 | TDEFL_FORCE_ALL_STATIC_BLOCKS = 0x40000, 44 | TDEFL_FORCE_ALL_RAW_BLOCKS = 0x80000 45 | }; 46 | 47 | /* High level compression functions: */ 48 | /* tdefl_compress_mem_to_heap() compresses a block in memory to a heap block allocated via malloc(). */ 49 | /* On entry: */ 50 | /* pSrc_buf, src_buf_len: Pointer and size of source block to compress. */ 51 | /* flags: The max match finder probes (default is 128) logically OR'd against the above flags. Higher probes are slower but improve compression. */ 52 | /* On return: */ 53 | /* Function returns a pointer to the compressed data, or NULL on failure. */ 54 | /* *pOut_len will be set to the compressed data's size, which could be larger than src_buf_len on uncompressible data. */ 55 | /* The caller must free() the returned block when it's no longer needed. */ 56 | MINIZ_EXPORT void *tdefl_compress_mem_to_heap(const void *pSrc_buf, size_t src_buf_len, size_t *pOut_len, int flags); 57 | 58 | /* tdefl_compress_mem_to_mem() compresses a block in memory to another block in memory. */ 59 | /* Returns 0 on failure. */ 60 | MINIZ_EXPORT size_t tdefl_compress_mem_to_mem(void *pOut_buf, size_t out_buf_len, const void *pSrc_buf, size_t src_buf_len, int flags); 61 | 62 | /* Compresses an image to a compressed PNG file in memory. */ 63 | /* On entry: */ 64 | /* pImage, w, h, and num_chans describe the image to compress. num_chans may be 1, 2, 3, or 4. */ 65 | /* The image pitch in bytes per scanline will be w*num_chans. The leftmost pixel on the top scanline is stored first in memory. */ 66 | /* level may range from [0,10], use MZ_NO_COMPRESSION, MZ_BEST_SPEED, MZ_BEST_COMPRESSION, etc. or a decent default is MZ_DEFAULT_LEVEL */ 67 | /* If flip is true, the image will be flipped on the Y axis (useful for OpenGL apps). */ 68 | /* On return: */ 69 | /* Function returns a pointer to the compressed data, or NULL on failure. */ 70 | /* *pLen_out will be set to the size of the PNG image file. */ 71 | /* The caller must mz_free() the returned heap block (which will typically be larger than *pLen_out) when it's no longer needed. */ 72 | MINIZ_EXPORT void *tdefl_write_image_to_png_file_in_memory_ex(const void *pImage, int w, int h, int num_chans, size_t *pLen_out, mz_uint level, mz_bool flip); 73 | MINIZ_EXPORT void *tdefl_write_image_to_png_file_in_memory(const void *pImage, int w, int h, int num_chans, size_t *pLen_out); 74 | 75 | /* Output stream interface. The compressor uses this interface to write compressed data. It'll typically be called TDEFL_OUT_BUF_SIZE at a time. */ 76 | typedef mz_bool (*tdefl_put_buf_func_ptr)(const void *pBuf, int len, void *pUser); 77 | 78 | /* tdefl_compress_mem_to_output() compresses a block to an output stream. The above helpers use this function internally. */ 79 | MINIZ_EXPORT mz_bool tdefl_compress_mem_to_output(const void *pBuf, size_t buf_len, tdefl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags); 80 | 81 | enum 82 | { 83 | TDEFL_MAX_HUFF_TABLES = 3, 84 | TDEFL_MAX_HUFF_SYMBOLS_0 = 288, 85 | TDEFL_MAX_HUFF_SYMBOLS_1 = 32, 86 | TDEFL_MAX_HUFF_SYMBOLS_2 = 19, 87 | TDEFL_LZ_DICT_SIZE = 32768, 88 | TDEFL_LZ_DICT_SIZE_MASK = TDEFL_LZ_DICT_SIZE - 1, 89 | TDEFL_MIN_MATCH_LEN = 3, 90 | TDEFL_MAX_MATCH_LEN = 258 91 | }; 92 | 93 | /* TDEFL_OUT_BUF_SIZE MUST be large enough to hold a single entire compressed output block (using static/fixed Huffman codes). */ 94 | #if TDEFL_LESS_MEMORY 95 | enum 96 | { 97 | TDEFL_LZ_CODE_BUF_SIZE = 24 * 1024, 98 | TDEFL_OUT_BUF_SIZE = (TDEFL_LZ_CODE_BUF_SIZE * 13) / 10, 99 | TDEFL_MAX_HUFF_SYMBOLS = 288, 100 | TDEFL_LZ_HASH_BITS = 12, 101 | TDEFL_LEVEL1_HASH_SIZE_MASK = 4095, 102 | TDEFL_LZ_HASH_SHIFT = (TDEFL_LZ_HASH_BITS + 2) / 3, 103 | TDEFL_LZ_HASH_SIZE = 1 << TDEFL_LZ_HASH_BITS 104 | }; 105 | #else 106 | enum 107 | { 108 | TDEFL_LZ_CODE_BUF_SIZE = 64 * 1024, 109 | TDEFL_OUT_BUF_SIZE = (mz_uint)((TDEFL_LZ_CODE_BUF_SIZE * 13) / 10), 110 | TDEFL_MAX_HUFF_SYMBOLS = 288, 111 | TDEFL_LZ_HASH_BITS = 15, 112 | TDEFL_LEVEL1_HASH_SIZE_MASK = 4095, 113 | TDEFL_LZ_HASH_SHIFT = (TDEFL_LZ_HASH_BITS + 2) / 3, 114 | TDEFL_LZ_HASH_SIZE = 1 << TDEFL_LZ_HASH_BITS 115 | }; 116 | #endif 117 | 118 | /* The low-level tdefl functions below may be used directly if the above helper functions aren't flexible enough. The low-level functions don't make any heap allocations, unlike the above helper functions. */ 119 | typedef enum 120 | { 121 | TDEFL_STATUS_BAD_PARAM = -2, 122 | TDEFL_STATUS_PUT_BUF_FAILED = -1, 123 | TDEFL_STATUS_OKAY = 0, 124 | TDEFL_STATUS_DONE = 1 125 | } tdefl_status; 126 | 127 | /* Must map to MZ_NO_FLUSH, MZ_SYNC_FLUSH, etc. enums */ 128 | typedef enum 129 | { 130 | TDEFL_NO_FLUSH = 0, 131 | TDEFL_SYNC_FLUSH = 2, 132 | TDEFL_FULL_FLUSH = 3, 133 | TDEFL_FINISH = 4 134 | } tdefl_flush; 135 | 136 | /* tdefl's compression state structure. */ 137 | typedef struct 138 | { 139 | tdefl_put_buf_func_ptr m_pPut_buf_func; 140 | void *m_pPut_buf_user; 141 | mz_uint m_flags, m_max_probes[2]; 142 | int m_greedy_parsing; 143 | mz_uint m_adler32, m_lookahead_pos, m_lookahead_size, m_dict_size; 144 | mz_uint8 *m_pLZ_code_buf, *m_pLZ_flags, *m_pOutput_buf, *m_pOutput_buf_end; 145 | mz_uint m_num_flags_left, m_total_lz_bytes, m_lz_code_buf_dict_pos, m_bits_in, m_bit_buffer; 146 | mz_uint m_saved_match_dist, m_saved_match_len, m_saved_lit, m_output_flush_ofs, m_output_flush_remaining, m_finished, m_block_index, m_wants_to_finish; 147 | tdefl_status m_prev_return_status; 148 | const void *m_pIn_buf; 149 | void *m_pOut_buf; 150 | size_t *m_pIn_buf_size, *m_pOut_buf_size; 151 | tdefl_flush m_flush; 152 | const mz_uint8 *m_pSrc; 153 | size_t m_src_buf_left, m_out_buf_ofs; 154 | mz_uint8 m_dict[TDEFL_LZ_DICT_SIZE + TDEFL_MAX_MATCH_LEN - 1]; 155 | mz_uint16 m_huff_count[TDEFL_MAX_HUFF_TABLES][TDEFL_MAX_HUFF_SYMBOLS]; 156 | mz_uint16 m_huff_codes[TDEFL_MAX_HUFF_TABLES][TDEFL_MAX_HUFF_SYMBOLS]; 157 | mz_uint8 m_huff_code_sizes[TDEFL_MAX_HUFF_TABLES][TDEFL_MAX_HUFF_SYMBOLS]; 158 | mz_uint8 m_lz_code_buf[TDEFL_LZ_CODE_BUF_SIZE]; 159 | mz_uint16 m_next[TDEFL_LZ_DICT_SIZE]; 160 | mz_uint16 m_hash[TDEFL_LZ_HASH_SIZE]; 161 | mz_uint8 m_output_buf[TDEFL_OUT_BUF_SIZE]; 162 | } tdefl_compressor; 163 | 164 | /* Initializes the compressor. */ 165 | /* There is no corresponding deinit() function because the tdefl API's do not dynamically allocate memory. */ 166 | /* pBut_buf_func: If NULL, output data will be supplied to the specified callback. In this case, the user should call the tdefl_compress_buffer() API for compression. */ 167 | /* If pBut_buf_func is NULL the user should always call the tdefl_compress() API. */ 168 | /* flags: See the above enums (TDEFL_HUFFMAN_ONLY, TDEFL_WRITE_ZLIB_HEADER, etc.) */ 169 | MINIZ_EXPORT tdefl_status tdefl_init(tdefl_compressor *d, tdefl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags); 170 | 171 | /* Compresses a block of data, consuming as much of the specified input buffer as possible, and writing as much compressed data to the specified output buffer as possible. */ 172 | MINIZ_EXPORT tdefl_status tdefl_compress(tdefl_compressor *d, const void *pIn_buf, size_t *pIn_buf_size, void *pOut_buf, size_t *pOut_buf_size, tdefl_flush flush); 173 | 174 | /* tdefl_compress_buffer() is only usable when the tdefl_init() is called with a non-NULL tdefl_put_buf_func_ptr. */ 175 | /* tdefl_compress_buffer() always consumes the entire input buffer. */ 176 | MINIZ_EXPORT tdefl_status tdefl_compress_buffer(tdefl_compressor *d, const void *pIn_buf, size_t in_buf_size, tdefl_flush flush); 177 | 178 | MINIZ_EXPORT tdefl_status tdefl_get_prev_return_status(tdefl_compressor *d); 179 | MINIZ_EXPORT mz_uint32 tdefl_get_adler32(tdefl_compressor *d); 180 | 181 | /* Create tdefl_compress() flags given zlib-style compression parameters. */ 182 | /* level may range from [0,10] (where 10 is absolute max compression, but may be much slower on some files) */ 183 | /* window_bits may be -15 (raw deflate) or 15 (zlib) */ 184 | /* strategy may be either MZ_DEFAULT_STRATEGY, MZ_FILTERED, MZ_HUFFMAN_ONLY, MZ_RLE, or MZ_FIXED */ 185 | MINIZ_EXPORT mz_uint tdefl_create_comp_flags_from_zip_params(int level, int window_bits, int strategy); 186 | 187 | #ifndef MINIZ_NO_MALLOC 188 | /* Allocate the tdefl_compressor structure in C so that */ 189 | /* non-C language bindings to tdefl_ API don't need to worry about */ 190 | /* structure size and allocation mechanism. */ 191 | MINIZ_EXPORT tdefl_compressor *tdefl_compressor_alloc(void); 192 | MINIZ_EXPORT void tdefl_compressor_free(tdefl_compressor *pComp); 193 | #endif 194 | 195 | #ifdef __cplusplus 196 | } 197 | #endif 198 | 199 | #endif /*#ifndef MINIZ_NO_DEFLATE_APIS*/ 200 | -------------------------------------------------------------------------------- /Source/ThirdParty/miniz/miniz_tinfl.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "miniz_common.h" 3 | /* ------------------- Low-level Decompression API Definitions */ 4 | 5 | #ifndef MINIZ_NO_INFLATE_APIS 6 | 7 | #ifdef __cplusplus 8 | extern "C" 9 | { 10 | #endif 11 | /* Decompression flags used by tinfl_decompress(). */ 12 | /* TINFL_FLAG_PARSE_ZLIB_HEADER: If set, the input has a valid zlib header and ends with an adler32 checksum (it's a valid zlib stream). Otherwise, the input is a raw deflate stream. */ 13 | /* TINFL_FLAG_HAS_MORE_INPUT: If set, there are more input bytes available beyond the end of the supplied input buffer. If clear, the input buffer contains all remaining input. */ 14 | /* TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF: If set, the output buffer is large enough to hold the entire decompressed stream. If clear, the output buffer is at least the size of the dictionary (typically 32KB). */ 15 | /* TINFL_FLAG_COMPUTE_ADLER32: Force adler-32 checksum computation of the decompressed bytes. */ 16 | enum 17 | { 18 | TINFL_FLAG_PARSE_ZLIB_HEADER = 1, 19 | TINFL_FLAG_HAS_MORE_INPUT = 2, 20 | TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF = 4, 21 | TINFL_FLAG_COMPUTE_ADLER32 = 8 22 | }; 23 | 24 | /* High level decompression functions: */ 25 | /* tinfl_decompress_mem_to_heap() decompresses a block in memory to a heap block allocated via malloc(). */ 26 | /* On entry: */ 27 | /* pSrc_buf, src_buf_len: Pointer and size of the Deflate or zlib source data to decompress. */ 28 | /* On return: */ 29 | /* Function returns a pointer to the decompressed data, or NULL on failure. */ 30 | /* *pOut_len will be set to the decompressed data's size, which could be larger than src_buf_len on uncompressible data. */ 31 | /* The caller must call mz_free() on the returned block when it's no longer needed. */ 32 | MINIZ_EXPORT void *tinfl_decompress_mem_to_heap(const void *pSrc_buf, size_t src_buf_len, size_t *pOut_len, int flags); 33 | 34 | /* tinfl_decompress_mem_to_mem() decompresses a block in memory to another block in memory. */ 35 | /* Returns TINFL_DECOMPRESS_MEM_TO_MEM_FAILED on failure, or the number of bytes written on success. */ 36 | #define TINFL_DECOMPRESS_MEM_TO_MEM_FAILED ((size_t)(-1)) 37 | MINIZ_EXPORT size_t tinfl_decompress_mem_to_mem(void *pOut_buf, size_t out_buf_len, const void *pSrc_buf, size_t src_buf_len, int flags); 38 | 39 | /* tinfl_decompress_mem_to_callback() decompresses a block in memory to an internal 32KB buffer, and a user provided callback function will be called to flush the buffer. */ 40 | /* Returns 1 on success or 0 on failure. */ 41 | typedef int (*tinfl_put_buf_func_ptr)(const void *pBuf, int len, void *pUser); 42 | MINIZ_EXPORT int tinfl_decompress_mem_to_callback(const void *pIn_buf, size_t *pIn_buf_size, tinfl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags); 43 | 44 | struct tinfl_decompressor_tag; 45 | typedef struct tinfl_decompressor_tag tinfl_decompressor; 46 | 47 | #ifndef MINIZ_NO_MALLOC 48 | /* Allocate the tinfl_decompressor structure in C so that */ 49 | /* non-C language bindings to tinfl_ API don't need to worry about */ 50 | /* structure size and allocation mechanism. */ 51 | MINIZ_EXPORT tinfl_decompressor *tinfl_decompressor_alloc(void); 52 | MINIZ_EXPORT void tinfl_decompressor_free(tinfl_decompressor *pDecomp); 53 | #endif 54 | 55 | /* Max size of LZ dictionary. */ 56 | #define TINFL_LZ_DICT_SIZE 32768 57 | 58 | /* Return status. */ 59 | typedef enum 60 | { 61 | /* This flags indicates the inflator needs 1 or more input bytes to make forward progress, but the caller is indicating that no more are available. The compressed data */ 62 | /* is probably corrupted. If you call the inflator again with more bytes it'll try to continue processing the input but this is a BAD sign (either the data is corrupted or you called it incorrectly). */ 63 | /* If you call it again with no input you'll just get TINFL_STATUS_FAILED_CANNOT_MAKE_PROGRESS again. */ 64 | TINFL_STATUS_FAILED_CANNOT_MAKE_PROGRESS = -4, 65 | 66 | /* This flag indicates that one or more of the input parameters was obviously bogus. (You can try calling it again, but if you get this error the calling code is wrong.) */ 67 | TINFL_STATUS_BAD_PARAM = -3, 68 | 69 | /* This flags indicate the inflator is finished but the adler32 check of the uncompressed data didn't match. If you call it again it'll return TINFL_STATUS_DONE. */ 70 | TINFL_STATUS_ADLER32_MISMATCH = -2, 71 | 72 | /* This flags indicate the inflator has somehow failed (bad code, corrupted input, etc.). If you call it again without resetting via tinfl_init() it it'll just keep on returning the same status failure code. */ 73 | TINFL_STATUS_FAILED = -1, 74 | 75 | /* Any status code less than TINFL_STATUS_DONE must indicate a failure. */ 76 | 77 | /* This flag indicates the inflator has returned every byte of uncompressed data that it can, has consumed every byte that it needed, has successfully reached the end of the deflate stream, and */ 78 | /* if zlib headers and adler32 checking enabled that it has successfully checked the uncompressed data's adler32. If you call it again you'll just get TINFL_STATUS_DONE over and over again. */ 79 | TINFL_STATUS_DONE = 0, 80 | 81 | /* This flag indicates the inflator MUST have more input data (even 1 byte) before it can make any more forward progress, or you need to clear the TINFL_FLAG_HAS_MORE_INPUT */ 82 | /* flag on the next call if you don't have any more source data. If the source data was somehow corrupted it's also possible (but unlikely) for the inflator to keep on demanding input to */ 83 | /* proceed, so be sure to properly set the TINFL_FLAG_HAS_MORE_INPUT flag. */ 84 | TINFL_STATUS_NEEDS_MORE_INPUT = 1, 85 | 86 | /* This flag indicates the inflator definitely has 1 or more bytes of uncompressed data available, but it cannot write this data into the output buffer. */ 87 | /* Note if the source compressed data was corrupted it's possible for the inflator to return a lot of uncompressed data to the caller. I've been assuming you know how much uncompressed data to expect */ 88 | /* (either exact or worst case) and will stop calling the inflator and fail after receiving too much. In pure streaming scenarios where you have no idea how many bytes to expect this may not be possible */ 89 | /* so I may need to add some code to address this. */ 90 | TINFL_STATUS_HAS_MORE_OUTPUT = 2 91 | } tinfl_status; 92 | 93 | /* Initializes the decompressor to its initial state. */ 94 | #define tinfl_init(r) \ 95 | do \ 96 | { \ 97 | (r)->m_state = 0; \ 98 | } \ 99 | MZ_MACRO_END 100 | #define tinfl_get_adler32(r) (r)->m_check_adler32 101 | 102 | /* Main low-level decompressor coroutine function. This is the only function actually needed for decompression. All the other functions are just high-level helpers for improved usability. */ 103 | /* This is a universal API, i.e. it can be used as a building block to build any desired higher level decompression API. In the limit case, it can be called once per every byte input or output. */ 104 | MINIZ_EXPORT tinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_next, size_t *pIn_buf_size, mz_uint8 *pOut_buf_start, mz_uint8 *pOut_buf_next, size_t *pOut_buf_size, const mz_uint32 decomp_flags); 105 | 106 | /* Internal/private bits follow. */ 107 | enum 108 | { 109 | TINFL_MAX_HUFF_TABLES = 3, 110 | TINFL_MAX_HUFF_SYMBOLS_0 = 288, 111 | TINFL_MAX_HUFF_SYMBOLS_1 = 32, 112 | TINFL_MAX_HUFF_SYMBOLS_2 = 19, 113 | TINFL_FAST_LOOKUP_BITS = 10, 114 | TINFL_FAST_LOOKUP_SIZE = 1 << TINFL_FAST_LOOKUP_BITS 115 | }; 116 | 117 | #if MINIZ_HAS_64BIT_REGISTERS 118 | #define TINFL_USE_64BIT_BITBUF 1 119 | #else 120 | #define TINFL_USE_64BIT_BITBUF 0 121 | #endif 122 | 123 | #if TINFL_USE_64BIT_BITBUF 124 | typedef mz_uint64 tinfl_bit_buf_t; 125 | #define TINFL_BITBUF_SIZE (64) 126 | #else 127 | typedef mz_uint32 tinfl_bit_buf_t; 128 | #define TINFL_BITBUF_SIZE (32) 129 | #endif 130 | 131 | struct tinfl_decompressor_tag 132 | { 133 | mz_uint32 m_state, m_num_bits, m_zhdr0, m_zhdr1, m_z_adler32, m_final, m_type, m_check_adler32, m_dist, m_counter, m_num_extra, m_table_sizes[TINFL_MAX_HUFF_TABLES]; 134 | tinfl_bit_buf_t m_bit_buf; 135 | size_t m_dist_from_out_buf_start; 136 | mz_int16 m_look_up[TINFL_MAX_HUFF_TABLES][TINFL_FAST_LOOKUP_SIZE]; 137 | mz_int16 m_tree_0[TINFL_MAX_HUFF_SYMBOLS_0 * 2]; 138 | mz_int16 m_tree_1[TINFL_MAX_HUFF_SYMBOLS_1 * 2]; 139 | mz_int16 m_tree_2[TINFL_MAX_HUFF_SYMBOLS_2 * 2]; 140 | mz_uint8 m_code_size_0[TINFL_MAX_HUFF_SYMBOLS_0]; 141 | mz_uint8 m_code_size_1[TINFL_MAX_HUFF_SYMBOLS_1]; 142 | mz_uint8 m_code_size_2[TINFL_MAX_HUFF_SYMBOLS_2]; 143 | mz_uint8 m_raw_header[4], m_len_codes[TINFL_MAX_HUFF_SYMBOLS_0 + TINFL_MAX_HUFF_SYMBOLS_1 + 137]; 144 | }; 145 | 146 | #ifdef __cplusplus 147 | } 148 | #endif 149 | 150 | #endif /*#ifndef MINIZ_NO_INFLATE_APIS*/ 151 | --------------------------------------------------------------------------------