├── README.md ├── makefile ├── LICENSE ├── src ├── showsheets.vcxproj.filters ├── showsheets.cpp ├── pole.h ├── showsheets.vcxproj └── pole.cpp ├── showsheets.sln └── .gitignore /README.md: -------------------------------------------------------------------------------- 1 | # ShowSheets 2 | This is a very simple command-line app to set all EXCEL sheets to visible. -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | #~/user/bin/make -f 2 | EXECUTABLE = showsheets 3 | SO_LIBRARY = showsheets.so 4 | 5 | CC = gcc 6 | CXX = g++ 7 | CFLAGS = -fPIC -Wno-enum-conversion -O3 8 | CXXFLAGS = -fPIC -std=c++11 -O3 -Wfatal-errors -Werror 9 | LDFLAGS = -pthread 10 | 11 | SRC_SHOWSHEETS = \ 12 | src/showsheets.cpp \ 13 | src/pole.cpp 14 | 15 | 16 | # Object files 17 | OBJS = \ 18 | $(SRC_SHOWSHEETS:.cpp=.o) 19 | 20 | # Rules 21 | all: $(EXECUTABLE) $(SO_LIBRARY) 22 | 23 | $(EXECUTABLE): $(OBJS) 24 | $(CXX) $(CXXFLAGS) $(LDFLAGS) $(CPP_FILES) -o $@ $^ 25 | 26 | $(SO_LIBRARY): $(OBJS) 27 | $(CXX) $(LDFLAGS) -shared -o $@ $^ 28 | 29 | %.o: %.cpp 30 | $(CXX) $(CXXFLAGS) -c $< -o $@ 31 | 32 | %.o: %.c 33 | $(CC) $(CFLAGS) -c $< -o $@ 34 | 35 | clean: 36 | rm -rf `find Source -name '*.o'` $(EXECUTABLE) $(SO_LIBRARY) 37 | 38 | 39 | BINDIR ?= ${PREFIX}/bin 40 | 41 | install: 42 | @cp -p bin/showsheets ${PREFIX}/showsheets 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | Copyright (c) 2019 David B. Heise 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 5 | 6 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 7 | 8 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /src/showsheets.vcxproj.filters: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;hm;inl;inc;ipp;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | Source Files 20 | 21 | 22 | Source Files 23 | 24 | 25 | 26 | 27 | Header Files 28 | 29 | 30 | -------------------------------------------------------------------------------- /showsheets.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 16 4 | VisualStudioVersion = 16.0.29424.173 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "showsheets", "src\showsheets.vcxproj", "{936802B4-1D28-4638-8147-F7FEEE38149E}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|x64 = Debug|x64 11 | Debug|x86 = Debug|x86 12 | Release|x64 = Release|x64 13 | Release|x86 = Release|x86 14 | EndGlobalSection 15 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 16 | {936802B4-1D28-4638-8147-F7FEEE38149E}.Debug|x64.ActiveCfg = Debug|x64 17 | {936802B4-1D28-4638-8147-F7FEEE38149E}.Debug|x64.Build.0 = Debug|x64 18 | {936802B4-1D28-4638-8147-F7FEEE38149E}.Debug|x86.ActiveCfg = Debug|Win32 19 | {936802B4-1D28-4638-8147-F7FEEE38149E}.Debug|x86.Build.0 = Debug|Win32 20 | {936802B4-1D28-4638-8147-F7FEEE38149E}.Release|x64.ActiveCfg = Release|x64 21 | {936802B4-1D28-4638-8147-F7FEEE38149E}.Release|x64.Build.0 = Release|x64 22 | {936802B4-1D28-4638-8147-F7FEEE38149E}.Release|x86.ActiveCfg = Release|Win32 23 | {936802B4-1D28-4638-8147-F7FEEE38149E}.Release|x86.Build.0 = Release|Win32 24 | EndGlobalSection 25 | GlobalSection(SolutionProperties) = preSolution 26 | HideSolutionNode = FALSE 27 | EndGlobalSection 28 | GlobalSection(ExtensibilityGlobals) = postSolution 29 | SolutionGuid = {4C9D4007-1A87-4137-9FBA-3B1A15BB0853} 30 | EndGlobalSection 31 | EndGlobal 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # ---> VisualStudio 2 | ## Ignore Visual Studio temporary files, build results, and 3 | ## files generated by popular Visual Studio add-ons. 4 | 5 | # User-specific files 6 | *.suo 7 | *.user 8 | *.userosscache 9 | *.sln.docstates 10 | 11 | # User-specific files (MonoDevelop/Xamarin Studio) 12 | *.userprefs 13 | 14 | # Build results 15 | [Dd]ebug/ 16 | [Dd]ebugPublic/ 17 | [Rr]elease/ 18 | [Rr]eleases/ 19 | x64/ 20 | x86/ 21 | build/ 22 | bld/ 23 | [Bb]in/ 24 | [Oo]bj/ 25 | 26 | # Visual Studio 2015 cache/options directory 27 | .vs/ 28 | # Uncomment if you have tasks that create the project's static files in wwwroot 29 | #wwwroot/ 30 | 31 | # MSTest test Results 32 | [Tt]est[Rr]esult*/ 33 | [Bb]uild[Ll]og.* 34 | 35 | # NUNIT 36 | *.VisualState.xml 37 | TestResult.xml 38 | 39 | # Build Results of an ATL Project 40 | [Dd]ebugPS/ 41 | [Rr]eleasePS/ 42 | dlldata.c 43 | 44 | # DNX 45 | project.lock.json 46 | artifacts/ 47 | 48 | *_i.c 49 | *_p.c 50 | *_i.h 51 | *.ilk 52 | *.meta 53 | *.obj 54 | *.pch 55 | *.pdb 56 | *.pgc 57 | *.pgd 58 | *.rsp 59 | *.sbr 60 | *.tlb 61 | *.tli 62 | *.tlh 63 | *.tmp 64 | *.tmp_proj 65 | *.log 66 | *.vspscc 67 | *.vssscc 68 | .builds 69 | *.pidb 70 | *.svclog 71 | *.scc 72 | 73 | # Chutzpah Test files 74 | _Chutzpah* 75 | 76 | # Visual C++ cache files 77 | ipch/ 78 | *.aps 79 | *.ncb 80 | *.opensdf 81 | *.sdf 82 | *.cachefile 83 | 84 | # Visual Studio profiler 85 | *.psess 86 | *.vsp 87 | *.vspx 88 | *.sap 89 | 90 | # TFS 2012 Local Workspace 91 | $tf/ 92 | 93 | # Guidance Automation Toolkit 94 | *.gpState 95 | 96 | # ReSharper is a .NET coding add-in 97 | _ReSharper*/ 98 | *.[Rr]e[Ss]harper 99 | *.DotSettings.user 100 | 101 | # JustCode is a .NET coding add-in 102 | .JustCode 103 | 104 | # TeamCity is a build add-in 105 | _TeamCity* 106 | 107 | # DotCover is a Code Coverage Tool 108 | *.dotCover 109 | 110 | # NCrunch 111 | _NCrunch_* 112 | .*crunch*.local.xml 113 | nCrunchTemp_* 114 | 115 | # MightyMoose 116 | *.mm.* 117 | AutoTest.Net/ 118 | 119 | # Web workbench (sass) 120 | .sass-cache/ 121 | 122 | # Installshield output folder 123 | [Ee]xpress/ 124 | 125 | # DocProject is a documentation generator add-in 126 | DocProject/buildhelp/ 127 | DocProject/Help/*.HxT 128 | DocProject/Help/*.HxC 129 | DocProject/Help/*.hhc 130 | DocProject/Help/*.hhk 131 | DocProject/Help/*.hhp 132 | DocProject/Help/Html2 133 | DocProject/Help/html 134 | 135 | # Click-Once directory 136 | publish/ 137 | 138 | # Publish Web Output 139 | *.[Pp]ublish.xml 140 | *.azurePubxml 141 | # TODO: Comment the next line if you want to checkin your web deploy settings 142 | # but database connection strings (with potential passwords) will be unencrypted 143 | *.pubxml 144 | *.publishproj 145 | 146 | # NuGet Packages 147 | *.nupkg 148 | # The packages folder can be ignored because of Package Restore 149 | **/packages/* 150 | # except build/, which is used as an MSBuild target. 151 | !**/packages/build/ 152 | # Uncomment if necessary however generally it will be regenerated when needed 153 | #!**/packages/repositories.config 154 | 155 | # Windows Azure Build Output 156 | csx/ 157 | *.build.csdef 158 | 159 | # Windows Store app package directory 160 | AppPackages/ 161 | 162 | # Visual Studio cache files 163 | # files ending in .cache can be ignored 164 | *.[Cc]ache 165 | # but keep track of directories ending in .cache 166 | !*.[Cc]ache/ 167 | 168 | # Others 169 | ClientBin/ 170 | [Ss]tyle[Cc]op.* 171 | ~$* 172 | *~ 173 | *.dbmdl 174 | *.dbproj.schemaview 175 | *.pfx 176 | *.publishsettings 177 | node_modules/ 178 | orleans.codegen.cs 179 | 180 | # RIA/Silverlight projects 181 | Generated_Code/ 182 | 183 | # Backup & report files from converting an old project file 184 | # to a newer Visual Studio version. Backup files are not needed, 185 | # because we have git ;-) 186 | _UpgradeReport_Files/ 187 | Backup*/ 188 | UpgradeLog*.XML 189 | UpgradeLog*.htm 190 | 191 | # SQL Server files 192 | *.mdf 193 | *.ldf 194 | 195 | # Business Intelligence projects 196 | *.rdl.data 197 | *.bim.layout 198 | *.bim_*.settings 199 | 200 | # Microsoft Fakes 201 | FakesAssemblies/ 202 | 203 | # Node.js Tools for Visual Studio 204 | .ntvs_analysis.dat 205 | 206 | # Visual Studio 6 build log 207 | *.plg 208 | 209 | # Visual Studio 6 workspace options file 210 | *.opt 211 | 212 | # Visual Studio LightSwitch build output 213 | **/*.HTMLClient/GeneratedArtifacts 214 | **/*.DesktopClient/GeneratedArtifacts 215 | **/*.DesktopClient/ModelManifest.xml 216 | **/*.Server/GeneratedArtifacts 217 | **/*.Server/ModelManifest.xml 218 | _Pvt_Extensions 219 | 220 | # ---> VisualStudioCode 221 | .settings 222 | 223 | -------------------------------------------------------------------------------- /src/showsheets.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "pole.h" 9 | 10 | 11 | void showUsage() { 12 | std::cout << "showsheets {file}" << std::endl; 13 | } 14 | 15 | enum class fileType : unsigned char { 16 | OLESS, 17 | ZIP, 18 | Other 19 | }; 20 | 21 | fileType ReadFileType(std::string file) { 22 | std::ifstream stream(file, std::ios::binary | std::ios::ate); 23 | if (stream.is_open()) { 24 | char buffer[4]; 25 | stream.seekg(0, std::ios::beg); 26 | stream.read(buffer, 4); 27 | stream.close(); 28 | 29 | if (((unsigned char)buffer[0]) == 0xD0 && ((unsigned char)buffer[1]) == 0xCF && ((unsigned char)buffer[2]) == 0x11 && ((unsigned char)buffer[3]) == 0xE0) { 30 | return fileType::OLESS; 31 | } else if (((unsigned char)buffer[0]) == 0x50 && ((unsigned char)buffer[1]) == 0x4B) { 32 | return fileType::ZIP; 33 | } 34 | } 35 | return fileType::Other; 36 | } 37 | 38 | struct RecordHeader { 39 | unsigned short Type; 40 | unsigned short Length; 41 | }; 42 | 43 | // see: https://docs.microsoft.com/en-us/openspecs/office_file_formats/ms-xls/b9ec509a-235d-424e-871d-f8e721106501 44 | struct BoundSheetHeader { 45 | unsigned int lbPlyPos; 46 | unsigned short hsState : 2; 47 | unsigned short unused1 : 6; 48 | unsigned short dt : 8; 49 | }; 50 | // see: https://docs.microsoft.com/en-us/openspecs/office_file_formats/ms-xls/05162858-0ca9-44cb-bb07-a720928f63f8 51 | struct ShortXLUnicodeString { 52 | unsigned char cch; 53 | unsigned short fHighByte : 1; 54 | unsigned short reserved1 : 7; 55 | std::string rgb; 56 | 57 | static ShortXLUnicodeString Read(const unsigned char* buffer, const unsigned int offset) { 58 | 59 | ShortXLUnicodeString ans; 60 | unsigned int index = offset; 61 | 62 | ans.cch = buffer[index]; 63 | index += 1; 64 | 65 | ans.fHighByte = buffer[index] & 128; 66 | ans.reserved1 = buffer[index] & 127; 67 | index++; 68 | 69 | int byteCount = 0; 70 | if (ans.fHighByte == 0x0) { 71 | byteCount = ans.cch; 72 | } 73 | else { 74 | byteCount = ans.cch * 2; 75 | } 76 | 77 | std::string name(reinterpret_cast(buffer + index), byteCount); 78 | ans.rgb = name; 79 | 80 | return ans; 81 | } 82 | }; 83 | 84 | std::string GetVisibilityStr(unsigned short visibility) { 85 | std::string ans; 86 | switch (visibility) { 87 | case 0x00: ans = "Visible"; break; 88 | case 0x01: ans = "Hidden"; break; 89 | case 0x02: ans = "Very Hidden"; break; 90 | default: ans = "Unknown/Undocumented visiblitiy"; break; 91 | } 92 | return ans; 93 | } 94 | 95 | void makeSheetsVisible(std::string xlsFile) { 96 | POLE::Storage* storage = new POLE::Storage(xlsFile.c_str()); 97 | storage->open(true, false); 98 | if (storage->result() == POLE::Storage::Ok) { 99 | 100 | std::list entries; 101 | entries = storage->entries("/"); 102 | std::list::iterator it; 103 | for (it = entries.begin(); it != entries.end(); it++) { 104 | std::string name = *it; 105 | if (name == "Workbook" || name == "Book" || name == "WorkBook") { 106 | POLE::Stream* stream = new POLE::Stream(storage, "/" + name); 107 | 108 | unsigned char recordHeader[4]; 109 | 110 | while (! stream->eof()) { 111 | auto bytesRead = stream->read(recordHeader, 4); 112 | if (bytesRead != 4) { 113 | break; 114 | } 115 | else { 116 | auto header = reinterpret_cast(recordHeader); 117 | 118 | unsigned char* block = new unsigned char[header->Length]; 119 | bytesRead = stream->read(block, header->Length); 120 | 121 | if (bytesRead == header->Length) { 122 | 123 | if (header->Type == 0x0085) { //BoundSheet 124 | 125 | auto bs = reinterpret_cast(block); 126 | auto name = ShortXLUnicodeString::Read(block, 6); 127 | std::cout << "Sheet: \"" << name.rgb << "\", Visiblity: \"" << GetVisibilityStr(bs->hsState) << "\"" << std::endl; 128 | if (bs->hsState != 0) { 129 | bs->hsState = 0; 130 | 131 | auto pos = stream->tell(); 132 | stream->seek(pos-(header->Length)); 133 | 134 | auto bytesWritten = stream->write(block, 6); 135 | if (bytesWritten != 6) { 136 | std::cout << "Could not write to stream!" << std::endl; 137 | } 138 | 139 | stream->seek(pos); 140 | } 141 | } 142 | } 143 | delete[] block; 144 | } 145 | } 146 | stream->flush(); 147 | delete stream; 148 | } 149 | } 150 | } 151 | else { 152 | std::cerr << "Could not open file as OLESS file for writing" << std::endl; 153 | } 154 | 155 | storage->close(); 156 | delete storage; 157 | } 158 | 159 | 160 | int main(int argc, char* argv[]) 161 | { 162 | std::vector arguments(argv + 1, argv + argc); 163 | 164 | if (argc > 1) { 165 | auto file = arguments[0]; 166 | auto ftype = ReadFileType(file); 167 | 168 | if (ftype == fileType::OLESS) { 169 | makeSheetsVisible(file); 170 | } else if (ftype == fileType::ZIP) { 171 | std::cerr << "I HAVEN'T DONE THIS WORK YET!!" << std::endl; 172 | } else { 173 | std::cerr << "Unsupported file type" << std::endl; 174 | } 175 | } 176 | else { 177 | showUsage(); 178 | } 179 | return 0; 180 | } 181 | -------------------------------------------------------------------------------- /src/pole.h: -------------------------------------------------------------------------------- 1 | /* POLE - Portable C++ library to access OLE Storage 2 | Copyright (C) 2002-2005 Ariya Hidayat 3 | 4 | Performance optimization, API improvements: Dmitry Fedorov 5 | Copyright 2009-2014 6 | 7 | Fix for more than 236 mbat block entries : Michel Boudinot 8 | Copyright 2010 9 | 10 | Considerable rework to allow for creation and updating of structured storage: Stephen Baum 11 | Copyright 2013 12 | 13 | Added GetAllStreams, reworked datatypes 14 | Copyright 2013 Felix Gorny from Bitplane 15 | 16 | More datatype changes to allow for 32 and 64 bit code, some fixes involving incremental updates, flushing 17 | Copyright 2013 18 | 19 | Version: 0.5.3 20 | 21 | Redistribution and use in source and binary forms, with or without 22 | modification, are permitted provided that the following conditions 23 | are met: 24 | * Redistributions of source code must retain the above copyright notice, 25 | this list of conditions and the following disclaimer. 26 | * Redistributions in binary form must reproduce the above copyright notice, 27 | this list of conditions and the following disclaimer in the documentation 28 | and/or other materials provided with the distribution. 29 | * Neither the name of the authors nor the names of its contributors may be 30 | used to endorse or promote products derived from this software without 31 | specific prior written permission. 32 | 33 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 34 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 35 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 36 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 37 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 38 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 39 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 40 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 41 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 42 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 43 | THE POSSIBILITY OF SUCH DAMAGE. 44 | */ 45 | 46 | /* 47 | Unicode notes: 48 | 49 | Filenames are considered to be encoded in UTF-8 encoding. On windows they 50 | can be re-encoded into UTF16 and proper wchar_t APIs will be used to open files. 51 | This is a default behavior for windows and is defined by the macro POLE_USE_UTF16_FILENAMES. 52 | 53 | Using a provided function and a modern c++ compiler it's easy to encode a 54 | wide string into utf8 char*: 55 | std::string POLE::UTF16toUTF8(const std::wstring &utf16); 56 | */ 57 | 58 | #ifndef POLE_H 59 | #define POLE_H 60 | 61 | #include 62 | #include 63 | #include 64 | 65 | namespace POLE 66 | { 67 | 68 | #if defined WIN32 || defined WIN64 || defined _WIN32 || defined _WIN64 || defined _MSVC 69 | #define POLE_USE_UTF16_FILENAMES 70 | #define POLE_WIN 71 | typedef __int32 int32; 72 | typedef __int64 int64; 73 | typedef unsigned __int32 uint32; 74 | typedef unsigned __int64 uint64; 75 | #else 76 | typedef int int32; 77 | typedef long long int64; 78 | typedef unsigned int uint32; 79 | typedef unsigned long long uint64; 80 | #endif 81 | 82 | typedef uint64 t_offset; 83 | 84 | #ifdef POLE_USE_UTF16_FILENAMES 85 | std::string UTF16toUTF8(const std::wstring& utf16); 86 | std::wstring UTF8toUTF16(const std::string& utf8); 87 | #endif //POLE_USE_UTF16_FILENAMES 88 | 89 | class StorageIO; 90 | class Stream; 91 | class StreamIO; 92 | 93 | class Storage 94 | { 95 | friend class Stream; 96 | friend class StreamOut; 97 | 98 | public: 99 | 100 | // for Storage::result() 101 | enum { Ok, OpenFailed, NotOLE, BadOLE, UnknownError }; 102 | 103 | /** 104 | * Constructs a storage with name filename. 105 | **/ 106 | Storage(const char* filename); 107 | 108 | /** 109 | * Destroys the storage. 110 | **/ 111 | ~Storage(); 112 | 113 | /** 114 | * Opens the storage. Returns true if no error occurs. 115 | **/ 116 | bool open(bool bWriteAccess = false, bool bCreate = false); 117 | 118 | /** 119 | * Closes the storage. 120 | **/ 121 | void close(); 122 | 123 | /** 124 | * Returns the error code of last operation. 125 | **/ 126 | int result(); 127 | 128 | /** 129 | * Finds all stream and directories in given path. 130 | **/ 131 | std::list entries(const std::string& path = "/"); 132 | 133 | /** 134 | * Returns true if specified entry name is a directory. 135 | */ 136 | bool isDirectory(const std::string& name); 137 | 138 | /** 139 | * Returns true if specified entry name exists. 140 | */ 141 | bool exists(const std::string& name); 142 | 143 | /** 144 | * Returns true if storage can be modified. 145 | */ 146 | bool isWriteable(); 147 | 148 | /** 149 | * Deletes a specified stream or directory. If directory, it will 150 | * recursively delete everything underneath said directory. 151 | * returns true for success 152 | */ 153 | bool deleteByName(const std::string& name); 154 | 155 | /** 156 | * Returns an accumulation of information, hopefully useful for determining if the storage 157 | * should be defragmented. 158 | */ 159 | 160 | void GetStats(uint64* pEntries, uint64* pUnusedEntries, 161 | uint64* pBigBlocks, uint64* pUnusedBigBlocks, 162 | uint64* pSmallBlocks, uint64* pUnusedSmallBlocks); 163 | 164 | std::list GetAllStreams(const std::string& storageName); 165 | 166 | private: 167 | StorageIO* io; 168 | 169 | // no copy or assign 170 | Storage(const Storage&); 171 | Storage& operator=(const Storage&); 172 | 173 | }; 174 | 175 | class Stream 176 | { 177 | friend class Storage; 178 | friend class StorageIO; 179 | 180 | public: 181 | 182 | /** 183 | * Creates a new stream. 184 | */ 185 | // name must be absolute, e.g "/Workbook" 186 | Stream(Storage* storage, const std::string& name, bool bCreate = false, int64 streamSize = 0); 187 | 188 | /** 189 | * Destroys the stream. 190 | */ 191 | ~Stream(); 192 | 193 | /** 194 | * Returns the full stream name. 195 | */ 196 | std::string fullName(); 197 | 198 | /** 199 | * Returns the stream size. 200 | **/ 201 | uint64 size(); 202 | 203 | /** 204 | * Changes the stream size (note this is done automatically if you write beyond the old size. 205 | * Use this primarily as a preamble to rewriting a stream that is already open. Of course, you 206 | * could simply delete the stream first). 207 | **/ 208 | void setSize(int64 newSize); 209 | 210 | /** 211 | * Returns the current read/write position. 212 | **/ 213 | uint64 tell(); 214 | 215 | /** 216 | * Sets the read/write position. 217 | **/ 218 | void seek(uint64 pos); 219 | 220 | /** 221 | * Reads a byte. 222 | **/ 223 | int64 getch(); 224 | 225 | /** 226 | * Reads a block of data. 227 | **/ 228 | uint64 read(unsigned char* data, uint64 maxlen); 229 | 230 | /** 231 | * Writes a block of data. 232 | **/ 233 | uint64 write(unsigned char* data, uint64 len); 234 | 235 | /** 236 | * Makes sure that any changes for the stream (and the structured storage) have been written to disk. 237 | **/ 238 | void flush(); 239 | 240 | /** 241 | * Returns true if the read/write position is past the file. 242 | **/ 243 | bool eof(); 244 | 245 | /** 246 | * Returns true whenever error occurs. 247 | **/ 248 | bool fail(); 249 | 250 | private: 251 | StreamIO* io; 252 | 253 | // no copy or assign 254 | Stream(const Stream&); 255 | Stream& operator=(const Stream&); 256 | }; 257 | 258 | } 259 | 260 | #endif // POLE_H 261 | -------------------------------------------------------------------------------- /src/showsheets.vcxproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Debug 6 | Win32 7 | 8 | 9 | Release 10 | Win32 11 | 12 | 13 | Debug 14 | x64 15 | 16 | 17 | Release 18 | x64 19 | 20 | 21 | 22 | 16.0 23 | {936802B4-1D28-4638-8147-F7FEEE38149E} 24 | Win32Proj 25 | showsheets 26 | 10.0 27 | $(SolutionDir)Bin\$(Platform)\$(Configuration)\ 28 | $(SolutionDir)Obj\$(ProjectName)_$(PlatformShortName)_$(Configuration)\ 29 | 30 | 31 | 32 | Application 33 | true 34 | v142 35 | Unicode 36 | 37 | 38 | Application 39 | false 40 | v142 41 | true 42 | Unicode 43 | 44 | 45 | Application 46 | true 47 | v142 48 | Unicode 49 | 50 | 51 | Application 52 | false 53 | v142 54 | true 55 | Unicode 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | true 77 | 78 | 79 | true 80 | 81 | 82 | false 83 | 84 | 85 | false 86 | 87 | 88 | 89 | 90 | 91 | Level3 92 | Disabled 93 | true 94 | WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) 95 | true 96 | 97 | 98 | Console 99 | true 100 | 101 | 102 | 103 | 104 | 105 | 106 | Level3 107 | Disabled 108 | true 109 | _DEBUG;_CONSOLE;%(PreprocessorDefinitions) 110 | true 111 | 112 | 113 | Console 114 | true 115 | 116 | 117 | 118 | 119 | 120 | 121 | Level3 122 | MaxSpeed 123 | true 124 | true 125 | true 126 | WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) 127 | true 128 | 129 | 130 | Console 131 | true 132 | true 133 | true 134 | 135 | 136 | 137 | 138 | 139 | 140 | Level3 141 | MaxSpeed 142 | true 143 | true 144 | true 145 | NDEBUG;_CONSOLE;%(PreprocessorDefinitions) 146 | true 147 | 148 | 149 | Console 150 | true 151 | true 152 | true 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | -------------------------------------------------------------------------------- /src/pole.cpp: -------------------------------------------------------------------------------- 1 | /* POLE - Portable C++ library to access OLE Storage 2 | Copyright (C) 2002-2005 Ariya Hidayat 3 | 4 | Performance optimization: Dmitry Fedorov 5 | Copyright 2009 6 | 7 | Fix for more than 236 mbat block entries : Michel Boudinot 8 | Copyright 2010 9 | 10 | Considerable rework to allow for creation and updating of structured storage : Stephen Baum 11 | Copyright 2013 12 | 13 | Added GetAllStreams, reworked datatypes 14 | Copyright 2013 Felix Gorny from Bitplane 15 | 16 | More datatype changes to allow for 32 and 64 bit code, some fixes involving incremental updates, flushing 17 | Copyright 2013 18 | 19 | Version: 0.5.2 20 | 21 | Redistribution and use in source and binary forms, with or without 22 | modification, are permitted provided that the following conditions 23 | are met: 24 | * Redistributions of source code must retain the above copyright notice, 25 | this list of conditions and the following disclaimer. 26 | * Redistributions in binary form must reproduce the above copyright notice, 27 | this list of conditions and the following disclaimer in the documentation 28 | and/or other materials provided with the distribution. 29 | * Neither the name of the authors nor the names of its contributors may be 30 | used to endorse or promote products derived from this software without 31 | specific prior written permission. 32 | 33 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 34 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 35 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 36 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 37 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 38 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 39 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 40 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 41 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 42 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 43 | THE POSSIBILITY OF SUCH DAMAGE. 44 | */ 45 | 46 | #include 47 | #include 48 | #include 49 | #include 50 | #include 51 | #include 52 | #include 53 | 54 | #include 55 | 56 | #include "pole.h" 57 | 58 | #ifdef POLE_USE_UTF16_FILENAMES 59 | #include 60 | #endif //POLE_USE_UTF16_FILENAMES 61 | 62 | // enable to activate debugging output 63 | // #define POLE_DEBUG 64 | #define CACHEBUFSIZE 4096 //a presumably reasonable size for the read cache 65 | 66 | namespace POLE 67 | { 68 | 69 | class Header 70 | { 71 | public: 72 | unsigned char id[8]; // signature, or magic identifier 73 | uint64 b_shift; // bbat->blockSize = 1 << b_shift 74 | uint64 s_shift; // sbat->blockSize = 1 << s_shift 75 | uint64 num_bat; // blocks allocated for big bat 76 | uint64 dirent_start; // starting block for directory info 77 | uint64 threshold; // switch from small to big file (usually 4K) 78 | uint64 sbat_start; // starting block index to store small bat 79 | uint64 num_sbat; // blocks allocated for small bat 80 | uint64 mbat_start; // starting block to store meta bat 81 | uint64 num_mbat; // blocks allocated for meta bat 82 | uint64 bb_blocks[109]; 83 | bool dirty; // Needs to be written 84 | 85 | Header(); 86 | bool valid(); 87 | void load(const unsigned char* buffer); 88 | void save(unsigned char* buffer); 89 | void debug(); 90 | }; 91 | 92 | class AllocTable 93 | { 94 | public: 95 | static const uint64 Eof; 96 | static const uint64 Avail; 97 | static const uint64 Bat; 98 | static const uint64 MetaBat; 99 | uint64 blockSize; 100 | AllocTable(); 101 | void clear(); 102 | uint64 count(); 103 | uint64 unusedCount(); 104 | void resize(uint64 newsize); 105 | void preserve(uint64 n); 106 | void set(uint64 index, uint64 val); 107 | unsigned unused(); 108 | void setChain(std::vector); 109 | std::vector follow(uint64 start); 110 | uint64 operator[](uint64 index); 111 | void load(const unsigned char* buffer, uint64 len); 112 | void save(unsigned char* buffer); 113 | uint64 size(); 114 | void debug(); 115 | bool isDirty(); 116 | void markAsDirty(uint64 dataIndex, int64 bigBlockSize); 117 | void flush(std::vector blocks, StorageIO* const io, int64 bigBlockSize); 118 | private: 119 | std::vector data; 120 | std::vector dirtyBlocks; 121 | bool bMaybeFragmented; 122 | AllocTable(const AllocTable&); 123 | AllocTable& operator=(const AllocTable&); 124 | }; 125 | 126 | class DirEntry 127 | { 128 | public: 129 | DirEntry() : valid(), name(), dir(), size(), start(), prev(), next(), child() {} 130 | bool valid; // false if invalid (should be skipped) 131 | std::string name; // the name, not in unicode anymore 132 | bool dir; // true if directory 133 | uint64 size; // size (not valid if directory) 134 | uint64 start; // starting block 135 | uint64 prev; // previous sibling 136 | uint64 next; // next sibling 137 | uint64 child; // first child 138 | int compare(const DirEntry& de); 139 | int compare(const std::string& name2); 140 | 141 | }; 142 | 143 | class DirTree 144 | { 145 | public: 146 | static const uint64 End; 147 | DirTree(int64 bigBlockSize); 148 | void clear(int64 bigBlockSize); 149 | inline uint64 entryCount(); 150 | uint64 unusedEntryCount(); 151 | DirEntry* entry(uint64 index); 152 | DirEntry* entry(const std::string& name, bool create = false, int64 bigBlockSize = 0, StorageIO* const io = 0, int64 streamSize = 0); 153 | int64 indexOf(DirEntry* e); 154 | int64 parent(uint64 index); 155 | std::string fullName(uint64 index); 156 | std::vector children(uint64 index); 157 | uint64 find_child(uint64 index, const std::string& name, uint64& closest); 158 | void load(unsigned char* buffer, uint64 len); 159 | void save(unsigned char* buffer); 160 | uint64 size(); 161 | void debug(); 162 | bool isDirty(); 163 | void markAsDirty(uint64 dataIndex, int64 bigBlockSize); 164 | void flush(std::vector blocks, StorageIO* const io, int64 bigBlockSize, uint64 sb_start, uint64 sb_size); 165 | uint64 unused(); 166 | void findParentAndSib(uint64 inIdx, const std::string& inFullName, uint64& parentIdx, uint64& sibIdx); 167 | uint64 findSib(uint64 inIdx, uint64 sibIdx); 168 | void deleteEntry(DirEntry* entry, const std::string& inFullName, int64 bigBlockSize); 169 | private: 170 | std::vector entries; 171 | std::vector dirtyBlocks; 172 | DirTree(const DirTree&); 173 | DirTree& operator=(const DirTree&); 174 | }; 175 | 176 | class StorageIO 177 | { 178 | public: 179 | Storage* storage; // owner 180 | std::string filename; // filename 181 | std::fstream file; // associated with above name 182 | int64 result; // result of operation 183 | bool opened; // true if file is opened 184 | uint64 filesize; // size of the file 185 | bool writeable; // true if the file can be modified 186 | 187 | Header* header; // storage header 188 | DirTree* dirtree; // directory tree 189 | AllocTable* bbat; // allocation table for big blocks 190 | AllocTable* sbat; // allocation table for small blocks 191 | 192 | std::vector sb_blocks; // blocks for "small" files 193 | std::vector mbat_blocks; // blocks for doubly indirect indices to big blocks 194 | std::vector mbat_data; // the additional indices to big blocks 195 | bool mbatDirty; // If true, mbat_blocks need to be written 196 | 197 | std::list streams; 198 | 199 | StorageIO(Storage* storage, const char* filename); 200 | ~StorageIO(); 201 | 202 | bool open(bool bWriteAccess = false, bool bCreate = false); 203 | void close(); 204 | void flush(); 205 | void load(bool bWriteAccess); 206 | void create(); 207 | void init(); 208 | bool deleteByName(const std::string& fullName); 209 | 210 | bool deleteNode(DirEntry* entry, const std::string& fullName); 211 | 212 | bool deleteLeaf(DirEntry* entry, const std::string& fullName); 213 | 214 | uint64 loadBigBlocks(std::vector blocks, unsigned char* buffer, uint64 maxlen); 215 | 216 | uint64 loadBigBlock(uint64 block, unsigned char* buffer, uint64 maxlen); 217 | 218 | uint64 saveBigBlocks(std::vector blocks, uint64 offset, unsigned char* buffer, uint64 len); 219 | 220 | uint64 saveBigBlock(uint64 block, uint64 offset, unsigned char* buffer, uint64 len); 221 | 222 | uint64 loadSmallBlocks(std::vector blocks, unsigned char* buffer, uint64 maxlen); 223 | 224 | uint64 loadSmallBlock(uint64 block, unsigned char* buffer, uint64 maxlen); 225 | 226 | uint64 saveSmallBlocks(std::vector blocks, uint64 offset, unsigned char* buffer, uint64 len, int64 startAtBlock = 0); 227 | 228 | uint64 saveSmallBlock(uint64 block, uint64 offset, unsigned char* buffer, uint64 len); 229 | 230 | StreamIO* streamIO(const std::string& name, bool bCreate = false, int64 streamSize = 0); 231 | 232 | void flushbbat(); 233 | 234 | void flushsbat(); 235 | 236 | std::vector getbbatBlocks(bool bLoading); 237 | 238 | uint64 ExtendFile(std::vector* chain); 239 | 240 | void addbbatBlock(); 241 | 242 | private: 243 | // no copy or assign 244 | StorageIO(const StorageIO&); 245 | StorageIO& operator=(const StorageIO&); 246 | 247 | }; 248 | 249 | class StreamIO 250 | { 251 | public: 252 | StorageIO* io; 253 | int64 entryIdx; //needed because a pointer to DirEntry will change whenever entries vector changes. 254 | std::string fullName; 255 | bool eof; 256 | bool fail; 257 | 258 | StreamIO(StorageIO* io, DirEntry* entry); 259 | ~StreamIO(); 260 | uint64 size(); 261 | void setSize(uint64 newSize); 262 | void seek(uint64 pos); 263 | uint64 tell(); 264 | int64 getch(); 265 | uint64 read(unsigned char* data, uint64 maxlen); 266 | uint64 read(uint64 pos, unsigned char* data, uint64 maxlen); 267 | uint64 write(unsigned char* data, uint64 len); 268 | uint64 write(uint64 pos, unsigned char* data, uint64 len); 269 | void flush(); 270 | 271 | private: 272 | std::vector blocks; 273 | 274 | // no copy or assign 275 | StreamIO(const StreamIO&); 276 | StreamIO& operator=(const StreamIO&); 277 | 278 | // pointer for read 279 | uint64 m_pos; 280 | 281 | // simple cache system to speed-up getch() 282 | unsigned char* cache_data; 283 | uint64 cache_size; 284 | uint64 cache_pos; 285 | void updateCache(); 286 | }; 287 | 288 | }; // namespace POLE 289 | 290 | using namespace POLE; 291 | 292 | #ifdef POLE_USE_UTF16_FILENAMES 293 | 294 | std::string POLE::UTF16toUTF8(const std::wstring& utf16) { 295 | std::wstring_convert, wchar_t> converter; 296 | return converter.to_bytes(utf16); 297 | } 298 | 299 | std::wstring POLE::UTF8toUTF16(const std::string& utf8) { 300 | std::wstring_convert, wchar_t> converter; 301 | return converter.from_bytes(utf8); 302 | } 303 | 304 | #endif //POLE_USE_UTF16_FILENAMES 305 | 306 | static void fileCheck(std::fstream& file) 307 | { 308 | bool bGood, bFail, bEof, bBad; 309 | bool bNOTOK; 310 | bGood = file.good(); 311 | bFail = file.fail(); 312 | bEof = file.eof(); 313 | bBad = file.bad(); 314 | if (bFail || bEof || bBad) 315 | bNOTOK = true; //this doesn't really do anything, but it is a good place to set a breakpoint! 316 | file.clear(); 317 | } 318 | 319 | 320 | static inline uint32 readU16(const unsigned char* ptr) 321 | { 322 | return ptr[0] + (ptr[1] << 8); 323 | } 324 | 325 | static inline uint32 readU32(const unsigned char* ptr) 326 | { 327 | return ptr[0] + (ptr[1] << 8) + (ptr[2] << 16) + (ptr[3] << 24); 328 | } 329 | 330 | static inline void writeU16(unsigned char* ptr, uint32 data) 331 | { 332 | ptr[0] = (unsigned char)(data & 0xff); 333 | ptr[1] = (unsigned char)((data >> 8) & 0xff); 334 | } 335 | 336 | static inline void writeU32(unsigned char* ptr, uint32 data) 337 | { 338 | ptr[0] = (unsigned char)(data & 0xff); 339 | ptr[1] = (unsigned char)((data >> 8) & 0xff); 340 | ptr[2] = (unsigned char)((data >> 16) & 0xff); 341 | ptr[3] = (unsigned char)((data >> 24) & 0xff); 342 | } 343 | 344 | static const unsigned char pole_magic[] = 345 | { 0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1 }; 346 | 347 | // =========== Header ========== 348 | 349 | Header::Header() 350 | : b_shift(9), // [1EH,02] size of sectors in power-of-two; typically 9 indicating 512-byte sectors 351 | s_shift(6), // [20H,02] size of mini-sectors in power-of-two; typically 6 indicating 64-byte mini-sectors 352 | num_bat(0), // [2CH,04] number of SECTs in the FAT chain 353 | dirent_start(0), // [30H,04] first SECT in the directory chain 354 | threshold(4096), // [38H,04] maximum size for a mini stream; typically 4096 bytes 355 | sbat_start(0), // [3CH,04] first SECT in the MiniFAT chain 356 | num_sbat(0), // [40H,04] number of SECTs in the MiniFAT chain 357 | mbat_start(AllocTable::Eof),// [44H,04] first SECT in the DIFAT chain 358 | num_mbat(0), // [48H,04] number of SECTs in the DIFAT chain 359 | dirty(true) 360 | 361 | { 362 | for (unsigned int i = 0; i < 8; i++) 363 | id[i] = pole_magic[i]; 364 | for (unsigned int i = 0; i < 109; i++) 365 | bb_blocks[i] = AllocTable::Avail; 366 | } 367 | 368 | bool Header::valid() 369 | { 370 | if (threshold != 4096) return false; 371 | if (num_bat == 0) return false; 372 | //if( (num_bat > 109) && (num_bat > (num_mbat * 127) + 109)) return false; // dima: incorrect check, number may be arbitrary larger 373 | if ((num_bat < 109) && (num_mbat != 0)) return false; 374 | if (s_shift > b_shift) return false; 375 | if (b_shift <= 6) return false; 376 | if (b_shift >= 31) return false; 377 | 378 | return true; 379 | } 380 | 381 | void Header::load(const unsigned char* buffer) { 382 | b_shift = readU16(buffer + 0x1e); // [1EH,02] size of sectors in power-of-two; typically 9 indicating 512-byte sectors and 12 for 4096 383 | s_shift = readU16(buffer + 0x20); // [20H,02] size of mini-sectors in power-of-two; typically 6 indicating 64-byte mini-sectors 384 | num_bat = readU32(buffer + 0x2c); // [2CH,04] number of SECTs in the FAT chain 385 | dirent_start = readU32(buffer + 0x30); // [30H,04] first SECT in the directory chain 386 | threshold = readU32(buffer + 0x38); // [38H,04] maximum size for a mini stream; typically 4096 bytes 387 | sbat_start = readU32(buffer + 0x3c); // [3CH,04] first SECT in the MiniFAT chain 388 | num_sbat = readU32(buffer + 0x40); // [40H,04] number of SECTs in the MiniFAT chain 389 | mbat_start = readU32(buffer + 0x44); // [44H,04] first SECT in the DIFAT chain 390 | num_mbat = readU32(buffer + 0x48); // [48H,04] number of SECTs in the DIFAT chain 391 | 392 | for (unsigned int i = 0; i < 8; i++) 393 | id[i] = buffer[i]; 394 | 395 | // [4CH,436] the SECTs of first 109 FAT sectors 396 | for (unsigned int i = 0; i < 109; i++) 397 | bb_blocks[i] = readU32(buffer + 0x4C + i * 4); 398 | dirty = false; 399 | } 400 | 401 | void Header::save(unsigned char* buffer) 402 | { 403 | memset(buffer, 0, 0x4c); 404 | memcpy(buffer, pole_magic, 8); // ole signature 405 | writeU32(buffer + 8, 0); // unknown 406 | writeU32(buffer + 12, 0); // unknown 407 | writeU32(buffer + 16, 0); // unknown 408 | writeU16(buffer + 24, 0x003e); // revision ? 409 | writeU16(buffer + 26, 3); // version ? 410 | writeU16(buffer + 28, 0xfffe); // unknown 411 | writeU16(buffer + 0x1e, (uint32)b_shift); 412 | writeU16(buffer + 0x20, (uint32)s_shift); 413 | writeU32(buffer + 0x2c, (uint32)num_bat); 414 | writeU32(buffer + 0x30, (uint32)dirent_start); 415 | writeU32(buffer + 0x38, (uint32)threshold); 416 | writeU32(buffer + 0x3c, (uint32)sbat_start); 417 | writeU32(buffer + 0x40, (uint32)num_sbat); 418 | writeU32(buffer + 0x44, (uint32)mbat_start); 419 | writeU32(buffer + 0x48, (uint32)num_mbat); 420 | 421 | for (unsigned int i = 0; i < 109; i++) 422 | writeU32(buffer + 0x4C + i * 4, (uint32)bb_blocks[i]); 423 | dirty = false; 424 | } 425 | 426 | void Header::debug() 427 | { 428 | std::cout << std::endl; 429 | std::cout << "b_shift " << b_shift << std::endl; 430 | std::cout << "s_shift " << s_shift << std::endl; 431 | std::cout << "num_bat " << num_bat << std::endl; 432 | std::cout << "dirent_start " << dirent_start << std::endl; 433 | std::cout << "threshold " << threshold << std::endl; 434 | std::cout << "sbat_start " << sbat_start << std::endl; 435 | std::cout << "num_sbat " << num_sbat << std::endl; 436 | std::cout << "mbat_start " << mbat_start << std::endl; 437 | std::cout << "num_mbat " << num_mbat << std::endl; 438 | 439 | uint64 s = (num_bat <= 109) ? num_bat : 109; 440 | std::cout << "bat blocks: "; 441 | for (uint64 i = 0; i < s; i++) 442 | std::cout << bb_blocks[i] << " "; 443 | std::cout << std::endl; 444 | } 445 | 446 | // =========== AllocTable ========== 447 | 448 | const uint64 AllocTable::Avail = 0xffffffff; 449 | const uint64 AllocTable::Eof = 0xfffffffe; 450 | const uint64 AllocTable::Bat = 0xfffffffd; 451 | const uint64 AllocTable::MetaBat = 0xfffffffc; 452 | 453 | AllocTable::AllocTable() 454 | : blockSize(4096), 455 | data(), 456 | dirtyBlocks(), 457 | bMaybeFragmented(true) 458 | { 459 | // initial size 460 | resize(128); 461 | } 462 | 463 | uint64 AllocTable::count() 464 | { 465 | return static_cast(data.size()); 466 | } 467 | 468 | uint64 AllocTable::unusedCount() 469 | { 470 | uint64 maxIdx = count(); 471 | uint64 nFound = 0; 472 | for (uint64 idx = 0; idx < maxIdx; idx++) 473 | { 474 | if (data[idx] == Avail) 475 | nFound++; 476 | } 477 | return nFound; 478 | } 479 | 480 | void AllocTable::resize(uint64 newsize) 481 | { 482 | uint64 oldsize = static_cast(data.size()); 483 | data.resize(newsize); 484 | if (newsize > oldsize) 485 | for (uint64 i = oldsize; i < newsize; i++) 486 | data[i] = Avail; 487 | } 488 | 489 | // make sure there're still free blocks 490 | void AllocTable::preserve(uint64 n) 491 | { 492 | std::vector pre; 493 | for (unsigned i = 0; i < n; i++) 494 | pre.push_back(unused()); 495 | } 496 | 497 | uint64 AllocTable::operator[](uint64 index) 498 | { 499 | uint64 result; 500 | result = data[index]; 501 | return result; 502 | } 503 | 504 | void AllocTable::set(uint64 index, uint64 value) 505 | { 506 | if (index >= count()) resize(index + 1); 507 | data[index] = value; 508 | if (value == Avail) 509 | bMaybeFragmented = true; 510 | } 511 | 512 | void AllocTable::setChain(std::vector chain) 513 | { 514 | if (chain.size()) 515 | { 516 | for (unsigned i = 0; i < chain.size() - 1; i++) 517 | set(chain[i], chain[i + 1]); 518 | set(chain[chain.size() - 1], AllocTable::Eof); 519 | } 520 | } 521 | 522 | // follow 523 | std::vector AllocTable::follow(uint64 start) 524 | { 525 | std::vector chain; 526 | 527 | if (start >= count()) return chain; 528 | 529 | uint64 p = start; 530 | while (p < count()) 531 | { 532 | if (p == (uint64)Eof) break; 533 | if (p == (uint64)Bat) break; 534 | if (p == (uint64)MetaBat) break; 535 | if (p >= count()) break; 536 | chain.push_back(p); 537 | if (data[p] >= count()) break; 538 | p = data[p]; 539 | } 540 | 541 | return chain; 542 | } 543 | 544 | unsigned AllocTable::unused() 545 | { 546 | // find first available block 547 | unsigned int maxIdx = (unsigned int)data.size(); 548 | if (bMaybeFragmented) 549 | { 550 | for (unsigned i = 0; i < maxIdx; i++) 551 | if (data[i] == Avail) 552 | return i; 553 | } 554 | 555 | // completely full, so enlarge the table 556 | unsigned int block = maxIdx; 557 | resize(maxIdx); 558 | bMaybeFragmented = false; 559 | return block; 560 | } 561 | 562 | void AllocTable::load(const unsigned char* buffer, uint64 len) 563 | { 564 | resize(len / 4); 565 | for (unsigned i = 0; i < count(); i++) 566 | set(i, readU32(buffer + i * 4)); 567 | } 568 | 569 | // return space required to save this dirtree 570 | uint64 AllocTable::size() 571 | { 572 | return count() * 4; 573 | } 574 | 575 | void AllocTable::save(unsigned char* buffer) 576 | { 577 | for (uint64 i = 0; i < count(); i++) 578 | writeU32(buffer + i * 4, (uint32)data[i]); 579 | } 580 | 581 | bool AllocTable::isDirty() 582 | { 583 | return (dirtyBlocks.size() > 0); 584 | } 585 | 586 | void AllocTable::markAsDirty(uint64 dataIndex, int64 bigBlockSize) 587 | { 588 | uint64 dbidx = dataIndex / (bigBlockSize / sizeof(uint32)); 589 | for (uint64 idx = 0; idx < static_cast(dirtyBlocks.size()); idx++) 590 | { 591 | if (dirtyBlocks[idx] == dbidx) 592 | return; 593 | } 594 | dirtyBlocks.push_back(dbidx); 595 | } 596 | 597 | void AllocTable::flush(std::vector blocks, StorageIO* const io, int64 bigBlockSize) 598 | { 599 | unsigned char* buffer = new unsigned char[bigBlockSize * blocks.size()]; 600 | save(buffer); 601 | for (uint64 idx = 0; idx < static_cast(blocks.size()); idx++) 602 | { 603 | bool bDirty = false; 604 | for (uint64 idx2 = 0; idx2 < static_cast(dirtyBlocks.size()); idx2++) 605 | { 606 | if (dirtyBlocks[idx2] == idx) 607 | { 608 | bDirty = true; 609 | break; 610 | } 611 | } 612 | if (bDirty) 613 | io->saveBigBlock(blocks[idx], 0, &buffer[bigBlockSize * idx], bigBlockSize); 614 | } 615 | dirtyBlocks.clear(); 616 | delete[] buffer; 617 | } 618 | 619 | void AllocTable::debug() 620 | { 621 | std::cout << "block size " << data.size() << std::endl; 622 | for (unsigned i = 0; i < data.size(); i++) 623 | { 624 | if (data[i] == Avail) continue; 625 | std::cout << i << ": "; 626 | if (data[i] == Eof) std::cout << "[eof]"; 627 | else if (data[i] == Bat) std::cout << "[bat]"; 628 | else if (data[i] == MetaBat) std::cout << "[metabat]"; 629 | else std::cout << data[i]; 630 | std::cout << std::endl; 631 | } 632 | } 633 | 634 | // =========== DirEntry ========== 635 | // "A node with a shorter name is less than a node with a inter name" 636 | // "For nodes with the same length names, compare the two names." 637 | // --Windows Compound Binary File Format Specification, Section 2.5 638 | int DirEntry::compare(const DirEntry& de) 639 | { 640 | return compare(de.name); 641 | } 642 | 643 | int DirEntry::compare(const std::string& name2) 644 | { 645 | if (name.length() < name2.length()) 646 | return -1; 647 | else if (name.length() > name2.length()) 648 | return 1; 649 | else 650 | return name.compare(name2); 651 | } 652 | 653 | 654 | // =========== DirTree ========== 655 | 656 | const uint64 DirTree::End = 0xffffffff; 657 | 658 | DirTree::DirTree(int64 bigBlockSize) 659 | : entries(), 660 | dirtyBlocks() 661 | { 662 | clear(bigBlockSize); 663 | } 664 | 665 | void DirTree::clear(int64 bigBlockSize) 666 | { 667 | // leave only root entry 668 | entries.resize(1); 669 | entries[0].valid = true; 670 | entries[0].name = "Root Entry"; 671 | entries[0].dir = true; 672 | entries[0].size = 0; 673 | entries[0].start = End; 674 | entries[0].prev = End; 675 | entries[0].next = End; 676 | entries[0].child = End; 677 | markAsDirty(0, bigBlockSize); 678 | } 679 | 680 | inline uint64 DirTree::entryCount() 681 | { 682 | return entries.size(); 683 | } 684 | 685 | uint64 DirTree::unusedEntryCount() 686 | { 687 | uint64 nFound = 0; 688 | for (uint64 idx = 0; idx < entryCount(); idx++) 689 | { 690 | if (!entries[idx].valid) 691 | nFound++; 692 | } 693 | return nFound; 694 | } 695 | 696 | DirEntry* DirTree::entry(uint64 index) 697 | { 698 | if (index >= entryCount()) return (DirEntry*)0; 699 | return &entries[index]; 700 | } 701 | 702 | int64 DirTree::indexOf(DirEntry* e) 703 | { 704 | for (uint64 i = 0; i < entryCount(); i++) 705 | if (entry(i) == e) return i; 706 | 707 | return -1; 708 | } 709 | 710 | int64 DirTree::parent(uint64 index) 711 | { 712 | // brute-force, basically we iterate for each entries, find its children 713 | // and check if one of the children is 'index' 714 | for (uint64 j = 0; j < entryCount(); j++) 715 | { 716 | std::vector chi = children(j); 717 | for (unsigned i = 0; i < chi.size(); i++) 718 | if (chi[i] == index) 719 | return j; 720 | } 721 | 722 | return -1; 723 | } 724 | 725 | std::string DirTree::fullName(uint64 index) 726 | { 727 | // don't use root name ("Root Entry"), just give "/" 728 | if (index == 0) return "/"; 729 | 730 | std::string result = entry(index)->name; 731 | result.insert(0, "/"); 732 | uint64 p = parent(index); 733 | DirEntry* _entry = 0; 734 | while (p > 0) 735 | { 736 | _entry = entry(p); 737 | if (_entry->dir && _entry->valid) 738 | { 739 | result.insert(0, _entry->name); 740 | result.insert(0, "/"); 741 | } 742 | --p; 743 | index = p; 744 | if (index <= 0) break; 745 | } 746 | return result; 747 | } 748 | 749 | // given a fullname (e.g "/ObjectPool/_1020961869"), find the entry 750 | // if not found and create is false, return 0 751 | // if create is true, a new entry is returned 752 | DirEntry* DirTree::entry(const std::string& name, bool create, int64 bigBlockSize, StorageIO* const io, int64 streamSize) 753 | { 754 | if (!name.length()) return (DirEntry*)0; 755 | 756 | // quick check for "/" (that's root) 757 | if (name == "/") return entry(0); 758 | 759 | // split the names, e.g "/ObjectPool/_1020961869" will become: 760 | // "ObjectPool" and "_1020961869" 761 | std::list names; 762 | std::string::size_type start = 0, end = 0; 763 | if (name[0] == '/') start++; 764 | int levelsLeft = 0; 765 | while (start < name.length()) 766 | { 767 | end = name.find_first_of('/', start); 768 | if (end == std::string::npos) end = name.length(); 769 | names.push_back(name.substr(start, end - start)); 770 | levelsLeft++; 771 | start = end + 1; 772 | } 773 | 774 | // start from root 775 | int64 index = 0; 776 | 777 | // trace one by one 778 | std::list::iterator it; 779 | 780 | for (it = names.begin(); it != names.end(); ++it) 781 | { 782 | // find among the children of index 783 | levelsLeft--; 784 | uint64 child = 0; 785 | 786 | 787 | /* 788 | // dima: this block is really inefficient 789 | std::vector chi = children( index ); 790 | for( unsigned i = 0; i < chi.size(); i++ ) 791 | { 792 | DirEntry* ce = entry( chi[i] ); 793 | if( ce ) 794 | if( ce->valid && ( ce->name.length()>1 ) ) 795 | if( ce->name == *it ) { 796 | child = chi[i]; 797 | break; 798 | } 799 | } 800 | */ 801 | // dima: performance optimisation of the previous 802 | uint64 closest = End; 803 | child = find_child(index, *it, closest); 804 | 805 | // traverse to the child 806 | if (child > 0) index = child; 807 | else 808 | { 809 | // not found among children 810 | if (!create || !io->writeable) return (DirEntry*)0; 811 | 812 | // create a new entry 813 | uint64 parent2 = index; 814 | index = unused(); 815 | DirEntry* e = entry(index); 816 | e->valid = true; 817 | e->name = *it; 818 | e->dir = (levelsLeft > 0); 819 | if (!e->dir) 820 | e->size = streamSize; 821 | else 822 | e->size = 0; 823 | e->start = AllocTable::Eof; 824 | e->child = End; 825 | if (closest == End) 826 | { 827 | e->prev = End; 828 | e->next = entry(parent2)->child; 829 | entry(parent2)->child = index; 830 | markAsDirty(parent2, bigBlockSize); 831 | } 832 | else 833 | { 834 | DirEntry* closeE = entry(closest); 835 | if (closeE->compare(*e) < 0) 836 | { 837 | e->prev = closeE->next; 838 | e->next = End; 839 | closeE->next = index; 840 | } 841 | else 842 | { 843 | e->next = closeE->prev; 844 | e->prev = End; 845 | closeE->prev = index; 846 | } 847 | markAsDirty(closest, bigBlockSize); 848 | } 849 | markAsDirty(index, bigBlockSize); 850 | uint64 bbidx = index / (bigBlockSize / 128); 851 | std::vector blocks = io->bbat->follow(io->header->dirent_start); 852 | while (blocks.size() <= bbidx) 853 | { 854 | uint64 nblock = io->bbat->unused(); 855 | if (blocks.size() > 0) 856 | { 857 | io->bbat->set(blocks[static_cast(blocks.size()) - 1], nblock); 858 | io->bbat->markAsDirty(blocks[static_cast(blocks.size()) - 1], bigBlockSize); 859 | } 860 | io->bbat->set(nblock, AllocTable::Eof); 861 | io->bbat->markAsDirty(nblock, bigBlockSize); 862 | blocks.push_back(nblock); 863 | uint64 bbidx = nblock / (io->bbat->blockSize / sizeof(uint64)); 864 | while (bbidx >= io->header->num_bat) 865 | io->addbbatBlock(); 866 | } 867 | } 868 | } 869 | 870 | return entry(index); 871 | } 872 | 873 | // helper function: recursively find siblings of index 874 | void dirtree_find_siblings(DirTree* dirtree, std::vector& result, 875 | uint64 index) 876 | { 877 | DirEntry* e = dirtree->entry(index); 878 | if (!e) return; 879 | if (e->prev != DirTree::End) 880 | dirtree_find_siblings(dirtree, result, e->prev); 881 | result.push_back(index); 882 | if (e->next != DirTree::End) 883 | dirtree_find_siblings(dirtree, result, e->next); 884 | } 885 | 886 | std::vector DirTree::children(uint64 index) 887 | { 888 | std::vector result; 889 | 890 | DirEntry* e = entry(index); 891 | if (e) if (e->valid && e->child < entryCount()) 892 | dirtree_find_siblings(this, result, e->child); 893 | 894 | return result; 895 | } 896 | 897 | uint64 dirtree_find_sibling(DirTree* dirtree, uint64 index, const std::string& name, uint64& closest) { 898 | 899 | uint64 count = dirtree->entryCount(); 900 | DirEntry* e = dirtree->entry(index); 901 | if (!e || !e->valid) return 0; 902 | int cval = e->compare(name); 903 | if (cval == 0) 904 | return index; 905 | if (cval > 0) 906 | { 907 | if (e->prev > 0 && e->prev < count) 908 | return dirtree_find_sibling(dirtree, e->prev, name, closest); 909 | } 910 | else 911 | { 912 | if (e->next > 0 && e->next < count) 913 | return dirtree_find_sibling(dirtree, e->next, name, closest); 914 | } 915 | closest = index; 916 | return 0; 917 | } 918 | 919 | uint64 DirTree::find_child(uint64 index, const std::string& name, uint64& closest) { 920 | 921 | uint64 count = entryCount(); 922 | DirEntry* p = entry(index); 923 | if (p && p->valid && p->child < count) 924 | return dirtree_find_sibling(this, p->child, name, closest); 925 | 926 | return 0; 927 | } 928 | 929 | void DirTree::load(unsigned char* buffer, uint64 size) 930 | { 931 | entries.clear(); 932 | 933 | for (uint64 i = 0; i < size / 128; i++) 934 | { 935 | uint64 p = i * 128; 936 | 937 | // would be < 32 if first char in the name isn't printable 938 | unsigned prefix = 32; 939 | 940 | // parse name of this entry, which stored as Unicode 16-bit 941 | std::string name; 942 | int name_len = readU16(buffer + 0x40 + p); 943 | if (name_len > 64) name_len = 64; 944 | for (int j = 0; (buffer[j + p]) && (j < name_len); j += 2) 945 | name.append(1, buffer[j + p]); 946 | 947 | // first char isn't printable ? remove it... 948 | if (buffer[p] < 32) 949 | { 950 | prefix = buffer[0]; 951 | name.erase(0, 1); 952 | } 953 | 954 | // 2 = file (aka stream), 1 = directory (aka storage), 5 = root 955 | unsigned type = buffer[0x42 + p]; 956 | 957 | DirEntry e; 958 | e.valid = (type != 0); 959 | e.name = name; 960 | e.start = readU32(buffer + 0x74 + p); 961 | e.size = readU32(buffer + 0x78 + p); 962 | e.prev = readU32(buffer + 0x44 + p); 963 | e.next = readU32(buffer + 0x48 + p); 964 | e.child = readU32(buffer + 0x4C + p); 965 | e.dir = (type != 2); 966 | 967 | // sanity checks 968 | if ((type != 2) && (type != 1) && (type != 5)) e.valid = false; 969 | if (name_len < 1) e.valid = false; 970 | 971 | entries.push_back(e); 972 | } 973 | } 974 | 975 | // return space required to save this dirtree 976 | uint64 DirTree::size() 977 | { 978 | return entryCount() * 128; 979 | } 980 | 981 | void DirTree::save(unsigned char* buffer) 982 | { 983 | memset(buffer, 0, size()); 984 | 985 | // root is fixed as "Root Entry" 986 | DirEntry* root = entry(0); 987 | std::string name = "Root Entry"; 988 | for (unsigned int j = 0; j < name.length(); j++) 989 | buffer[j * 2] = name[j]; 990 | writeU16(buffer + 0x40, static_cast(name.length() * 2 + 2)); 991 | writeU32(buffer + 0x74, 0xffffffff); 992 | writeU32(buffer + 0x78, 0); 993 | writeU32(buffer + 0x44, 0xffffffff); 994 | writeU32(buffer + 0x48, 0xffffffff); 995 | writeU32(buffer + 0x4c, (uint32)root->child); 996 | buffer[0x42] = 5; 997 | //buffer[ 0x43 ] = 1; 998 | 999 | for (unsigned int i = 1; i < entryCount(); i++) 1000 | { 1001 | DirEntry* e = entry(i); 1002 | if (!e) continue; 1003 | if (e->dir) 1004 | { 1005 | e->start = 0xffffffff; 1006 | e->size = 0; 1007 | } 1008 | 1009 | // max length for name is 32 chars 1010 | std::string name = e->name; 1011 | if (name.length() > 32) 1012 | name.erase(32, name.length()); 1013 | 1014 | // write name as Unicode 16-bit 1015 | for (unsigned j = 0; j < name.length(); j++) 1016 | buffer[i * 128 + j * 2] = name[j]; 1017 | 1018 | writeU16(buffer + i * 128 + 0x40, static_cast(name.length() * 2 + 2)); 1019 | writeU32(buffer + i * 128 + 0x74, (uint32)e->start); 1020 | writeU32(buffer + i * 128 + 0x78, (uint32)e->size); 1021 | writeU32(buffer + i * 128 + 0x44, (uint32)e->prev); 1022 | writeU32(buffer + i * 128 + 0x48, (uint32)e->next); 1023 | writeU32(buffer + i * 128 + 0x4c, (uint32)e->child); 1024 | if (!e->valid) 1025 | buffer[i * 128 + 0x42] = 0; //STGTY_INVALID 1026 | else 1027 | buffer[i * 128 + 0x42] = e->dir ? 1 : 2; //STGTY_STREAM or STGTY_STORAGE 1028 | buffer[i * 128 + 0x43] = 1; // always black 1029 | } 1030 | } 1031 | 1032 | bool DirTree::isDirty() 1033 | { 1034 | return (dirtyBlocks.size() > 0); 1035 | } 1036 | 1037 | 1038 | void DirTree::markAsDirty(uint64 dataIndex, int64 bigBlockSize) 1039 | { 1040 | uint64 dbidx = dataIndex / (bigBlockSize / 128); 1041 | for (uint64 idx = 0; idx < static_cast(dirtyBlocks.size()); idx++) 1042 | { 1043 | if (dirtyBlocks[idx] == dbidx) 1044 | return; 1045 | } 1046 | dirtyBlocks.push_back(dbidx); 1047 | } 1048 | 1049 | void DirTree::flush(std::vector blocks, StorageIO* const io, int64 bigBlockSize, uint64 sb_start, uint64 sb_size) 1050 | { 1051 | uint64 bufLen = size(); 1052 | unsigned char* buffer = new unsigned char[bufLen]; 1053 | save(buffer); 1054 | writeU32(buffer + 0x74, (uint32)sb_start); 1055 | writeU32(buffer + 0x78, (uint32)sb_size); 1056 | for (uint64 idx = 0; idx < static_cast(blocks.size()); idx++) 1057 | { 1058 | bool bDirty = false; 1059 | for (uint64 idx2 = 0; idx2 < static_cast(dirtyBlocks.size()); idx2++) 1060 | { 1061 | if (dirtyBlocks[idx2] == idx) 1062 | { 1063 | bDirty = true; 1064 | break; 1065 | } 1066 | } 1067 | uint64 bytesToWrite = bigBlockSize; 1068 | uint64 pos = bigBlockSize * idx; 1069 | if ((bufLen - pos) < bytesToWrite) 1070 | bytesToWrite = bufLen - pos; 1071 | if (bDirty) 1072 | io->saveBigBlock(blocks[idx], 0, &buffer[pos], bytesToWrite); 1073 | } 1074 | dirtyBlocks.clear(); 1075 | delete[] buffer; 1076 | } 1077 | 1078 | uint64 DirTree::unused() 1079 | { 1080 | for (uint64 idx = 0; idx < static_cast(entryCount()); idx++) 1081 | { 1082 | if (!entries[idx].valid) 1083 | return idx; 1084 | } 1085 | entries.push_back(DirEntry()); 1086 | return entryCount() - 1; 1087 | } 1088 | 1089 | // Utility function to get the index of the parent dirEntry, given that we already have a full name it is relatively fast. 1090 | // Then look for a sibling dirEntry that points to inIdx. In some circumstances, the dirEntry at inIdx will be the direct child 1091 | // of the parent, in which case sibIdx will be returned as 0. A failure is indicated if both parentIdx and sibIdx are returned as 0. 1092 | 1093 | void DirTree::findParentAndSib(uint64 inIdx, const std::string& inFullName, uint64& parentIdx, uint64& sibIdx) 1094 | { 1095 | sibIdx = 0; 1096 | parentIdx = 0; 1097 | if (inIdx == 0 || inIdx >= entryCount() || inFullName == "/" || inFullName == "") 1098 | return; 1099 | std::string localName = inFullName; 1100 | if (localName[0] != '/') 1101 | localName = '/' + localName; 1102 | std::string parentName = localName; 1103 | if (parentName[parentName.size() - 1] == '/') 1104 | parentName = parentName.substr(0, parentName.size() - 1); 1105 | std::string::size_type lastSlash; 1106 | lastSlash = parentName.find_last_of('/'); 1107 | if (lastSlash == std::string::npos) 1108 | return; 1109 | if (lastSlash == 0) 1110 | lastSlash = 1; //leave root 1111 | parentName = parentName.substr(0, lastSlash); 1112 | DirEntry* parent2 = entry(parentName); 1113 | parentIdx = indexOf(parent2); 1114 | if (parent2->child == inIdx) 1115 | return; //successful return, no sibling points to inIdx 1116 | sibIdx = findSib(inIdx, parent2->child); 1117 | } 1118 | 1119 | // Utility function to get the index of the sibling dirEntry which points to inIdx. It is the responsibility of the original caller 1120 | // to start with the root sibling - i.e., sibIdx should be pointed to by the parent node's child. 1121 | 1122 | uint64 DirTree::findSib(uint64 inIdx, uint64 sibIdx) 1123 | { 1124 | DirEntry* sib = entry(sibIdx); 1125 | if (!sib || !sib->valid) 1126 | return 0; 1127 | if (sib->next == inIdx || sib->prev == inIdx) 1128 | return sibIdx; 1129 | DirEntry* targetSib = entry(inIdx); 1130 | int cval = sib->compare(*targetSib); 1131 | if (cval > 0) 1132 | return findSib(inIdx, sib->prev); 1133 | else 1134 | return findSib(inIdx, sib->next); 1135 | } 1136 | 1137 | void DirTree::deleteEntry(DirEntry* dirToDel, const std::string& inFullName, int64 bigBlockSize) 1138 | { 1139 | uint64 parentIdx; 1140 | uint64 sibIdx; 1141 | uint64 inIdx = indexOf(dirToDel); 1142 | uint64 nEntries = entryCount(); 1143 | findParentAndSib(inIdx, inFullName, parentIdx, sibIdx); 1144 | uint64 replIdx; 1145 | if (!dirToDel->next || dirToDel->next > nEntries) 1146 | replIdx = dirToDel->prev; 1147 | else 1148 | { 1149 | DirEntry* sibNext = entry(dirToDel->next); 1150 | if (!sibNext->prev || sibNext->prev > nEntries) 1151 | { 1152 | replIdx = dirToDel->next; 1153 | sibNext->prev = dirToDel->prev; 1154 | markAsDirty(replIdx, bigBlockSize); 1155 | } 1156 | else 1157 | { 1158 | DirEntry* smlSib = sibNext; 1159 | int64 smlIdx = dirToDel->next; 1160 | DirEntry* smlrSib; 1161 | int64 smlrIdx = -1; 1162 | for (; ; ) 1163 | { 1164 | smlrIdx = smlSib->prev; 1165 | smlrSib = entry(smlrIdx); 1166 | if (!smlrSib->prev || smlrSib->prev > nEntries) 1167 | break; 1168 | smlSib = smlrSib; 1169 | smlIdx = smlrIdx; 1170 | } 1171 | replIdx = smlSib->prev; 1172 | smlSib->prev = smlrSib->next; 1173 | smlrSib->prev = dirToDel->prev; 1174 | smlrSib->next = dirToDel->next; 1175 | markAsDirty(smlIdx, bigBlockSize); 1176 | markAsDirty(smlrIdx, bigBlockSize); 1177 | } 1178 | } 1179 | if (sibIdx) 1180 | { 1181 | DirEntry* sib = entry(sibIdx); 1182 | if (sib->next == inIdx) 1183 | sib->next = replIdx; 1184 | else 1185 | sib->prev = replIdx; 1186 | markAsDirty(sibIdx, bigBlockSize); 1187 | } 1188 | else 1189 | { 1190 | DirEntry* parNode = entry(parentIdx); 1191 | parNode->child = replIdx; 1192 | markAsDirty(parentIdx, bigBlockSize); 1193 | } 1194 | dirToDel->valid = false; //indicating that this entry is not in use 1195 | markAsDirty(inIdx, bigBlockSize); 1196 | } 1197 | 1198 | 1199 | void DirTree::debug() 1200 | { 1201 | for (unsigned i = 0; i < entryCount(); i++) 1202 | { 1203 | DirEntry* e = entry(i); 1204 | if (!e) continue; 1205 | std::cout << i << ": "; 1206 | if (!e->valid) std::cout << "INVALID "; 1207 | std::cout << e->name << " "; 1208 | if (e->dir) std::cout << "(Dir) "; 1209 | else std::cout << "(File) "; 1210 | std::cout << e->size << " "; 1211 | std::cout << "s:" << e->start << " "; 1212 | std::cout << "("; 1213 | if (e->child == End) std::cout << "-"; else std::cout << e->child; 1214 | std::cout << " "; 1215 | if (e->prev == End) std::cout << "-"; else std::cout << e->prev; 1216 | std::cout << ":"; 1217 | if (e->next == End) std::cout << "-"; else std::cout << e->next; 1218 | std::cout << ")"; 1219 | std::cout << std::endl; 1220 | } 1221 | } 1222 | 1223 | // =========== StorageIO ========== 1224 | 1225 | StorageIO::StorageIO(Storage* st, const char* fname) 1226 | : storage(st), 1227 | filename(fname), 1228 | file(), 1229 | result(Storage::Ok), 1230 | opened(false), 1231 | filesize(0), 1232 | writeable(false), 1233 | header(new Header()), 1234 | dirtree(new DirTree(1 << header->b_shift)), 1235 | bbat(new AllocTable()), 1236 | sbat(new AllocTable()), 1237 | sb_blocks(), 1238 | mbat_blocks(), 1239 | mbat_data(), 1240 | mbatDirty(), 1241 | streams() 1242 | { 1243 | bbat->blockSize = (uint64)1 << header->b_shift; 1244 | sbat->blockSize = (uint64)1 << header->s_shift; 1245 | } 1246 | 1247 | StorageIO::~StorageIO() 1248 | { 1249 | if (opened) close(); 1250 | delete sbat; 1251 | delete bbat; 1252 | delete dirtree; 1253 | delete header; 1254 | } 1255 | 1256 | bool StorageIO::open(bool bWriteAccess, bool bCreate) 1257 | { 1258 | // already opened ? close first 1259 | if (opened) 1260 | close(); 1261 | if (bCreate) 1262 | { 1263 | create(); 1264 | init(); 1265 | writeable = true; 1266 | } 1267 | else 1268 | { 1269 | writeable = bWriteAccess; 1270 | load(bWriteAccess); 1271 | } 1272 | 1273 | return result == Storage::Ok; 1274 | } 1275 | 1276 | void StorageIO::load(bool bWriteAccess) 1277 | { 1278 | unsigned char* buffer = 0; 1279 | uint64 buflen = 0; 1280 | std::vector blocks; 1281 | 1282 | // open the file, check for error 1283 | result = Storage::OpenFailed; 1284 | 1285 | #if defined(POLE_USE_UTF16_FILENAMES) 1286 | if (bWriteAccess) 1287 | file.open(UTF8toUTF16(filename).c_str(), std::ios::binary | std::ios::in | std::ios::out); 1288 | else 1289 | file.open(UTF8toUTF16(filename).c_str(), std::ios::binary | std::ios::in); 1290 | #else 1291 | if (bWriteAccess) 1292 | file.open(filename.c_str(), std::ios::binary | std::ios::in | std::ios::out); 1293 | else 1294 | file.open(filename.c_str(), std::ios::binary | std::ios::in); 1295 | #endif //defined(POLE_USE_UTF16_FILENAMES) && defined(POLE_WIN) 1296 | 1297 | if (!file.good()) return; 1298 | 1299 | // find size of input file 1300 | file.seekg(0, std::ios::end); 1301 | filesize = static_cast(file.tellg()); 1302 | 1303 | // load header 1304 | buffer = new unsigned char[512]; 1305 | file.seekg(0); 1306 | file.read((char*)buffer, 512); 1307 | fileCheck(file); 1308 | header->load(buffer); 1309 | delete[] buffer; 1310 | 1311 | // check OLE magic id 1312 | result = Storage::NotOLE; 1313 | for (unsigned i = 0; i < 8; i++) 1314 | if (header->id[i] != pole_magic[i]) 1315 | return; 1316 | 1317 | // sanity checks 1318 | result = Storage::BadOLE; 1319 | if (!header->valid()) return; 1320 | if (header->threshold != 4096) return; 1321 | 1322 | // important block size 1323 | bbat->blockSize = (uint64)1 << header->b_shift; 1324 | sbat->blockSize = (uint64)1 << header->s_shift; 1325 | 1326 | blocks = getbbatBlocks(true); 1327 | 1328 | // load big bat 1329 | buflen = static_cast(blocks.size())* bbat->blockSize; 1330 | if (buflen > 0) 1331 | { 1332 | buffer = new unsigned char[buflen]; 1333 | loadBigBlocks(blocks, buffer, buflen); 1334 | bbat->load(buffer, buflen); 1335 | delete[] buffer; 1336 | } 1337 | 1338 | // load small bat 1339 | blocks.clear(); 1340 | blocks = bbat->follow(header->sbat_start); 1341 | buflen = static_cast(blocks.size())* bbat->blockSize; 1342 | if (buflen > 0) 1343 | { 1344 | buffer = new unsigned char[buflen]; 1345 | loadBigBlocks(blocks, buffer, buflen); 1346 | sbat->load(buffer, buflen); 1347 | delete[] buffer; 1348 | } 1349 | 1350 | // load directory tree 1351 | blocks.clear(); 1352 | blocks = bbat->follow(header->dirent_start); 1353 | buflen = static_cast(blocks.size())* bbat->blockSize; 1354 | buffer = new unsigned char[buflen]; 1355 | loadBigBlocks(blocks, buffer, buflen); 1356 | dirtree->load(buffer, buflen); 1357 | unsigned sb_start = readU32(buffer + 0x74); 1358 | delete[] buffer; 1359 | 1360 | // fetch block chain as data for small-files 1361 | sb_blocks = bbat->follow(sb_start); // small files 1362 | 1363 | // for troubleshooting, just enable this block 1364 | #if 0 1365 | header->debug(); 1366 | sbat->debug(); 1367 | bbat->debug(); 1368 | dirtree->debug(); 1369 | #endif 1370 | 1371 | // so far so good 1372 | result = Storage::Ok; 1373 | opened = true; 1374 | } 1375 | 1376 | void StorageIO::create() { 1377 | // std::cout << "Creating " << filename << std::endl; 1378 | 1379 | #if defined(POLE_USE_UTF16_FILENAMES) 1380 | file.open(UTF8toUTF16(filename).c_str(), std::ios::binary | std::ios::in | std::ios::out | std::ios::trunc); 1381 | #else 1382 | file.open(filename.c_str(), std::ios::binary | std::ios::in | std::ios::out | std::ios::trunc); 1383 | #endif 1384 | if (!file.good()) 1385 | { 1386 | std::cerr << "Can't create " << filename << std::endl; 1387 | result = Storage::OpenFailed; 1388 | return; 1389 | } 1390 | 1391 | // so far so good 1392 | opened = true; 1393 | result = Storage::Ok; 1394 | } 1395 | 1396 | void StorageIO::init() 1397 | { 1398 | // Initialize parts of the header, directory entries, and big and small allocation tables 1399 | header->bb_blocks[0] = 0; 1400 | header->dirent_start = 1; 1401 | header->sbat_start = 2; 1402 | header->num_bat = 1; 1403 | header->num_sbat = 1; 1404 | header->dirty = true; 1405 | bbat->set(0, AllocTable::Eof); 1406 | bbat->markAsDirty(0, bbat->blockSize); 1407 | bbat->set(1, AllocTable::Eof); 1408 | bbat->markAsDirty(1, bbat->blockSize); 1409 | bbat->set(2, AllocTable::Eof); 1410 | bbat->markAsDirty(2, bbat->blockSize); 1411 | bbat->set(3, AllocTable::Eof); 1412 | bbat->markAsDirty(3, bbat->blockSize); 1413 | sb_blocks = bbat->follow(3); 1414 | mbatDirty = false; 1415 | } 1416 | 1417 | void StorageIO::flush() 1418 | { 1419 | if (header->dirty) 1420 | { 1421 | unsigned char* buffer = new unsigned char[512]; 1422 | header->save(buffer); 1423 | file.seekp(0); 1424 | file.write((char*)buffer, 512); 1425 | fileCheck(file); 1426 | delete[] buffer; 1427 | } 1428 | if (bbat->isDirty()) 1429 | flushbbat(); 1430 | if (sbat->isDirty()) 1431 | flushsbat(); 1432 | if (dirtree->isDirty()) 1433 | { 1434 | std::vector blocks; 1435 | blocks = bbat->follow(header->dirent_start); 1436 | uint64 sb_start = 0xffffffff; 1437 | if (sb_blocks.size() > 0) 1438 | sb_start = sb_blocks[0]; 1439 | dirtree->flush(blocks, this, bbat->blockSize, sb_start, static_cast(sb_blocks.size())* bbat->blockSize); 1440 | } 1441 | if (mbatDirty && mbat_blocks.size() > 0) 1442 | { 1443 | uint64 nBytes = bbat->blockSize * static_cast(mbat_blocks.size()); 1444 | unsigned char* buffer = new unsigned char[nBytes]; 1445 | uint64 sIdx = 0; 1446 | uint64 dcount = 0; 1447 | uint64 blockCapacity = bbat->blockSize / sizeof(uint64) - 1; 1448 | uint64 blockIdx = 0; 1449 | for (unsigned mdIdx = 0; mdIdx < mbat_data.size(); mdIdx++) 1450 | { 1451 | writeU32(buffer + sIdx, (uint32)mbat_data[mdIdx]); 1452 | sIdx += 4; 1453 | dcount++; 1454 | if (dcount == blockCapacity) 1455 | { 1456 | blockIdx++; 1457 | if (blockIdx == mbat_blocks.size()) 1458 | writeU32(buffer + sIdx, AllocTable::Eof); 1459 | else 1460 | writeU32(buffer + sIdx, (uint32)mbat_blocks[blockIdx]); 1461 | sIdx += 4; 1462 | dcount = 0; 1463 | } 1464 | } 1465 | saveBigBlocks(mbat_blocks, 0, buffer, nBytes); 1466 | delete[] buffer; 1467 | mbatDirty = false; 1468 | } 1469 | file.flush(); 1470 | fileCheck(file); 1471 | 1472 | /* Note on Microsoft implementation: 1473 | - directory entries are stored in the last block(s) 1474 | - BATs are as second to the last 1475 | - Meta BATs are third to the last 1476 | */ 1477 | } 1478 | 1479 | void StorageIO::close() 1480 | { 1481 | if (!opened) return; 1482 | 1483 | file.close(); 1484 | opened = false; 1485 | 1486 | std::list::iterator it; 1487 | for (it = streams.begin(); it != streams.end(); ++it) 1488 | delete* it; 1489 | } 1490 | 1491 | 1492 | StreamIO* StorageIO::streamIO(const std::string& name, bool bCreate, int64 streamSize) 1493 | { 1494 | // sanity check 1495 | if (!name.length()) return (StreamIO*)0; 1496 | 1497 | // search in the entries 1498 | DirEntry* entry = dirtree->entry(name, bCreate, bbat->blockSize, this, streamSize); 1499 | //if( entry) std::cout << "FOUND\n"; 1500 | if (!entry) return (StreamIO*)0; 1501 | //if( !entry->dir ) std::cout << " NOT DIR\n"; 1502 | if (entry->dir) return (StreamIO*)0; 1503 | 1504 | StreamIO* result2 = new StreamIO(this, entry); 1505 | result2->fullName = name; 1506 | 1507 | return result2; 1508 | } 1509 | 1510 | bool StorageIO::deleteByName(const std::string& fullName) 1511 | { 1512 | if (!fullName.length()) 1513 | return false; 1514 | if (!writeable) 1515 | return false; 1516 | DirEntry* entry = dirtree->entry(fullName); 1517 | if (!entry) 1518 | return false; 1519 | bool retVal; 1520 | if (entry->dir) 1521 | retVal = deleteNode(entry, fullName); 1522 | else 1523 | retVal = deleteLeaf(entry, fullName); 1524 | if (retVal) 1525 | flush(); 1526 | return retVal; 1527 | } 1528 | 1529 | bool StorageIO::deleteNode(DirEntry* entry, const std::string& fullName) 1530 | { 1531 | std::string lclName = fullName; 1532 | if (lclName[lclName.size() - 1] != '/') 1533 | lclName += '/'; 1534 | bool retVal = true; 1535 | while (entry->child && entry->child < dirtree->entryCount()) 1536 | { 1537 | DirEntry* childEnt = dirtree->entry(entry->child); 1538 | std::string childFullName = lclName + childEnt->name; 1539 | if (childEnt->dir) 1540 | retVal = deleteNode(childEnt, childFullName); 1541 | else 1542 | retVal = deleteLeaf(childEnt, childFullName); 1543 | if (!retVal) 1544 | return false; 1545 | } 1546 | dirtree->deleteEntry(entry, fullName, bbat->blockSize); 1547 | return retVal; 1548 | } 1549 | 1550 | bool StorageIO::deleteLeaf(DirEntry* entry, const std::string& fullName) 1551 | { 1552 | std::vector blocks; 1553 | if (entry->size >= header->threshold) 1554 | { 1555 | blocks = bbat->follow(entry->start); 1556 | for (unsigned idx = 0; idx < blocks.size(); idx++) 1557 | { 1558 | bbat->set(blocks[idx], AllocTable::Avail); 1559 | bbat->markAsDirty(idx, bbat->blockSize); 1560 | } 1561 | } 1562 | else 1563 | { 1564 | blocks = sbat->follow(entry->start); 1565 | for (unsigned idx = 0; idx < blocks.size(); idx++) 1566 | { 1567 | sbat->set(blocks[idx], AllocTable::Avail); 1568 | sbat->markAsDirty(idx, bbat->blockSize); 1569 | } 1570 | } 1571 | dirtree->deleteEntry(entry, fullName, bbat->blockSize); 1572 | return true; 1573 | } 1574 | 1575 | uint64 StorageIO::loadBigBlocks(std::vector blocks, 1576 | unsigned char* data, uint64 maxlen) 1577 | { 1578 | // sentinel 1579 | if (!data) return 0; 1580 | fileCheck(file); 1581 | if (!file.good()) return 0; 1582 | if (blocks.size() < 1) return 0; 1583 | if (maxlen == 0) return 0; 1584 | 1585 | // read block one by one, seems fast enough 1586 | uint64 bytes = 0; 1587 | for (unsigned int i = 0; (i < blocks.size()) & (bytes < maxlen); i++) 1588 | { 1589 | uint64 block = blocks[i]; 1590 | uint64 pos = bbat->blockSize * (block + 1); 1591 | uint64 p = (bbat->blockSize < maxlen - bytes) ? bbat->blockSize : maxlen - bytes; 1592 | if (pos + p > filesize) 1593 | p = filesize - pos; 1594 | file.seekg(pos); 1595 | file.read((char*)data + bytes, p); 1596 | fileCheck(file); 1597 | // should use gcount to see how many bytes were really returned - eof check... 1598 | bytes += p; 1599 | } 1600 | 1601 | return bytes; 1602 | } 1603 | 1604 | uint64 StorageIO::loadBigBlock(uint64 block, 1605 | unsigned char* data, uint64 maxlen) 1606 | { 1607 | // sentinel 1608 | if (!data) return 0; 1609 | fileCheck(file); 1610 | if (!file.good()) return 0; 1611 | 1612 | // wraps call for loadBigBlocks 1613 | std::vector blocks; 1614 | blocks.resize(1); 1615 | blocks[0] = block; 1616 | 1617 | return loadBigBlocks(blocks, data, maxlen); 1618 | } 1619 | 1620 | uint64 StorageIO::saveBigBlocks(std::vector blocks, uint64 offset, unsigned char* data, uint64 len) 1621 | { 1622 | // sentinel 1623 | if (!data) return 0; 1624 | fileCheck(file); 1625 | if (!file.good()) return 0; 1626 | if (blocks.size() < 1) return 0; 1627 | if (len == 0) return 0; 1628 | 1629 | // write block one by one, seems fast enough 1630 | uint64 bytes = 0; 1631 | for (unsigned int i = 0; (i < blocks.size()) & (bytes < len); i++) 1632 | { 1633 | uint64 block = blocks[i]; 1634 | uint64 pos = (bbat->blockSize * (block + 1)) + offset; 1635 | uint64 maxWrite = bbat->blockSize - offset; 1636 | uint64 tobeWritten = len - bytes; 1637 | if (tobeWritten > maxWrite) 1638 | tobeWritten = maxWrite; 1639 | file.seekp(pos); 1640 | file.write((char*)data + bytes, tobeWritten); 1641 | fileCheck(file); 1642 | 1643 | bytes += tobeWritten; 1644 | offset = 0; 1645 | if (filesize < pos + tobeWritten) 1646 | filesize = pos + tobeWritten; 1647 | } 1648 | 1649 | return bytes; 1650 | 1651 | } 1652 | 1653 | uint64 StorageIO::saveBigBlock(uint64 block, uint64 offset, unsigned char* data, uint64 len) 1654 | { 1655 | if (!data) return 0; 1656 | fileCheck(file); 1657 | if (!file.good()) return 0; 1658 | //wrap call for saveBigBlocks 1659 | std::vector blocks; 1660 | blocks.resize(1); 1661 | blocks[0] = block; 1662 | return saveBigBlocks(blocks, offset, data, len); 1663 | } 1664 | 1665 | // return number of bytes which has been read 1666 | uint64 StorageIO::loadSmallBlocks(std::vector blocks, 1667 | unsigned char* data, uint64 maxlen) 1668 | { 1669 | // sentinel 1670 | if (!data) return 0; 1671 | fileCheck(file); 1672 | if (!file.good()) return 0; 1673 | if (blocks.size() < 1) return 0; 1674 | if (maxlen == 0) return 0; 1675 | 1676 | // our own local buffer 1677 | unsigned char* buf = new unsigned char[bbat->blockSize]; 1678 | 1679 | // read small block one by one 1680 | uint64 bytes = 0; 1681 | for (unsigned int i = 0; (i < blocks.size()) & (bytes < maxlen); i++) 1682 | { 1683 | uint64 block = blocks[i]; 1684 | 1685 | // find where the small-block exactly is 1686 | uint64 pos = block * sbat->blockSize; 1687 | uint64 bbindex = pos / bbat->blockSize; 1688 | if (bbindex >= sb_blocks.size()) break; 1689 | 1690 | loadBigBlock(sb_blocks[bbindex], buf, bbat->blockSize); 1691 | 1692 | // copy the data 1693 | uint64 offset = pos % bbat->blockSize; 1694 | uint64 p = (maxlen - bytes < bbat->blockSize - offset) ? maxlen - bytes : bbat->blockSize - offset; 1695 | p = (sbat->blockSize < p) ? sbat->blockSize : p; 1696 | memcpy(data + bytes, buf + offset, p); 1697 | bytes += p; 1698 | } 1699 | 1700 | delete[] buf; 1701 | 1702 | return bytes; 1703 | } 1704 | 1705 | uint64 StorageIO::loadSmallBlock(uint64 block, 1706 | unsigned char* data, uint64 maxlen) 1707 | { 1708 | // sentinel 1709 | if (!data) return 0; 1710 | fileCheck(file); 1711 | if (!file.good()) return 0; 1712 | 1713 | // wraps call for loadSmallBlocks 1714 | std::vector blocks; 1715 | blocks.resize(1); 1716 | blocks.assign(1, block); 1717 | 1718 | return loadSmallBlocks(blocks, data, maxlen); 1719 | } 1720 | 1721 | 1722 | uint64 StorageIO::saveSmallBlocks(std::vector blocks, uint64 offset, 1723 | unsigned char* data, uint64 len, int64 startAtBlock) 1724 | { 1725 | // sentinel 1726 | if (!data) return 0; 1727 | fileCheck(file); 1728 | if (!file.good()) return 0; 1729 | if (blocks.size() < 1) return 0; 1730 | if (len == 0) return 0; 1731 | 1732 | // write block one by one, seems fast enough 1733 | uint64 bytes = 0; 1734 | for (uint64 i = startAtBlock; (i < blocks.size()) & (bytes < len); i++) 1735 | { 1736 | uint64 block = blocks[i]; 1737 | // find where the small-block exactly is 1738 | uint64 pos = block * sbat->blockSize; 1739 | uint64 bbindex = pos / bbat->blockSize; 1740 | if (bbindex >= sb_blocks.size()) break; 1741 | uint64 offset2 = pos % bbat->blockSize; 1742 | uint64 maxWrite = sbat->blockSize - offset; 1743 | uint64 tobeWritten = len - bytes; 1744 | if (tobeWritten > maxWrite) 1745 | tobeWritten = maxWrite; 1746 | saveBigBlock(sb_blocks[bbindex], offset2 + offset, data + bytes, tobeWritten); 1747 | bytes += tobeWritten; 1748 | offset = 0; 1749 | if (filesize < pos + tobeWritten) 1750 | filesize = pos + tobeWritten; 1751 | } 1752 | return bytes; 1753 | } 1754 | 1755 | uint64 StorageIO::saveSmallBlock(uint64 block, uint64 offset, unsigned char* data, uint64 len) 1756 | { 1757 | if (!data) return 0; 1758 | fileCheck(file); 1759 | if (!file.good()) return 0; 1760 | //wrap call for saveSmallBlocks 1761 | std::vector blocks; 1762 | blocks.resize(1); 1763 | blocks[0] = block; 1764 | return saveSmallBlocks(blocks, offset, data, len); 1765 | } 1766 | 1767 | void StorageIO::flushbbat() 1768 | { 1769 | std::vector blocks; 1770 | blocks = getbbatBlocks(false); 1771 | bbat->flush(blocks, this, bbat->blockSize); 1772 | } 1773 | 1774 | void StorageIO::flushsbat() 1775 | { 1776 | std::vector blocks; 1777 | blocks = bbat->follow(header->sbat_start); 1778 | sbat->flush(blocks, this, bbat->blockSize); 1779 | } 1780 | 1781 | std::vector StorageIO::getbbatBlocks(bool bLoading) 1782 | { 1783 | std::vector blocks; 1784 | // find blocks allocated to store big bat 1785 | // the first 109 blocks are in header, the rest in meta bat 1786 | blocks.clear(); 1787 | blocks.resize(header->num_bat); 1788 | 1789 | for (unsigned i = 0; i < 109; i++) 1790 | { 1791 | if (i >= header->num_bat) 1792 | break; 1793 | else 1794 | blocks[i] = header->bb_blocks[i]; 1795 | } 1796 | if (bLoading) 1797 | { 1798 | mbat_blocks.clear(); 1799 | mbat_data.clear(); 1800 | if ((header->num_bat > 109) && (header->num_mbat > 0)) 1801 | { 1802 | unsigned char* buffer2 = new unsigned char[bbat->blockSize]; 1803 | uint64 k = 109; 1804 | uint64 sector; 1805 | uint64 mdidx = 0; 1806 | for (uint64 r = 0; r < header->num_mbat; r++) 1807 | { 1808 | if (r == 0) // 1st meta bat location is in file header. 1809 | sector = header->mbat_start; 1810 | else // next meta bat location is the last current block value. 1811 | { 1812 | sector = blocks[--k]; 1813 | mdidx--; 1814 | } 1815 | mbat_blocks.push_back(sector); 1816 | mbat_data.resize(mbat_blocks.size() * (bbat->blockSize / 4)); 1817 | loadBigBlock(sector, buffer2, bbat->blockSize); 1818 | for (uint64 s = 0; s < bbat->blockSize; s += 4) 1819 | { 1820 | if (k >= header->num_bat) 1821 | break; 1822 | else 1823 | { 1824 | blocks[k] = readU32(buffer2 + s); 1825 | mbat_data[mdidx++] = blocks[k]; 1826 | k++; 1827 | } 1828 | } 1829 | } 1830 | if (mbat_data.size() != mdidx) mbat_data.resize(mdidx); 1831 | delete[] buffer2; 1832 | } 1833 | } 1834 | else 1835 | { 1836 | unsigned i = 109; 1837 | for (unsigned int idx = 0; idx < mbat_data.size(); idx++) 1838 | { 1839 | blocks[i++] = mbat_data[idx]; 1840 | if (i == header->num_bat) 1841 | break; 1842 | } 1843 | } 1844 | return blocks; 1845 | } 1846 | 1847 | uint64 StorageIO::ExtendFile(std::vector* chain) 1848 | { 1849 | uint64 newblockIdx = bbat->unused(); 1850 | bbat->set(newblockIdx, AllocTable::Eof); 1851 | uint64 bbidx = newblockIdx / (bbat->blockSize / sizeof(uint64)); 1852 | while (bbidx >= header->num_bat) 1853 | addbbatBlock(); 1854 | bbat->markAsDirty(newblockIdx, bbat->blockSize); 1855 | if (chain->size() > 0) 1856 | { 1857 | bbat->set((*chain)[chain->size() - 1], newblockIdx); 1858 | bbat->markAsDirty((*chain)[chain->size() - 1], bbat->blockSize); 1859 | } 1860 | chain->push_back(newblockIdx); 1861 | return newblockIdx; 1862 | } 1863 | 1864 | void StorageIO::addbbatBlock() 1865 | { 1866 | uint64 newblockIdx = bbat->unused(); 1867 | bbat->set(newblockIdx, AllocTable::MetaBat); 1868 | 1869 | if (header->num_bat < 109) 1870 | header->bb_blocks[header->num_bat] = newblockIdx; 1871 | else 1872 | { 1873 | mbatDirty = true; 1874 | mbat_data.push_back(newblockIdx); 1875 | uint64 metaIdx = header->num_bat - 109; 1876 | uint64 idxPerBlock = bbat->blockSize / sizeof(uint64) - 1; //reserve room for index to next block 1877 | uint64 idxBlock = metaIdx / idxPerBlock; 1878 | if (idxBlock == mbat_blocks.size()) 1879 | { 1880 | uint64 newmetaIdx = bbat->unused(); 1881 | bbat->set(newmetaIdx, AllocTable::MetaBat); 1882 | mbat_blocks.push_back(newmetaIdx); 1883 | if (header->num_mbat == 0) 1884 | header->mbat_start = newmetaIdx; 1885 | header->num_mbat++; 1886 | } 1887 | } 1888 | header->num_bat++; 1889 | header->dirty = true; 1890 | } 1891 | 1892 | 1893 | // =========== StreamIO ========== 1894 | 1895 | StreamIO::StreamIO(StorageIO* s, DirEntry* e) 1896 | : io(s), 1897 | entryIdx(io->dirtree->indexOf(e)), 1898 | fullName(), 1899 | blocks(), 1900 | eof(false), 1901 | fail(false), 1902 | m_pos(0), 1903 | cache_data(new unsigned char[CACHEBUFSIZE]), 1904 | cache_size(0), // indicating an empty cache 1905 | cache_pos(0) 1906 | { 1907 | if (e->size >= io->header->threshold) 1908 | blocks = io->bbat->follow(e->start); 1909 | else 1910 | blocks = io->sbat->follow(e->start); 1911 | } 1912 | 1913 | // FIXME tell parent we're gone 1914 | StreamIO::~StreamIO() 1915 | { 1916 | delete[] cache_data; 1917 | } 1918 | 1919 | void StreamIO::setSize(uint64 newSize) 1920 | { 1921 | bool bThresholdCrossed = false; 1922 | bool bOver = false; 1923 | 1924 | if (!io->writeable) 1925 | return; 1926 | DirEntry* entry = io->dirtree->entry(entryIdx); 1927 | if (newSize >= io->header->threshold && entry->size < io->header->threshold) 1928 | { 1929 | bThresholdCrossed = true; 1930 | bOver = true; 1931 | } 1932 | else if (newSize < io->header->threshold && entry->size >= io->header->threshold) 1933 | { 1934 | bThresholdCrossed = true; 1935 | bOver = false; 1936 | } 1937 | if (bThresholdCrossed) 1938 | { 1939 | // first, read what is already in the stream, limited by the requested new size. Note 1940 | // that the read can work precisely because we have not yet reset the size. 1941 | uint64 len = newSize; 1942 | if (len > entry->size) 1943 | len = entry->size; 1944 | unsigned char* buffer = 0; 1945 | uint64 savePos = tell(); 1946 | if (len) 1947 | { 1948 | buffer = new unsigned char[len]; 1949 | seek(0); 1950 | read(buffer, len); 1951 | } 1952 | // Now get rid of the existing blocks 1953 | if (bOver) 1954 | { 1955 | for (unsigned int idx = 0; idx < blocks.size(); idx++) 1956 | { 1957 | io->sbat->set(blocks[idx], AllocTable::Avail); 1958 | io->sbat->markAsDirty(idx, io->bbat->blockSize); 1959 | } 1960 | } 1961 | else 1962 | { 1963 | for (unsigned int idx = 0; idx < blocks.size(); idx++) 1964 | { 1965 | io->bbat->set(blocks[idx], AllocTable::Avail); 1966 | io->bbat->markAsDirty(idx, io->bbat->blockSize); 1967 | } 1968 | } 1969 | blocks.clear(); 1970 | entry->start = DirTree::End; 1971 | // Now change the size, and write the old data back into the stream, if any 1972 | entry->size = newSize; 1973 | io->dirtree->markAsDirty(io->dirtree->indexOf(entry), io->bbat->blockSize); 1974 | if (len) 1975 | { 1976 | write(0, buffer, len); 1977 | delete[] buffer; 1978 | } 1979 | if (savePos <= entry->size) 1980 | seek(savePos); 1981 | } 1982 | else if (entry->size != newSize) //simple case - no threshold was crossed, so just change the size 1983 | { 1984 | entry->size = newSize; 1985 | io->dirtree->markAsDirty(io->dirtree->indexOf(entry), io->bbat->blockSize); 1986 | } 1987 | 1988 | } 1989 | 1990 | void StreamIO::seek(uint64 pos) 1991 | { 1992 | m_pos = pos; 1993 | } 1994 | 1995 | uint64 StreamIO::tell() 1996 | { 1997 | return m_pos; 1998 | } 1999 | 2000 | int64 StreamIO::getch() 2001 | { 2002 | // past end-of-file ? 2003 | DirEntry* entry = io->dirtree->entry(entryIdx); 2004 | if (m_pos >= entry->size) return -1; 2005 | 2006 | // need to update cache ? 2007 | if (!cache_size || (m_pos < cache_pos) || 2008 | (m_pos >= cache_pos + cache_size)) 2009 | updateCache(); 2010 | 2011 | // something bad if we don't get good cache 2012 | if (!cache_size) return -1; 2013 | 2014 | int64 data = cache_data[m_pos - cache_pos]; 2015 | m_pos++; 2016 | 2017 | return data; 2018 | } 2019 | 2020 | uint64 StreamIO::read(uint64 pos, unsigned char* data, uint64 maxlen) 2021 | { 2022 | // sanity checks 2023 | if (!data) return 0; 2024 | if (maxlen == 0) return 0; 2025 | 2026 | uint64 totalbytes = 0; 2027 | 2028 | DirEntry* entry = io->dirtree->entry(entryIdx); 2029 | if (pos + maxlen > entry->size) 2030 | maxlen = entry->size - pos; 2031 | if (entry->size < io->header->threshold) 2032 | { 2033 | // small file 2034 | uint64 index = pos / io->sbat->blockSize; 2035 | 2036 | if (index >= blocks.size()) return 0; 2037 | 2038 | unsigned char* buf = new unsigned char[io->sbat->blockSize]; 2039 | uint64 offset = pos % io->sbat->blockSize; 2040 | while (totalbytes < maxlen) 2041 | { 2042 | if (index >= blocks.size()) break; 2043 | io->loadSmallBlock(blocks[index], buf, io->bbat->blockSize); 2044 | uint64 count = io->sbat->blockSize - offset; 2045 | if (count > maxlen - totalbytes) count = maxlen - totalbytes; 2046 | memcpy(data + totalbytes, buf + offset, count); 2047 | totalbytes += count; 2048 | offset = 0; 2049 | index++; 2050 | } 2051 | delete[] buf; 2052 | 2053 | } 2054 | else 2055 | { 2056 | // big file 2057 | uint64 index = pos / io->bbat->blockSize; 2058 | 2059 | if (index >= blocks.size()) return 0; 2060 | 2061 | unsigned char* buf = new unsigned char[io->bbat->blockSize]; 2062 | uint64 offset = pos % io->bbat->blockSize; 2063 | while (totalbytes < maxlen) 2064 | { 2065 | if (index >= blocks.size()) break; 2066 | io->loadBigBlock(blocks[index], buf, io->bbat->blockSize); 2067 | uint64 count = io->bbat->blockSize - offset; 2068 | if (count > maxlen - totalbytes) count = maxlen - totalbytes; 2069 | memcpy(data + totalbytes, buf + offset, count); 2070 | totalbytes += count; 2071 | index++; 2072 | offset = 0; 2073 | } 2074 | delete[] buf; 2075 | 2076 | } 2077 | 2078 | return totalbytes; 2079 | } 2080 | 2081 | uint64 StreamIO::read(unsigned char* data, uint64 maxlen) 2082 | { 2083 | uint64 bytes = read(tell(), data, maxlen); 2084 | m_pos += bytes; 2085 | return bytes; 2086 | } 2087 | 2088 | uint64 StreamIO::write(unsigned char* data, uint64 len) 2089 | { 2090 | return write(tell(), data, len); 2091 | } 2092 | 2093 | uint64 StreamIO::write(uint64 pos, unsigned char* data, uint64 len) 2094 | { 2095 | // sanity checks 2096 | if (!data) return 0; 2097 | if (len == 0) return 0; 2098 | if (!io->writeable) return 0; 2099 | 2100 | DirEntry* entry = io->dirtree->entry(entryIdx); 2101 | if (pos + len > entry->size) 2102 | setSize(pos + len); //reset size, possibly changing from small to large blocks 2103 | uint64 totalbytes = 0; 2104 | if (entry->size < io->header->threshold) 2105 | { 2106 | // small file 2107 | uint64 index = (pos + len - 1) / io->sbat->blockSize; 2108 | while (index >= blocks.size()) 2109 | { 2110 | uint64 nblock = io->sbat->unused(); 2111 | if (blocks.size() > 0) 2112 | { 2113 | io->sbat->set(blocks[blocks.size() - 1], nblock); 2114 | io->sbat->markAsDirty(blocks[blocks.size() - 1], io->bbat->blockSize); 2115 | } 2116 | io->sbat->set(nblock, AllocTable::Eof); 2117 | io->sbat->markAsDirty(nblock, io->bbat->blockSize); 2118 | blocks.resize(blocks.size() + 1); 2119 | blocks[blocks.size() - 1] = nblock; 2120 | uint64 bbidx = nblock / (io->bbat->blockSize / sizeof(unsigned int)); 2121 | while (bbidx >= io->header->num_sbat) 2122 | { 2123 | std::vector sbat_blocks = io->bbat->follow(io->header->sbat_start); 2124 | io->ExtendFile(&sbat_blocks); 2125 | io->header->num_sbat++; 2126 | io->header->dirty = true; //Header will have to be rewritten 2127 | } 2128 | uint64 sidx = nblock * io->sbat->blockSize / io->bbat->blockSize; 2129 | while (sidx >= io->sb_blocks.size()) 2130 | { 2131 | io->ExtendFile(&io->sb_blocks); 2132 | io->dirtree->markAsDirty(0, io->bbat->blockSize); //make sure to rewrite first directory block 2133 | } 2134 | } 2135 | uint64 offset = pos % io->sbat->blockSize; 2136 | index = pos / io->sbat->blockSize; 2137 | //if (index == 0) 2138 | totalbytes = io->saveSmallBlocks(blocks, offset, data, len, index); 2139 | } 2140 | else 2141 | { 2142 | uint64 index = (pos + len - 1) / io->bbat->blockSize; 2143 | while (index >= blocks.size()) 2144 | io->ExtendFile(&blocks); 2145 | uint64 offset = pos % io->bbat->blockSize; 2146 | uint64 remainder = len; 2147 | index = pos / io->bbat->blockSize; 2148 | while (remainder > 0) 2149 | { 2150 | if (index >= blocks.size()) break; 2151 | uint64 count = io->bbat->blockSize - offset; 2152 | if (remainder < count) 2153 | count = remainder; 2154 | io->saveBigBlock(blocks[index], offset, data + totalbytes, count); 2155 | totalbytes += count; 2156 | remainder -= count; 2157 | index++; 2158 | offset = 0; 2159 | } 2160 | } 2161 | if (blocks.size() > 0 && entry->start != blocks[0]) 2162 | { 2163 | entry->start = blocks[0]; 2164 | io->dirtree->markAsDirty(io->dirtree->indexOf(entry), io->bbat->blockSize); 2165 | } 2166 | m_pos += len; 2167 | return totalbytes; 2168 | } 2169 | 2170 | void StreamIO::flush() 2171 | { 2172 | io->flush(); 2173 | } 2174 | 2175 | void StreamIO::updateCache() 2176 | { 2177 | // sanity check 2178 | if (!cache_data) return; 2179 | 2180 | DirEntry* entry = io->dirtree->entry(entryIdx); 2181 | cache_pos = m_pos - (m_pos % CACHEBUFSIZE); 2182 | uint64 bytes = CACHEBUFSIZE; 2183 | if (cache_pos + bytes > entry->size) bytes = entry->size - cache_pos; 2184 | cache_size = read(cache_pos, cache_data, bytes); 2185 | } 2186 | 2187 | 2188 | // =========== Storage ========== 2189 | 2190 | Storage::Storage(const char* filename) 2191 | { 2192 | io = new StorageIO(this, filename); 2193 | } 2194 | 2195 | Storage::~Storage() 2196 | { 2197 | delete io; 2198 | } 2199 | 2200 | int Storage::result() 2201 | { 2202 | return (int)io->result; 2203 | } 2204 | 2205 | bool Storage::open(bool bWriteAccess, bool bCreate) 2206 | { 2207 | return io->open(bWriteAccess, bCreate); 2208 | } 2209 | 2210 | void Storage::close() 2211 | { 2212 | io->close(); 2213 | } 2214 | 2215 | std::list Storage::entries(const std::string& path) 2216 | { 2217 | std::list localResult; 2218 | DirTree* dt = io->dirtree; 2219 | DirEntry* e = dt->entry(path, false); 2220 | if (e && e->dir) 2221 | { 2222 | uint64 parent = dt->indexOf(e); 2223 | std::vector children = dt->children(parent); 2224 | for (uint64 i = 0; i < children.size(); i++) 2225 | localResult.push_back(dt->entry(children[i])->name); 2226 | } 2227 | 2228 | return localResult; 2229 | } 2230 | 2231 | bool Storage::isDirectory(const std::string& name) 2232 | { 2233 | DirEntry* e = io->dirtree->entry(name, false); 2234 | return e ? e->dir : false; 2235 | } 2236 | 2237 | bool Storage::exists(const std::string& name) 2238 | { 2239 | DirEntry* e = io->dirtree->entry(name, false); 2240 | return (e != 0); 2241 | } 2242 | 2243 | bool Storage::isWriteable() 2244 | { 2245 | return io->writeable; 2246 | } 2247 | 2248 | bool Storage::deleteByName(const std::string& name) 2249 | { 2250 | return io->deleteByName(name); 2251 | } 2252 | 2253 | void Storage::GetStats(uint64* pEntries, uint64* pUnusedEntries, 2254 | uint64* pBigBlocks, uint64* pUnusedBigBlocks, 2255 | uint64* pSmallBlocks, uint64* pUnusedSmallBlocks) 2256 | { 2257 | *pEntries = io->dirtree->entryCount(); 2258 | *pUnusedEntries = io->dirtree->unusedEntryCount(); 2259 | *pBigBlocks = io->bbat->count(); 2260 | *pUnusedBigBlocks = io->bbat->unusedCount(); 2261 | *pSmallBlocks = io->sbat->count(); 2262 | *pUnusedSmallBlocks = io->sbat->unusedCount(); 2263 | } 2264 | 2265 | // recursively collect stream names 2266 | void CollectStreams(std::list& result, DirTree* tree, DirEntry* parent, const std::string& path) 2267 | { 2268 | DirEntry* c = tree->entry(parent->child); 2269 | std::queue queue; 2270 | if (c) queue.push(c); 2271 | while (!queue.empty()) { 2272 | DirEntry* e = queue.front(); 2273 | queue.pop(); 2274 | if (e->dir) 2275 | CollectStreams(result, tree, e, path + e->name + "/"); 2276 | else 2277 | result.push_back(path + e->name); 2278 | DirEntry* p = tree->entry(e->prev); 2279 | if (p) queue.push(p); 2280 | DirEntry* n = tree->entry(e->next); 2281 | if (n) queue.push(n); 2282 | // not testing if p or n have already been processed; potential infinite loop in case of closed Entry chain 2283 | // it seems not to happen, though 2284 | } 2285 | } 2286 | 2287 | std::list Storage::GetAllStreams(const std::string& storageName) 2288 | { 2289 | std::list vresult; 2290 | DirEntry* e = io->dirtree->entry(storageName, false); 2291 | if (e && e->dir) CollectStreams(vresult, io->dirtree, e, storageName); 2292 | return vresult; 2293 | } 2294 | 2295 | // =========== Stream ========== 2296 | 2297 | Stream::Stream(Storage* storage, const std::string& name, bool bCreate, int64 streamSize) 2298 | : io(storage->io->streamIO(name, bCreate, (int)streamSize)) 2299 | { 2300 | } 2301 | 2302 | // FIXME tell parent we're gone 2303 | Stream::~Stream() 2304 | { 2305 | delete io; 2306 | } 2307 | 2308 | std::string Stream::fullName() 2309 | { 2310 | return io ? io->fullName : std::string(); 2311 | } 2312 | 2313 | uint64 Stream::tell() 2314 | { 2315 | return io ? io->tell() : 0; 2316 | } 2317 | 2318 | void Stream::seek(uint64 newpos) 2319 | { 2320 | if (io) 2321 | io->seek(newpos); 2322 | } 2323 | 2324 | uint64 Stream::size() 2325 | { 2326 | if (!io) 2327 | return 0; 2328 | DirEntry* entry = io->io->dirtree->entry(io->entryIdx); 2329 | return entry->size; 2330 | } 2331 | 2332 | void Stream::setSize(int64 newSize) 2333 | { 2334 | if (!io) 2335 | return; 2336 | if (newSize < 0) 2337 | return; 2338 | if (newSize > std::numeric_limits::max()) 2339 | return; 2340 | io->setSize(newSize); 2341 | } 2342 | 2343 | int64 Stream::getch() 2344 | { 2345 | return io ? io->getch() : 0; 2346 | } 2347 | 2348 | uint64 Stream::read(unsigned char* data, uint64 maxlen) 2349 | { 2350 | return io ? io->read(data, maxlen) : 0; 2351 | } 2352 | 2353 | uint64 Stream::write(unsigned char* data, uint64 len) 2354 | { 2355 | return io ? io->write(data, len) : 0; 2356 | } 2357 | 2358 | void Stream::flush() 2359 | { 2360 | if (io) 2361 | io->flush(); 2362 | } 2363 | 2364 | bool Stream::eof() 2365 | { 2366 | return io ? io->eof : false; 2367 | } 2368 | 2369 | bool Stream::fail() 2370 | { 2371 | return io ? io->fail : true; 2372 | } 2373 | --------------------------------------------------------------------------------