├── README.md ├── SlickBackup ├── SlickBackup.csproj ├── Program.cs └── BackupEngine.cs ├── TestBackup ├── Extensions.cs ├── TestBackup.csproj └── BackupTestClass.cs ├── LICENSE ├── SlickBackup.sln └── .gitignore /README.md: -------------------------------------------------------------------------------- 1 | # SlickBackup -------------------------------------------------------------------------------- /SlickBackup/SlickBackup.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | net5.0 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /TestBackup/Extensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace TestBackup 8 | { 9 | static class Extensions 10 | { 11 | public static void Shuffle(this Random rng, T[] array) 12 | { 13 | int n = array.Length; 14 | while (n > 1) 15 | { 16 | int k = rng.Next(n--); 17 | T temp = array[n]; 18 | array[n] = array[k]; 19 | array[k] = temp; 20 | } 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /TestBackup/TestBackup.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net5.0 5 | 6 | false 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 g3gg0.de 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /SlickBackup.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 16 4 | VisualStudioVersion = 16.0.31515.178 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SlickBackup", "SlickBackup\SlickBackup.csproj", "{C94CEF9A-5B6F-4E10-A0A8-740C7C8F6DAB}" 7 | EndProject 8 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestBackup", "TestBackup\TestBackup.csproj", "{A0B4C70B-7AAD-44AD-8F1C-30523629BC09}" 9 | EndProject 10 | Global 11 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 12 | Debug|Any CPU = Debug|Any CPU 13 | Release|Any CPU = Release|Any CPU 14 | EndGlobalSection 15 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 16 | {C94CEF9A-5B6F-4E10-A0A8-740C7C8F6DAB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 17 | {C94CEF9A-5B6F-4E10-A0A8-740C7C8F6DAB}.Debug|Any CPU.Build.0 = Debug|Any CPU 18 | {C94CEF9A-5B6F-4E10-A0A8-740C7C8F6DAB}.Release|Any CPU.ActiveCfg = Release|Any CPU 19 | {C94CEF9A-5B6F-4E10-A0A8-740C7C8F6DAB}.Release|Any CPU.Build.0 = Release|Any CPU 20 | {A0B4C70B-7AAD-44AD-8F1C-30523629BC09}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 21 | {A0B4C70B-7AAD-44AD-8F1C-30523629BC09}.Debug|Any CPU.Build.0 = Debug|Any CPU 22 | {A0B4C70B-7AAD-44AD-8F1C-30523629BC09}.Release|Any CPU.ActiveCfg = Release|Any CPU 23 | {A0B4C70B-7AAD-44AD-8F1C-30523629BC09}.Release|Any CPU.Build.0 = Release|Any CPU 24 | EndGlobalSection 25 | GlobalSection(SolutionProperties) = preSolution 26 | HideSolutionNode = FALSE 27 | EndGlobalSection 28 | GlobalSection(ExtensibilityGlobals) = postSolution 29 | SolutionGuid = {8C423271-E162-4035-AF34-CFF158A5DC2C} 30 | EndGlobalSection 31 | EndGlobal 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.rsuser 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Mono auto generated files 17 | mono_crash.* 18 | 19 | # Build results 20 | [Dd]ebug/ 21 | [Dd]ebugPublic/ 22 | [Rr]elease/ 23 | [Rr]eleases/ 24 | x64/ 25 | x86/ 26 | [Aa][Rr][Mm]/ 27 | [Aa][Rr][Mm]64/ 28 | bld/ 29 | [Bb]in/ 30 | [Oo]bj/ 31 | [Ll]og/ 32 | [Ll]ogs/ 33 | 34 | # Visual Studio 2015/2017 cache/options directory 35 | .vs/ 36 | # Uncomment if you have tasks that create the project's static files in wwwroot 37 | #wwwroot/ 38 | 39 | # Visual Studio 2017 auto generated files 40 | Generated\ Files/ 41 | 42 | # MSTest test Results 43 | [Tt]est[Rr]esult*/ 44 | [Bb]uild[Ll]og.* 45 | 46 | # NUnit 47 | *.VisualState.xml 48 | TestResult.xml 49 | nunit-*.xml 50 | 51 | # Build Results of an ATL Project 52 | [Dd]ebugPS/ 53 | [Rr]eleasePS/ 54 | dlldata.c 55 | 56 | # Benchmark Results 57 | BenchmarkDotNet.Artifacts/ 58 | 59 | # .NET Core 60 | project.lock.json 61 | project.fragment.lock.json 62 | artifacts/ 63 | 64 | # StyleCop 65 | StyleCopReport.xml 66 | 67 | # Files built by Visual Studio 68 | *_i.c 69 | *_p.c 70 | *_h.h 71 | *.ilk 72 | *.meta 73 | *.obj 74 | *.iobj 75 | *.pch 76 | *.pdb 77 | *.ipdb 78 | *.pgc 79 | *.pgd 80 | *.rsp 81 | *.sbr 82 | *.tlb 83 | *.tli 84 | *.tlh 85 | *.tmp 86 | *.tmp_proj 87 | *_wpftmp.csproj 88 | *.log 89 | *.vspscc 90 | *.vssscc 91 | .builds 92 | *.pidb 93 | *.svclog 94 | *.scc 95 | 96 | # Chutzpah Test files 97 | _Chutzpah* 98 | 99 | # Visual C++ cache files 100 | ipch/ 101 | *.aps 102 | *.ncb 103 | *.opendb 104 | *.opensdf 105 | *.sdf 106 | *.cachefile 107 | *.VC.db 108 | *.VC.VC.opendb 109 | 110 | # Visual Studio profiler 111 | *.psess 112 | *.vsp 113 | *.vspx 114 | *.sap 115 | 116 | # Visual Studio Trace Files 117 | *.e2e 118 | 119 | # TFS 2012 Local Workspace 120 | $tf/ 121 | 122 | # Guidance Automation Toolkit 123 | *.gpState 124 | 125 | # ReSharper is a .NET coding add-in 126 | _ReSharper*/ 127 | *.[Rr]e[Ss]harper 128 | *.DotSettings.user 129 | 130 | # TeamCity is a build add-in 131 | _TeamCity* 132 | 133 | # DotCover is a Code Coverage Tool 134 | *.dotCover 135 | 136 | # AxoCover is a Code Coverage Tool 137 | .axoCover/* 138 | !.axoCover/settings.json 139 | 140 | # Visual Studio code coverage results 141 | *.coverage 142 | *.coveragexml 143 | 144 | # NCrunch 145 | _NCrunch_* 146 | .*crunch*.local.xml 147 | nCrunchTemp_* 148 | 149 | # MightyMoose 150 | *.mm.* 151 | AutoTest.Net/ 152 | 153 | # Web workbench (sass) 154 | .sass-cache/ 155 | 156 | # Installshield output folder 157 | [Ee]xpress/ 158 | 159 | # DocProject is a documentation generator add-in 160 | DocProject/buildhelp/ 161 | DocProject/Help/*.HxT 162 | DocProject/Help/*.HxC 163 | DocProject/Help/*.hhc 164 | DocProject/Help/*.hhk 165 | DocProject/Help/*.hhp 166 | DocProject/Help/Html2 167 | DocProject/Help/html 168 | 169 | # Click-Once directory 170 | publish/ 171 | 172 | # Publish Web Output 173 | *.[Pp]ublish.xml 174 | *.azurePubxml 175 | # Note: Comment the next line if you want to checkin your web deploy settings, 176 | # but database connection strings (with potential passwords) will be unencrypted 177 | *.pubxml 178 | *.publishproj 179 | 180 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 181 | # checkin your Azure Web App publish settings, but sensitive information contained 182 | # in these scripts will be unencrypted 183 | PublishScripts/ 184 | 185 | # NuGet Packages 186 | *.nupkg 187 | # NuGet Symbol Packages 188 | *.snupkg 189 | # The packages folder can be ignored because of Package Restore 190 | **/[Pp]ackages/* 191 | # except build/, which is used as an MSBuild target. 192 | !**/[Pp]ackages/build/ 193 | # Uncomment if necessary however generally it will be regenerated when needed 194 | #!**/[Pp]ackages/repositories.config 195 | # NuGet v3's project.json files produces more ignorable files 196 | *.nuget.props 197 | *.nuget.targets 198 | 199 | # Microsoft Azure Build Output 200 | csx/ 201 | *.build.csdef 202 | 203 | # Microsoft Azure Emulator 204 | ecf/ 205 | rcf/ 206 | 207 | # Windows Store app package directories and files 208 | AppPackages/ 209 | BundleArtifacts/ 210 | Package.StoreAssociation.xml 211 | _pkginfo.txt 212 | *.appx 213 | *.appxbundle 214 | *.appxupload 215 | 216 | # Visual Studio cache files 217 | # files ending in .cache can be ignored 218 | *.[Cc]ache 219 | # but keep track of directories ending in .cache 220 | !?*.[Cc]ache/ 221 | 222 | # Others 223 | ClientBin/ 224 | ~$* 225 | *~ 226 | *.dbmdl 227 | *.dbproj.schemaview 228 | *.jfm 229 | *.pfx 230 | *.publishsettings 231 | orleans.codegen.cs 232 | 233 | # Including strong name files can present a security risk 234 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 235 | #*.snk 236 | 237 | # Since there are multiple workflows, uncomment next line to ignore bower_components 238 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 239 | #bower_components/ 240 | 241 | # RIA/Silverlight projects 242 | Generated_Code/ 243 | 244 | # Backup & report files from converting an old project file 245 | # to a newer Visual Studio version. Backup files are not needed, 246 | # because we have git ;-) 247 | _UpgradeReport_Files/ 248 | Backup*/ 249 | UpgradeLog*.XML 250 | UpgradeLog*.htm 251 | ServiceFabricBackup/ 252 | *.rptproj.bak 253 | 254 | # SQL Server files 255 | *.mdf 256 | *.ldf 257 | *.ndf 258 | 259 | # Business Intelligence projects 260 | *.rdl.data 261 | *.bim.layout 262 | *.bim_*.settings 263 | *.rptproj.rsuser 264 | *- [Bb]ackup.rdl 265 | *- [Bb]ackup ([0-9]).rdl 266 | *- [Bb]ackup ([0-9][0-9]).rdl 267 | 268 | # Microsoft Fakes 269 | FakesAssemblies/ 270 | 271 | # GhostDoc plugin setting file 272 | *.GhostDoc.xml 273 | 274 | # Node.js Tools for Visual Studio 275 | .ntvs_analysis.dat 276 | node_modules/ 277 | 278 | # Visual Studio 6 build log 279 | *.plg 280 | 281 | # Visual Studio 6 workspace options file 282 | *.opt 283 | 284 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 285 | *.vbw 286 | 287 | # Visual Studio LightSwitch build output 288 | **/*.HTMLClient/GeneratedArtifacts 289 | **/*.DesktopClient/GeneratedArtifacts 290 | **/*.DesktopClient/ModelManifest.xml 291 | **/*.Server/GeneratedArtifacts 292 | **/*.Server/ModelManifest.xml 293 | _Pvt_Extensions 294 | 295 | # Paket dependency manager 296 | .paket/paket.exe 297 | paket-files/ 298 | 299 | # FAKE - F# Make 300 | .fake/ 301 | 302 | # CodeRush personal settings 303 | .cr/personal 304 | 305 | # Python Tools for Visual Studio (PTVS) 306 | __pycache__/ 307 | *.pyc 308 | 309 | # Cake - Uncomment if you are using it 310 | # tools/** 311 | # !tools/packages.config 312 | 313 | # Tabs Studio 314 | *.tss 315 | 316 | # Telerik's JustMock configuration file 317 | *.jmconfig 318 | 319 | # BizTalk build output 320 | *.btp.cs 321 | *.btm.cs 322 | *.odx.cs 323 | *.xsd.cs 324 | 325 | # OpenCover UI analysis results 326 | OpenCover/ 327 | 328 | # Azure Stream Analytics local run output 329 | ASALocalRun/ 330 | 331 | # MSBuild Binary and Structured Log 332 | *.binlog 333 | 334 | # NVidia Nsight GPU debugger configuration file 335 | *.nvuser 336 | 337 | # MFractors (Xamarin productivity tool) working folder 338 | .mfractor/ 339 | 340 | # Local History for Visual Studio 341 | .localhistory/ 342 | 343 | # BeatPulse healthcheck temp database 344 | healthchecksdb 345 | 346 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 347 | MigrationBackup/ 348 | 349 | # Ionide (cross platform F# VS Code tools) working folder 350 | .ionide/ 351 | -------------------------------------------------------------------------------- /SlickBackup/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Linq; 4 | using System.Runtime.CompilerServices; 5 | using System.Runtime.Serialization; 6 | using System.Text; 7 | using System.Threading; 8 | using System.Xml.Serialization; 9 | 10 | [assembly: InternalsVisibleTo("TestBackup.BackupTestClass")] 11 | [assembly: InternalsVisibleTo("TestBackup")] 12 | [assembly: InternalsVisibleTo("BackupTestClass")] 13 | namespace SlickBackup 14 | { 15 | public class Program 16 | { 17 | private static StreamWriter LogFile = null; 18 | 19 | public class Backup 20 | { 21 | public string Title; 22 | public string Source; 23 | public string Destination; 24 | public string IgnoreList; 25 | public int Reindex = 10; 26 | public int AutoSave = 600; 27 | } 28 | 29 | public class BackupConfig 30 | { 31 | public Backup[] Backups; 32 | } 33 | 34 | 35 | static void Main(string[] args) 36 | { 37 | BackupConfig config = null; 38 | BackupEngine engine = null; 39 | Thread scanThread = null; 40 | bool done = false; 41 | string logFileName = "SlickBackup_" + DateTime.Now.ToString("yyyy.MM.dd_HH.mm.ss") + ".log"; 42 | 43 | LogFile = File.CreateText(logFileName); 44 | 45 | Console.CancelKeyPress += delegate (object? sender, ConsoleCancelEventArgs e) 46 | { 47 | e.Cancel = true; 48 | 49 | if (engine != null) 50 | { 51 | engine.SaveCache(); 52 | } 53 | 54 | done = true; 55 | }; 56 | 57 | try 58 | { 59 | using (var stream = File.OpenRead("slick.cfg")) 60 | { 61 | XmlSerializer ser = new XmlSerializer(typeof(BackupConfig)); 62 | config = ser.Deserialize(stream) as BackupConfig; 63 | } 64 | } 65 | catch (Exception ex) 66 | { 67 | config = new BackupConfig(); 68 | config.Backups = new Backup[1]; 69 | config.Backups[0] = new Backup() { Title = "Test", Source = "none", Destination = "none" }; 70 | 71 | using (var stream = File.OpenWrite("slick.cfg.example")) 72 | { 73 | XmlSerializer ser = new XmlSerializer(typeof(BackupConfig)); 74 | ser.Serialize(stream, config); 75 | } 76 | 77 | return; 78 | } 79 | 80 | scanThread = new Thread(() => 81 | { 82 | 83 | try 84 | { 85 | for (int pos = 0; pos < config.Backups.Length; pos++) 86 | { 87 | var backup = config.Backups[pos]; 88 | 89 | engine = new BackupEngine() 90 | { 91 | Title = "(" + (pos + 1) + "/" + config.Backups.Length + ") " + backup.Title, 92 | SourceFolder = backup.Source, 93 | DestinationFolder = backup.Destination, 94 | CacheUpdateCounterMax = backup.Reindex, 95 | AutoSaveTime = backup.AutoSave 96 | }; 97 | 98 | if (backup.IgnoreList != null) 99 | { 100 | engine.IgnoreList.AddRange(backup.IgnoreList.Replace(";", ",").Split(',').Select(s => s.Trim())); 101 | } 102 | 103 | LogFile.WriteLine("Starting Backup '{0}' on {1}", backup.Title, DateTime.Now.ToString()); 104 | LogFile.WriteLine(" Source '{0}'", backup.Source); 105 | LogFile.WriteLine(" Destination '{0}'", backup.Destination); 106 | LogFile.Flush(); 107 | 108 | engine.Log = LogFunc; 109 | engine.Execute(); 110 | 111 | LogFile.WriteLine("Finished '{0}' on {1}", backup.Title, DateTime.Now.ToString()); 112 | LogFile.WriteLine("-------------------------------------------------------------------------------------"); 113 | LogFile.Flush(); 114 | } 115 | } 116 | catch (Exception e) 117 | { 118 | Console.WriteLine("EXCEPTION: {0}", e.ToString()); 119 | } 120 | done = true; 121 | }); 122 | 123 | scanThread.Start(); 124 | 125 | while (!done) 126 | { 127 | if(engine == null) 128 | { 129 | continue; 130 | } 131 | Console.SetCursorPosition(0, 0); 132 | Console.WriteLine(""); 133 | Console.WriteLine("Title: {0,-80}", engine.Title); 134 | Console.WriteLine("State: {0,-80}", engine.State); 135 | Console.WriteLine("Progress Size: {0,-80}", MakeBar(engine.ProgressSize)); 136 | Console.WriteLine("Progress Files: {0,-80}", MakeBar(engine.ProgressFiles)); 137 | Console.WriteLine(""); 138 | Console.WriteLine("Source:"); 139 | Console.WriteLine(" Path: {0,-80}", CompressString(engine.SourceIndex.CurrentEntity)); 140 | Console.WriteLine(" Directories: {0,-80}", engine.SourceIndex.IndexedDirectories); 141 | Console.WriteLine(" Files: {0,-80}", engine.SourceIndex.IndexedFiles); 142 | Console.WriteLine(" Size: {0,-80}", FormatSize(engine.SourceIndex.IndexedSize)); 143 | Console.WriteLine(""); 144 | Console.WriteLine("Destination:"); 145 | Console.WriteLine(" Path: {0,-80}", CompressString(engine.DestinationIndex.CurrentEntity)); 146 | Console.WriteLine(" Directories: {0,-80}", engine.DestinationIndex.IndexedDirectories); 147 | Console.WriteLine(" Files: {0,-80}", engine.DestinationIndex.IndexedFiles); 148 | Console.WriteLine(" Size: {0,-80}", FormatSize(engine.DestinationIndex.IndexedSize)); 149 | Console.WriteLine(""); 150 | Console.WriteLine("Queue:"); 151 | Console.WriteLine(" Copy: {0} ({1}) ", engine.FilesToCopy, FormatSize(engine.SizeToCopy)); 152 | Console.WriteLine(" Update: {0} ({1}) ", engine.FilesToUpdate, FormatSize(engine.SizeToUpdate)); 153 | Console.WriteLine(" Delete: {0} ({1}) ", engine.FilesToDelete, FormatSize(engine.SizeToDelete)); 154 | Console.WriteLine(""); 155 | Console.WriteLine("Done"); 156 | Console.WriteLine(" Copy: {0} ({1}) ", engine.FilesCopied, FormatSize(engine.SizeCopied)); 157 | Console.WriteLine(" Update: {0} ({1}) ", engine.FilesUpdated, FormatSize(engine.SizeUpdated)); 158 | Console.WriteLine(" Delete: {0} ({1}) ", engine.FilesDeleted, FormatSize(engine.SizeDeleted)); 159 | Console.WriteLine(""); 160 | Console.WriteLine(""); 161 | 162 | var msgs = engine.Messages.Skip(Math.Max(0,engine.Messages.Count - 50)).ToArray(); 163 | foreach (var msg in msgs.Where(l => l.StartsWith("[ERROR]") || l.StartsWith("DELETE") || l.StartsWith("[INFO]")).Reverse().Take(20)) 164 | { 165 | Console.WriteLine(" {0}", CompressString(msg, 140).PadRight(140)); 166 | } 167 | Thread.Sleep(100); 168 | LogFile.Flush(); 169 | } 170 | 171 | foreach (var fail in engine.Messages) 172 | { 173 | Console.Error.WriteLine(fail); 174 | } 175 | LogFile.Close(); 176 | } 177 | 178 | private static void LogFunc(string line) 179 | { 180 | lock (LogFile) 181 | { 182 | try 183 | { 184 | LogFile.WriteLine(" {0}", line); 185 | } 186 | catch (Exception ex) 187 | { 188 | LogFile.WriteLine(" Exception when adding log entry: ", ex.Message); 189 | } 190 | } 191 | } 192 | 193 | private static string MakeBar(decimal progress, int width = 30) 194 | { 195 | //char[] parts = new char[] { ' ', '▏', '▎', '▍', '▌', '▋', '▊', '▉', '█' }; 196 | char[] parts = new char[] { ' ', ' ', ' ', ' ', '▌', '▌', '▌', '▌', '█' }; 197 | byte[] bar = new byte[width]; 198 | int whole_width = (int)Math.Floor(progress * width); 199 | decimal remainder_width = (progress * width) % 1.0m; 200 | 201 | int part_width = (int)Math.Floor(remainder_width * 8); 202 | string part_char = "" + parts[part_width]; 203 | 204 | if ((width - whole_width - 1) < 0) 205 | { 206 | part_char = ""; 207 | } 208 | 209 | int emptyChars = width - whole_width - 1; 210 | string line = "|" + new string('█', whole_width) + part_char + ((emptyChars > 0) ? new string(' ', emptyChars) : "") + "| " + (progress * 100).ToString("0.00") + "%"; 211 | return line; 212 | } 213 | 214 | private static string CompressString(string str, int maxLength = 80) 215 | { 216 | if(str == null) 217 | { 218 | return ""; 219 | } 220 | 221 | int maxPart = maxLength / 2 - 1; 222 | if (str.Length < maxLength) 223 | { 224 | return str; 225 | } 226 | return str.Substring(0, maxPart) + ".." + str.Substring(str.Length - maxPart, maxPart); 227 | } 228 | 229 | private static string FormatSize(decimal size) 230 | { 231 | string[] units = new[] { "Byte", "KiB", "MiB", "GiB", "TiB" }; 232 | int unit = 0; 233 | 234 | while(size > 1024 && unit < units.Length - 1) 235 | { 236 | size /= 1024; 237 | unit++; 238 | } 239 | 240 | return size.ToString("0.00") + " " + units[unit]; 241 | } 242 | } 243 | } 244 | -------------------------------------------------------------------------------- /TestBackup/BackupTestClass.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.VisualStudio.TestTools.UnitTesting; 2 | using SlickBackup; 3 | using System; 4 | using System.Collections.Generic; 5 | using System.IO; 6 | using System.Linq; 7 | using System.Text; 8 | 9 | namespace TestBackup 10 | { 11 | [TestClass] 12 | public class BackupTestClass 13 | { 14 | private Random Rnd = new Random(); 15 | private char[] InvalidChars = Path.GetInvalidFileNameChars(); 16 | private List ValidChars = new(); 17 | private string[] PrefixChars = new string[] { ".", "_", "#", "" }; 18 | 19 | [TestInitialize] 20 | public void TestInit() 21 | { 22 | for (int num = 0; num < 255; num++) 23 | { 24 | if (!InvalidChars.Contains((char)num)) 25 | { 26 | ValidChars.Add((char)num); 27 | } 28 | } 29 | } 30 | 31 | private string GetRandomName(int length) 32 | { 33 | return ""; 34 | StringBuilder ret = new(); 35 | ret.Append(PrefixChars[Rnd.Next(0, PrefixChars.Length - 1)]); 36 | for (int num = 0; num < length; num++) 37 | { 38 | ret.Append(ValidChars[Rnd.Next(0, ValidChars.Count - 1)]); 39 | } 40 | return ret.ToString(); 41 | } 42 | 43 | [TestMethod] 44 | public void TestFullCopy() 45 | { 46 | TestFullCopy(false); 47 | } 48 | 49 | [TestMethod] 50 | public void TestFullCopyRandomized() 51 | { 52 | TestFullCopy(true); 53 | } 54 | 55 | public void TestFullCopy(bool randomized) 56 | { 57 | string srcDir = Path.Combine(Path.GetTempPath(), "BackupSource"); 58 | string dstDir = Path.Combine(Path.GetTempPath(), "BackupDestination"); 59 | 60 | if (Directory.Exists(srcDir)) 61 | { 62 | Directory.Delete(srcDir, true); 63 | } 64 | if (Directory.Exists(dstDir)) 65 | { 66 | Directory.Delete(dstDir, true); 67 | } 68 | try 69 | { 70 | Directory.CreateDirectory(srcDir); 71 | Directory.CreateDirectory(dstDir); 72 | 73 | FillDirectory(srcDir, 10, 3, 10, "", randomized); 74 | 75 | BackupEngine e = new(); 76 | e.SourceFolder = srcDir; 77 | e.DestinationFolder = dstDir; 78 | 79 | e.Execute(); 80 | 81 | Assert.IsTrue(File.Exists(dstDir + Path.DirectorySeparatorChar + "_dst_cache.sbc"), "Missing cache file in destination folder"); 82 | CompareDirectories(srcDir, dstDir); 83 | } 84 | finally 85 | { 86 | Directory.Delete(srcDir, true); 87 | Directory.Delete(dstDir, true); 88 | } 89 | } 90 | 91 | [TestMethod] 92 | public void TestDelete() 93 | { 94 | string srcDir = Path.Combine(Path.GetTempPath(), "BackupSource"); 95 | string dstDir = Path.Combine(Path.GetTempPath(), "BackupDestination"); 96 | 97 | if (Directory.Exists(srcDir)) 98 | { 99 | Directory.Delete(srcDir, true); 100 | } 101 | if (Directory.Exists(dstDir)) 102 | { 103 | Directory.Delete(dstDir, true); 104 | } 105 | try 106 | { 107 | Directory.CreateDirectory(srcDir); 108 | Directory.CreateDirectory(dstDir); 109 | 110 | FillDirectory(srcDir, 10, 3, 10, "", false); 111 | 112 | BackupEngine e = new(); 113 | e.SourceFolder = srcDir; 114 | e.DestinationFolder = dstDir; 115 | 116 | e.Execute(); 117 | 118 | Assert.IsTrue(File.Exists(dstDir + Path.DirectorySeparatorChar + "_dst_cache.sbc"), "Missing cache file in destination folder"); 119 | CompareDirectories(srcDir, dstDir); 120 | 121 | DeleteRandom(srcDir); 122 | 123 | e.Execute(); 124 | CompareDirectories(srcDir, dstDir); 125 | } 126 | finally 127 | { 128 | Directory.Delete(srcDir, true); 129 | Directory.Delete(dstDir, true); 130 | } 131 | } 132 | 133 | [TestMethod] 134 | public void TestUpdate() 135 | { 136 | string srcDir = Path.Combine(Path.GetTempPath(), "BackupSource"); 137 | string dstDir = Path.Combine(Path.GetTempPath(), "BackupDestination"); 138 | 139 | if (Directory.Exists(srcDir)) 140 | { 141 | Directory.Delete(srcDir, true); 142 | } 143 | if (Directory.Exists(dstDir)) 144 | { 145 | Directory.Delete(dstDir, true); 146 | } 147 | try 148 | { 149 | Directory.CreateDirectory(srcDir); 150 | Directory.CreateDirectory(dstDir); 151 | 152 | FillDirectory(srcDir, 10, 3, 10, "", false); 153 | 154 | BackupEngine e = new(); 155 | e.SourceFolder = srcDir; 156 | e.DestinationFolder = dstDir; 157 | 158 | e.Execute(); 159 | 160 | Assert.IsTrue(File.Exists(dstDir + Path.DirectorySeparatorChar + "_dst_cache.sbc"), "Missing cache file in destination folder"); 161 | CompareDirectories(srcDir, dstDir); 162 | 163 | UpdateRandom(srcDir); 164 | 165 | e = new(); 166 | e.SourceFolder = srcDir; 167 | e.DestinationFolder = dstDir; 168 | e.Execute(); 169 | CompareDirectories(srcDir, dstDir); 170 | } 171 | finally 172 | { 173 | Directory.Delete(srcDir, true); 174 | Directory.Delete(dstDir, true); 175 | } 176 | } 177 | 178 | private void DeleteRandom(string dstDir, double probability = 0.01f) 179 | { 180 | foreach (var file in Directory.EnumerateFiles(dstDir)) 181 | { 182 | if (Rnd.NextDouble() < probability) 183 | { 184 | File.Delete(file); 185 | } 186 | } 187 | foreach (var dir in Directory.EnumerateDirectories(dstDir)) 188 | { 189 | if (Rnd.NextDouble() < probability) 190 | { 191 | Directory.Delete(dir, true); 192 | } 193 | else 194 | { 195 | DeleteRandom(dir); 196 | } 197 | } 198 | } 199 | 200 | private void UpdateRandom(string dstDir, double probability = 0.01f) 201 | { 202 | foreach (var file in Directory.EnumerateFiles(dstDir)) 203 | { 204 | if (Rnd.NextDouble() < probability) 205 | { 206 | var f = File.AppendText(file); 207 | f.WriteLine("Update" + Environment.NewLine); 208 | f.Flush(); 209 | f.Close(); 210 | } 211 | } 212 | foreach (var dir in Directory.EnumerateDirectories(dstDir)) 213 | { 214 | UpdateRandom(dir); 215 | } 216 | } 217 | 218 | private void CompareDirectories(string srcDir, string dstDir) 219 | { 220 | Assert.IsTrue(Directory.Exists(srcDir), "Source folder does not exist"); 221 | Assert.IsTrue(Directory.Exists(dstDir), "Destination folder does not exist"); 222 | 223 | var srcFiles = Directory.EnumerateFiles(srcDir).Select(f => new FileInfo(f).Name); 224 | var dstFiles = Directory.EnumerateFiles(dstDir).Where(s => !s.EndsWith(".sbc")).Select(f => new FileInfo(f).Name); 225 | 226 | Assert.IsTrue(srcFiles.SequenceEqual(dstFiles), "File list not equal"); 227 | 228 | var srcDirs = Directory.GetDirectories(srcDir).Select(f => new DirectoryInfo(f).Name); 229 | var dstDirs = Directory.GetDirectories(dstDir).Select(f => new DirectoryInfo(f).Name); 230 | 231 | Assert.IsTrue(srcDirs.SequenceEqual(dstDirs), "Directory list not equal"); 232 | 233 | foreach(string file in srcFiles) 234 | { 235 | var srcInfo = new FileInfo(srcDir + Path.DirectorySeparatorChar + file); 236 | var dstInfo = new FileInfo(dstDir + Path.DirectorySeparatorChar + file); 237 | 238 | Assert.AreEqual(srcInfo.Length, dstInfo.Length, "File lengths have to match"); 239 | Assert.AreEqual(srcInfo.LastWriteTime, dstInfo.LastWriteTime, "LastWriteTime have to match"); 240 | Assert.AreEqual(srcInfo.Attributes, dstInfo.Attributes, "Attributes have to match"); 241 | } 242 | 243 | foreach (string file in srcDirs) 244 | { 245 | var srcInfo = new FileInfo(srcDir + Path.DirectorySeparatorChar + file); 246 | var dstInfo = new FileInfo(dstDir + Path.DirectorySeparatorChar + file); 247 | 248 | CompareDirectories(srcInfo.FullName, dstInfo.FullName); 249 | } 250 | } 251 | 252 | private void FillDirectory(string srcDir, int dirCount, int depth, int fileCount, string prefix = "", bool randomize = false) 253 | { 254 | int thisDirs = randomize ? Rnd.Next(dirCount) : dirCount; 255 | int thisFiles = randomize ? Rnd.Next(fileCount) : fileCount; 256 | 257 | if (depth > 0) 258 | { 259 | for (int dir = 0; dir < thisDirs; dir++) 260 | { 261 | string dirName = srcDir + Path.DirectorySeparatorChar + "Dir_" + prefix + dir; 262 | Directory.CreateDirectory(dirName); 263 | FillDirectory(dirName, dirCount, depth - 1, fileCount, prefix + "_", randomize); 264 | } 265 | } 266 | 267 | for (int file = 0; file < thisFiles; file++) 268 | { 269 | string fileName = srcDir + Path.DirectorySeparatorChar + "File_" + prefix + file; 270 | var writer = File.CreateText(fileName); 271 | 272 | writer.Write("Content of " + fileName + Environment.NewLine); 273 | writer.Flush(); 274 | writer.Close(); 275 | } 276 | } 277 | 278 | [TestMethod] 279 | public void TestMatchAlgorithm() 280 | { 281 | TestMatchAlgorithmRoutine(1000); 282 | } 283 | [TestMethod] 284 | public void TestMatchAlgorithmMulti () 285 | { 286 | for (int num = 0; num < 1000; num++) 287 | { 288 | TestMatchAlgorithmRoutine(1000); 289 | } 290 | } 291 | 292 | [TestMethod] 293 | public void TestMatchAlgorithmFolder() 294 | { 295 | TestMatchAlgorithmRoutineFolders(20, 2); 296 | } 297 | 298 | 299 | public void TestMatchAlgorithmRoutine(int fileCount = 100) 300 | { 301 | List srcNodes = new(); 302 | List dstNodes = new(); 303 | for (int num = 0; num < fileCount; num++) 304 | { 305 | string name = GetRandomName(Rnd.Next(1, 64)); 306 | srcNodes.Add(new BackupEngine.TreeNode() { Name = name + "_Matching_" + num, Type = BackupEngine.eType.File, LastChange = num, Length = num }); 307 | dstNodes.Add(new BackupEngine.TreeNode() { Name = name + "_Matching_" + num, Type = BackupEngine.eType.File, LastChange = num, Length = num }); 308 | } 309 | for (int num = 0; num < fileCount; num++) 310 | { 311 | string name = GetRandomName(Rnd.Next(1, 64)); 312 | srcNodes.Add(new BackupEngine.TreeNode() { Name = name + "UpdateDate1_" + num, Type = BackupEngine.eType.File, LastChange = num + 1, Length = num }); 313 | dstNodes.Add(new BackupEngine.TreeNode() { Name = name + "UpdateDate1_" + num, Type = BackupEngine.eType.File, LastChange = num, Length = num }); 314 | } 315 | for (int num = 0; num < fileCount; num++) 316 | { 317 | string name = GetRandomName(Rnd.Next(1, 64)); 318 | srcNodes.Add(new BackupEngine.TreeNode() { Name = name + "UpdateDate2_" + num, Type = BackupEngine.eType.File, LastChange = num, Length = num }); 319 | dstNodes.Add(new BackupEngine.TreeNode() { Name = name + "UpdateDate2_" + num, Type = BackupEngine.eType.File, LastChange = num + 1, Length = num }); 320 | } 321 | for (int num = 0; num < fileCount; num++) 322 | { 323 | string name = GetRandomName(Rnd.Next(1, 64)); 324 | srcNodes.Add(new BackupEngine.TreeNode() { Name = name + "UpdateLength1_" + num, Type = BackupEngine.eType.File, LastChange = num, Length = num + 1 }); 325 | dstNodes.Add(new BackupEngine.TreeNode() { Name = name + "UpdateLength1_" + num, Type = BackupEngine.eType.File, LastChange = num, Length = num }); 326 | } 327 | for (int num = 0; num < fileCount; num++) 328 | { 329 | string name = GetRandomName(Rnd.Next(1, 64)); 330 | srcNodes.Add(new BackupEngine.TreeNode() { Name = name + "UpdateLength2_" + num, Type = BackupEngine.eType.File, LastChange = num, Length = num }); 331 | dstNodes.Add(new BackupEngine.TreeNode() { Name = name + "UpdateLength2_" + num, Type = BackupEngine.eType.File, LastChange = num, Length = num + 1 }); 332 | } 333 | for (int num = 0; num < fileCount; num++) 334 | { 335 | string name = GetRandomName(Rnd.Next(1, 64)); 336 | srcNodes.Add(new BackupEngine.TreeNode() { Name = name + "SourceOnly_" + num, Type = BackupEngine.eType.File, LastChange = num, Length = num }); 337 | } 338 | for (int num = 0; num < fileCount; num++) 339 | { 340 | string name = GetRandomName(Rnd.Next(1, 64)); 341 | dstNodes.Add(new BackupEngine.TreeNode() { Name = name + "DestinationOnly_" + num, Type = BackupEngine.eType.File, LastChange = num, Length = num }); 342 | } 343 | 344 | BackupEngine e = new(); 345 | BackupEngine.TreeNode srcRoot = new() { Children = srcNodes.ToArray() }; 346 | BackupEngine.TreeNode dstRoot = new() { Children = dstNodes.ToArray() }; 347 | 348 | Rnd.Shuffle(srcRoot.Children); 349 | Rnd.Shuffle(dstRoot.Children); 350 | 351 | e.MatchDirectory("source", srcRoot, "destination", dstRoot); 352 | 353 | Assert.IsTrue(e.FileDeleteQueue.Where(e => !e.Name.Contains("DestinationOly")).Any()); 354 | Assert.AreEqual(fileCount, e.FileDeleteQueue.Count); 355 | 356 | Assert.AreEqual(0, e.FileCopyQueue.Where(e => !e.Key.StartsWith("source\\")).Count()); 357 | Assert.AreEqual(0, e.FileCopyQueue.Where(e => !e.Value.StartsWith("destination\\")).Count()); 358 | Assert.AreEqual(0, e.FileCopyQueue.Where(e => !e.Key.Contains("SourceOnly")).Count()); 359 | Assert.AreEqual(0, e.FileCopyQueue.Where(e => !e.Value.Contains("SourceOnly")).Count()); 360 | Assert.AreEqual(fileCount, e.FileCopyQueue.Count); 361 | 362 | Assert.AreEqual(0, e.FileUpdateQueue.Where(e => !e.Key.StartsWith("source\\")).Count()); 363 | Assert.AreEqual(0, e.FileUpdateQueue.Where(e => !e.Value.StartsWith("destination\\")).Count()); 364 | Assert.AreEqual(0, e.FileUpdateQueue.Where(e => !e.Key.Contains("Update")).Count()); 365 | Assert.AreEqual(0, e.FileUpdateQueue.Where(e => !e.Value.Contains("Update")).Count()); 366 | Assert.AreEqual(fileCount * 4, e.FileUpdateQueue.Count); 367 | } 368 | 369 | public void TestMatchAlgorithmRoutineFolders(int folderCount = 100, int fileCount = 10) 370 | { 371 | int folderCountUpdate = folderCount; 372 | int folderCountSourceOnly = folderCount + 1; 373 | int folderCountDestinationOnly = folderCount + 2; 374 | 375 | List srcFolders = new(); 376 | List dstFolders = new(); 377 | 378 | for (int folderNum = 0; folderNum < folderCountUpdate; folderNum++) 379 | { 380 | string name = GetRandomName(Rnd.Next(1, 64)); 381 | 382 | List srcNodes = new(); 383 | List dstNodes = new(); 384 | 385 | for (int fileNum = 0; fileNum < fileCount; fileNum++) 386 | { 387 | string fileName = GetRandomName(Rnd.Next(1, 64)); 388 | srcNodes.Add(new BackupEngine.TreeNode() { Name = fileName + "_UpdateFile_" + fileNum, Type = BackupEngine.eType.File, LastChange = folderNum + 1, Length = folderNum }); 389 | dstNodes.Add(new BackupEngine.TreeNode() { Name = fileName + "_UpdateFile_" + fileNum, Type = BackupEngine.eType.File, LastChange = folderNum, Length = folderNum }); 390 | } 391 | 392 | BackupEngine.TreeNode srcFolder = new BackupEngine.TreeNode() { Name = name + "_MatchingDir_" + folderNum, Type = BackupEngine.eType.Directory, LastChange = folderNum, Length = folderNum }; 393 | BackupEngine.TreeNode dstFolder = new BackupEngine.TreeNode() { Name = name + "_MatchingDir_" + folderNum, Type = BackupEngine.eType.Directory, LastChange = folderNum, Length = folderNum }; 394 | srcFolder.Children = srcNodes.ToArray(); 395 | dstFolder.Children = dstNodes.ToArray(); 396 | 397 | Rnd.Shuffle(srcFolder.Children); 398 | Rnd.Shuffle(dstFolder.Children); 399 | 400 | srcFolders.Add(srcFolder); 401 | dstFolders.Add(dstFolder); 402 | } 403 | for (int folderNum = 0; folderNum < folderCountSourceOnly; folderNum++) 404 | { 405 | string name = GetRandomName(Rnd.Next(1, 64)); 406 | 407 | List srcNodes = new(); 408 | 409 | for (int fileNum = 0; fileNum < fileCount; fileNum++) 410 | { 411 | string fileName = GetRandomName(Rnd.Next(1, 64)); 412 | srcNodes.Add(new BackupEngine.TreeNode() { Name = fileName + "_SourceOnlyFile_" + fileNum, Type = BackupEngine.eType.File, LastChange = folderNum + 1, Length = folderNum }); 413 | } 414 | 415 | BackupEngine.TreeNode srcFolder = new BackupEngine.TreeNode() { Name = name + "_SourceOnlyDir_" + folderNum, Type = BackupEngine.eType.Directory, LastChange = folderNum, Length = folderNum }; 416 | srcFolder.Children = srcNodes.ToArray(); 417 | 418 | Rnd.Shuffle(srcFolder.Children); 419 | 420 | srcFolders.Add(srcFolder); 421 | } 422 | for (int folderNum = 0; folderNum < folderCountDestinationOnly; folderNum++) 423 | { 424 | string name = GetRandomName(Rnd.Next(1, 64)); 425 | 426 | List dstNodes = new(); 427 | 428 | for (int fileNum = 0; fileNum < fileCount; fileNum++) 429 | { 430 | string fileName = GetRandomName(Rnd.Next(1, 64)); 431 | dstNodes.Add(new BackupEngine.TreeNode() { Name = fileName + "_DestinationOnlyFile_" + fileNum, Type = BackupEngine.eType.File, LastChange = folderNum, Length = folderNum }); 432 | } 433 | 434 | BackupEngine.TreeNode dstFolder = new BackupEngine.TreeNode() { Name = name + "_DestinationOnlyDir_" + folderNum, Type = BackupEngine.eType.Directory, LastChange = folderNum, Length = folderNum }; 435 | dstFolder.Children = dstNodes.ToArray(); 436 | 437 | Rnd.Shuffle(dstFolder.Children); 438 | 439 | dstFolders.Add(dstFolder); 440 | } 441 | 442 | BackupEngine e = new(); 443 | BackupEngine.TreeNode srcRoot = new() { Children = srcFolders.ToArray() }; 444 | BackupEngine.TreeNode dstRoot = new() { Children = dstFolders.ToArray() }; 445 | 446 | Rnd.Shuffle(srcRoot.Children); 447 | Rnd.Shuffle(dstRoot.Children); 448 | 449 | e.MatchDirectory("source", srcRoot, "destination", dstRoot); 450 | 451 | Assert.IsTrue(e.FileDeleteQueue.Where(e => !e.Name.Contains("_DestinationOnlyFile_")).Any()); 452 | Assert.AreEqual(folderCountDestinationOnly, e.FileDeleteQueue.Count); 453 | Assert.AreEqual(folderCountDestinationOnly * fileCount, e.FilesToDelete); 454 | 455 | Assert.AreEqual(0, e.FileCopyQueue.Where(e => !e.Key.StartsWith("source\\")).Count()); 456 | Assert.AreEqual(0, e.FileCopyQueue.Where(e => !e.Value.StartsWith("destination\\")).Count()); 457 | Assert.AreEqual(folderCountSourceOnly, e.FileCopyQueue.Where(e => !e.Key.Contains("_SourceOnlyFile_")).Count()); 458 | Assert.AreEqual(folderCountSourceOnly, e.FileCopyQueue.Where(e => !e.Value.Contains("_SourceOnlyFile_")).Count()); 459 | Assert.AreEqual(folderCountSourceOnly * fileCount + folderCountSourceOnly, e.FileCopyQueue.Count); 460 | Assert.AreEqual(folderCountSourceOnly * fileCount + folderCountSourceOnly, e.FilesToCopy); 461 | 462 | 463 | Assert.AreEqual(0, e.FileUpdateQueue.Where(e => !e.Key.StartsWith("source\\")).Count()); 464 | Assert.AreEqual(0, e.FileUpdateQueue.Where(e => !e.Value.StartsWith("destination\\")).Count()); 465 | Assert.AreEqual(0, e.FileUpdateQueue.Where(e => !e.Key.Contains("_UpdateFile_")).Count()); 466 | Assert.AreEqual(0, e.FileUpdateQueue.Where(e => !e.Value.Contains("_UpdateFile_")).Count()); 467 | Assert.AreEqual(folderCount * fileCount, e.FileUpdateQueue.Count); 468 | Assert.AreEqual(folderCount * fileCount, e.FilesToUpdate); 469 | 470 | } 471 | 472 | } 473 | } 474 | -------------------------------------------------------------------------------- /SlickBackup/BackupEngine.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Runtime.Serialization; 6 | using System.Runtime.Serialization.Formatters.Binary; 7 | using System.Security.Cryptography; 8 | using System.Text; 9 | using System.Threading; 10 | 11 | using System.Text.Json; 12 | using System.Text.Json.Serialization; 13 | using System.Xml.Serialization; 14 | using System.ComponentModel; 15 | using System.Runtime.CompilerServices; 16 | using System.Threading.Tasks; 17 | using System.Reflection.Metadata.Ecma335; 18 | using System.IO.Compression; 19 | 20 | namespace SlickBackup 21 | { 22 | public class BackupEngine 23 | { 24 | public int AutoSaveTime = 10 * 60; 25 | public int CacheUpdateCounterMax = 10; 26 | 27 | private static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0); 28 | internal string Title = ""; 29 | internal string SourceFolder = ""; 30 | internal string DestinationFolder = ""; 31 | 32 | internal TreeInfo SourceIndex = new(); 33 | internal TreeInfo DestinationIndex = new(); 34 | 35 | internal Dictionary FileUpdateQueue = new(); 36 | internal Dictionary FileCopyQueue = new(); 37 | internal List FileDeleteQueue = new(); 38 | 39 | internal long SizeToCopy = 0; 40 | internal long SizeToUpdate = 0; 41 | internal long SizeToDelete = 0; 42 | internal long SizeDeleted = 0; 43 | internal long SizeCopied = 0; 44 | internal long SizeUpdated = 0; 45 | internal long FilesCopied = 0; 46 | internal long FilesUpdated = 0; 47 | internal long FilesDeleted = 0; 48 | internal long FilesToCopy = 0; 49 | internal long FilesToUpdate = 0; 50 | internal long FilesToDelete = 0; 51 | 52 | internal long DirectoriesCopied = 0; 53 | 54 | internal eState State = eState.Init; 55 | private readonly SHA256 Sha256 = SHA256.Create(); 56 | 57 | internal List Messages = new(); 58 | internal List IgnoreList = new(); 59 | private FileStream LockFileHandle; 60 | internal ParallelOptions ParallelOptions = new(); 61 | internal Action Log; 62 | 63 | public BackupEngine() 64 | { 65 | IgnoreList.Add(".sbc$"); 66 | 67 | int maxPar = (int)Math.Ceiling((Environment.ProcessorCount * 0.75)); 68 | ParallelOptions.MaxDegreeOfParallelism = maxPar; 69 | } 70 | 71 | public enum eState 72 | { 73 | Init, 74 | Scan, 75 | Match, 76 | Copy, 77 | Update, 78 | Delete, 79 | Done 80 | } 81 | 82 | public enum eType : int 83 | { 84 | Directory = 0, 85 | File = 1, 86 | Special = 2 87 | } 88 | 89 | [Serializable] 90 | public class TreeInfo 91 | { 92 | public TreeNode Root; 93 | public long IndexedDirectories; 94 | public long IndexedFiles; 95 | public long CacheUpdateCounter; 96 | public decimal IndexedSize; 97 | internal string CurrentEntity; 98 | 99 | [JsonIgnore] 100 | internal bool Modified = false; 101 | [JsonIgnore] 102 | internal DateTime LastSaveTime; 103 | [JsonIgnore] 104 | internal Thread SaveThread; 105 | } 106 | 107 | [Serializable] 108 | public class TreeNode 109 | { 110 | public string Name; 111 | public long Length; 112 | 113 | [JsonIgnore] 114 | public long SizeRecursive 115 | { 116 | get 117 | { 118 | if (Type == eType.Directory) 119 | { 120 | return Children.Sum(c => c.SizeRecursive); 121 | } 122 | return Length; 123 | } 124 | } 125 | 126 | [JsonIgnore] 127 | public long FilesRecursive 128 | { 129 | get 130 | { 131 | if (Type == eType.Directory) 132 | { 133 | return Children.Sum(c => c.FilesRecursive); 134 | } 135 | return 1; 136 | } 137 | } 138 | 139 | public eType Type; 140 | public long LastChange; 141 | public TreeNode[] Children = Array.Empty(); 142 | [JsonIgnore] 143 | internal string FullPath = ""; 144 | 145 | public override string ToString() 146 | { 147 | return Name; 148 | } 149 | 150 | public TreeNode this[string name] 151 | { 152 | get 153 | { 154 | return Children.Where(c => c.Name == name).First(); 155 | } 156 | } 157 | } 158 | 159 | public decimal ProgressSize 160 | { 161 | get 162 | { 163 | decimal total = 0; 164 | decimal done = 0; 165 | switch (State) 166 | { 167 | case eState.Init: 168 | case eState.Scan: 169 | case eState.Match: 170 | return 0; 171 | case eState.Done: 172 | return 1; 173 | case eState.Delete: 174 | total = SizeToDelete; 175 | done = SizeDeleted; 176 | break; 177 | case eState.Copy: 178 | total = SizeToCopy + SizeToUpdate; 179 | done = SizeCopied + SizeUpdated; 180 | break; 181 | } 182 | 183 | if (total == 0) 184 | { 185 | return 0; 186 | } 187 | return done / total; 188 | } 189 | } 190 | 191 | public decimal ProgressFiles 192 | { 193 | get 194 | { 195 | decimal total = 0; 196 | decimal done = 0; 197 | switch (State) 198 | { 199 | case eState.Init: 200 | case eState.Scan: 201 | case eState.Match: 202 | return 0; 203 | case eState.Done: 204 | return 1; 205 | case eState.Delete: 206 | total = FilesToDelete; 207 | done = FilesDeleted; 208 | break; 209 | case eState.Copy: 210 | total = FilesToCopy + FilesToUpdate; 211 | done = FilesCopied + FilesUpdated; 212 | break; 213 | } 214 | 215 | if (total == 0) 216 | { 217 | return 0; 218 | } 219 | return done / total; 220 | } 221 | } 222 | 223 | internal void MatchFiles() 224 | { 225 | State = eState.Match; 226 | 227 | MatchDirectory(SourceIndex.Root.Name, SourceIndex.Root, DestinationIndex.Root.Name, DestinationIndex.Root); 228 | } 229 | 230 | internal void MatchDirectory(string srcPath, TreeNode src, string dstPath, TreeNode dst) 231 | { 232 | SourceIndex.CurrentEntity = srcPath; 233 | DestinationIndex.CurrentEntity = dstPath; 234 | 235 | var srcList = src.Children.OrderBy(e => e.Name).ToArray(); 236 | var dstList = dst.Children.OrderBy(e => e.Name).ToArray(); 237 | int srcPos = 0; 238 | int dstPos = 0; 239 | 240 | List> subDirs = new(); 241 | 242 | while (srcPos < srcList.Length || dstPos < dstList.Length) 243 | { 244 | TreeNode s = null; 245 | TreeNode d = null; 246 | 247 | if (srcPos < srcList.Length) 248 | { 249 | s = srcList[srcPos]; 250 | } 251 | if (dstPos < dstList.Length) 252 | { 253 | d = dstList[dstPos]; 254 | } 255 | 256 | int comp = 0; 257 | 258 | if (s == null) 259 | { 260 | comp = 1; 261 | } 262 | else if (d == null) 263 | { 264 | comp = -1; 265 | } 266 | else 267 | { 268 | comp = s.Name.CompareTo(d.Name); 269 | } 270 | 271 | if (comp < 0) 272 | { 273 | /* those which are missing in destination have to get copied. Also whose type doesn't match anymore */ 274 | s.FullPath = Path.Combine(srcPath, s.Name); 275 | 276 | if (s.Type == eType.Directory) 277 | { 278 | AddDirectory(src[s.Name], s.FullPath, Path.Combine(dstPath, s.Name)); 279 | } 280 | else 281 | { 282 | lock (FileCopyQueue) 283 | { 284 | FileCopyQueue.Add(s.FullPath, Path.Combine(dstPath, s.Name)); 285 | } 286 | FilesToCopy++; 287 | SizeToCopy += s.Length; 288 | } 289 | srcPos++; 290 | } 291 | else if (comp > 0) 292 | { 293 | /* those which are missing in source directory can be deleted in destination directory also */ 294 | d.FullPath = Path.Combine(dstPath, d.Name); 295 | string sourcePath = Path.Combine(srcPath, d.Name); 296 | 297 | if (File.Exists(sourcePath) && !IsIgnored(new FileInfo(sourcePath))) 298 | { 299 | AddMessage("[ERROR] Consistency check failed. " + d.FullPath + " would get deleted, but still exists in source"); 300 | //return; 301 | } 302 | if (Directory.Exists(sourcePath) && !IsIgnored(new DirectoryInfo(sourcePath))) 303 | { 304 | AddMessage("[ERROR] Consistency check failed. " + d.FullPath + " would get deleted, but still exists in source"); 305 | //return; 306 | } 307 | lock (FileDeleteQueue) 308 | { 309 | FileDeleteQueue.Add(d); 310 | } 311 | FilesToDelete += d.FilesRecursive; 312 | SizeToDelete += d.SizeRecursive; 313 | dstPos++; 314 | } 315 | else if (s.Type == eType.File && (s.LastChange != d.LastChange || s.Length != d.Length)) 316 | { 317 | lock (FileUpdateQueue) 318 | { 319 | FileUpdateQueue.Add(Path.Combine(srcPath, s.Name), Path.Combine(dstPath, s.Name)); 320 | } 321 | FilesToUpdate++; 322 | SizeToUpdate += s.Length; 323 | srcPos++; 324 | dstPos++; 325 | } 326 | else if (s.Type == eType.Directory) 327 | { 328 | /* recurse directories */ 329 | subDirs.Add(new Tuple(s, d)); 330 | srcPos++; 331 | dstPos++; 332 | } 333 | else 334 | { 335 | srcPos++; 336 | dstPos++; 337 | } 338 | } 339 | 340 | Parallel.ForEach(subDirs, ParallelOptions, t => 341 | { 342 | MatchDirectory(Path.Combine(srcPath, t.Item1.Name), t.Item1, Path.Combine(dstPath, t.Item2.Name), t.Item2); 343 | }); 344 | 345 | #if false 346 | /* those which are missing in source directory can be deleted in destination directory also */ 347 | var missInSource = dst.Children.Where(predicate: s => !src.Children.Where(d => s.Name == d.Name && s.Type == d.Type).Any()); 348 | foreach (var d in missInSource) 349 | { 350 | d.FullPath = Path.Combine(dstPath, d.Name); 351 | EntitiesToDelete.Add(d); 352 | SizeToDelete += d.SizeRecursive; 353 | } 354 | 355 | /* those which are missing in destination have to get copied. Also whose type doesn't match anymore */ 356 | var missInDestination = src.Children.Where(s => !dst.Children.Where(d => s.Name == d.Name).Any() || dst.Children.Where(d => s.Name == d.Name && s.Type != d.Type).Any()); 357 | foreach (var s in missInDestination) 358 | { 359 | s.FullPath = Path.Combine(srcPath, s.Name); 360 | 361 | if (s.Type == eType.Directory) 362 | { 363 | AddDirectory(src[s.Name], s.FullPath, Path.Combine(dstPath, s.Name)); 364 | } 365 | else 366 | { 367 | FilesToCopy.Add(s.FullPath, Path.Combine(dstPath, s.Name)); 368 | SizeToCopy += s.Length; 369 | } 370 | } 371 | 372 | var changed = src.Children.Where(s => s.Type != eType.Directory).Where(s => dst.Children.Where(d => s.Name == d.Name && (s.LastChange != d.LastChange || s.Length != d.Length)).Any()); 373 | foreach (var s in changed) 374 | { 375 | FilesToUpdate.Add(Path.Combine(srcPath, s.Name), Path.Combine(dstPath, s.Name)); 376 | SizeToUpdate += s.Length; 377 | } 378 | 379 | /* recurse source side directories */ 380 | Parallel.ForEach(src.Children.Where(d => d.Type == eType.Directory), new ParallelOptions { MaxDegreeOfParallelism = (int)Math.Ceiling((Environment.ProcessorCount * 0.75)) }, dir => 381 | //foreach (var dir in src.Children.Where(d => d.Type == eType.Directory)) 382 | { 383 | if (!missInDestination.Contains(dir)) 384 | { 385 | var dstDir = dst.Children.Where(d => dir.Name == d.Name).First(); 386 | MatchDirectory(Path.Combine(srcPath, dir.Name), dir, Path.Combine(dstPath, dir.Name), dstDir); 387 | } 388 | }); 389 | #endif 390 | } 391 | 392 | private int NodeSort(TreeNode x, TreeNode y) 393 | { 394 | return x.Name.CompareTo(y.Name); 395 | } 396 | 397 | private void AddDirectory(TreeNode srcNode, string src, string dst) 398 | { 399 | /* add an dummy entry that hints a directory to be created in the case of empty directories */ 400 | lock (FileCopyQueue) 401 | { 402 | FileCopyQueue.Add(src, dst); 403 | FilesToCopy++; 404 | } 405 | 406 | foreach (var info in srcNode.Children.Where(c => c.Type != eType.Directory)) 407 | { 408 | lock (FileCopyQueue) 409 | { 410 | FileCopyQueue.Add(Path.Combine(src, info.Name), Path.Combine(dst, info.Name)); 411 | } 412 | FilesToCopy++; 413 | SizeToCopy += info.Length; 414 | } 415 | foreach (var info in srcNode.Children.Where(c => c.Type == eType.Directory)) 416 | { 417 | AddDirectory(info, Path.Combine(src, info.Name), Path.Combine(dst, info.Name)); 418 | } 419 | } 420 | 421 | private ulong FileChecksum(string filename) 422 | { 423 | using FileStream stream = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); 424 | byte[] hash = Sha256.ComputeHash(stream); 425 | return BitConverter.ToUInt64(hash, 0); 426 | } 427 | 428 | public void BuildTree() 429 | { 430 | State = eState.Scan; 431 | 432 | SourceIndex.Root = new TreeNode() { Name = SourceFolder }; 433 | DestinationIndex.Root = new TreeNode() { Name = DestinationFolder }; 434 | 435 | Thread srcThread = new(() => 436 | { 437 | IndexDirectory(SourceIndex, SourceFolder, SourceIndex.Root, true); 438 | }); 439 | 440 | Thread dstThread = new(() => 441 | { 442 | bool indexValid = false; 443 | 444 | indexValid |= LoadCache(DestinationFolder, "_dst_cache.sbc", ref DestinationIndex); 445 | if (!indexValid) 446 | { 447 | indexValid |= LoadCache(DestinationFolder, "_dst_cache_bak.sbc", ref DestinationIndex); 448 | } 449 | 450 | 451 | if (indexValid) 452 | { 453 | if (DestinationIndex.CacheUpdateCounter >= CacheUpdateCounterMax) 454 | { 455 | string msg = "Used cache " + DestinationIndex.CacheUpdateCounter + "/" + CacheUpdateCounterMax + " times, reindexing..."; 456 | 457 | DestinationIndex.CurrentEntity = msg; 458 | AddMessage("[INFO] " + msg); 459 | indexValid = false; 460 | } 461 | else 462 | { 463 | AddMessage(" Used cache " + DestinationIndex.CacheUpdateCounter + "/" + CacheUpdateCounterMax + " times. " + (CacheUpdateCounterMax - DestinationIndex.CacheUpdateCounter) + " runs until reindexing."); 464 | AddMessage(" File count: " + DestinationIndex.IndexedFiles); 465 | AddMessage(" File sizes: " + DestinationIndex.IndexedSize + " (" + FormatSize(DestinationIndex.IndexedSize) + ")"); 466 | } 467 | } 468 | else 469 | { 470 | AddMessage("[INFO] Cache invalid. Indexing destination directory."); 471 | } 472 | 473 | if (!indexValid) 474 | { 475 | DestinationIndex = new(); 476 | DestinationIndex.Root = new TreeNode() { Name = DestinationFolder }; 477 | 478 | DateTime start = DateTime.Now; 479 | IndexDirectory(DestinationIndex, DestinationFolder, DestinationIndex.Root); 480 | DateTime end = DateTime.Now; 481 | AddMessage("[INFO] Indexing finished, took " + (end - start).TotalSeconds + " seconds."); 482 | 483 | DestinationIndex.Modified = true; 484 | } 485 | SaveCache(); 486 | }); 487 | 488 | srcThread.Name = "Scan source"; 489 | dstThread.Name = "Scan destination"; 490 | srcThread.Start(); 491 | dstThread.Start(); 492 | 493 | while (dstThread.IsAlive || srcThread.IsAlive) 494 | { 495 | Thread.Sleep(50); 496 | } 497 | } 498 | 499 | public static void CopyTo(Stream src, Stream dest) 500 | { 501 | byte[] bytes = new byte[4096]; 502 | 503 | int cnt; 504 | 505 | while ((cnt = src.Read(bytes, 0, bytes.Length)) != 0) 506 | { 507 | dest.Write(bytes, 0, cnt); 508 | } 509 | } 510 | 511 | public static byte[] Zip(string str) 512 | { 513 | var bytes = Encoding.UTF8.GetBytes(str); 514 | 515 | using (var msi = new MemoryStream(bytes)) 516 | using (var mso = new MemoryStream()) 517 | { 518 | using (var gs = new GZipStream(mso, CompressionMode.Compress)) 519 | { 520 | CopyTo(msi, gs); 521 | } 522 | 523 | return mso.ToArray(); 524 | } 525 | } 526 | 527 | public static string Unzip(byte[] bytes) 528 | { 529 | using (var msi = new MemoryStream(bytes)) 530 | using (var mso = new MemoryStream()) 531 | { 532 | using (var gs = new GZipStream(msi, CompressionMode.Decompress)) 533 | { 534 | CopyTo(gs, mso); 535 | } 536 | 537 | return Encoding.UTF8.GetString(mso.ToArray()); 538 | } 539 | } 540 | 541 | private bool LoadCache(string destinationFolder, string file, ref TreeInfo destinationIndex) 542 | { 543 | bool ret = false; 544 | try 545 | { 546 | string cachedIndex = Path.Combine(DestinationFolder, file); 547 | 548 | if(!File.Exists(cachedIndex)) 549 | { 550 | return false; 551 | } 552 | 553 | destinationIndex.CurrentEntity = "Reading '" + cachedIndex + "'"; 554 | 555 | string jsonString = null; 556 | byte[] binary = File.ReadAllBytes(cachedIndex); 557 | try 558 | { 559 | jsonString = Unzip(binary); 560 | } 561 | catch(Exception ex) 562 | { 563 | AddMessage("[INFO] Cache reading failed: " + ex.ToString()); 564 | jsonString = null; 565 | } 566 | if (jsonString == null) 567 | { 568 | jsonString = File.ReadAllText(cachedIndex); 569 | } 570 | destinationIndex = JsonSerializer.Deserialize(jsonString, new JsonSerializerOptions() { IncludeFields = true }); 571 | destinationIndex.Root.Name = DestinationFolder; 572 | destinationIndex.CacheUpdateCounter++; 573 | destinationIndex.CurrentEntity = ""; 574 | destinationIndex.Modified = false; 575 | 576 | destinationIndex.CurrentEntity = "Used cache " + destinationIndex.CacheUpdateCounter + "/" + CacheUpdateCounterMax + " times"; 577 | if (destinationIndex.CacheUpdateCounter == CacheUpdateCounterMax / 2) 578 | { 579 | destinationIndex.CurrentEntity = "Used cache " + destinationIndex.CacheUpdateCounter + "/" + CacheUpdateCounterMax + " times, updating cached sizes"; 580 | destinationIndex.IndexedSize = destinationIndex.Root.SizeRecursive; 581 | } 582 | 583 | ret = true; 584 | } 585 | catch (Exception ex) 586 | { 587 | } 588 | 589 | return ret; 590 | } 591 | public void SaveCache() 592 | { 593 | string cachedIndex = Path.Combine(DestinationFolder, "_dst_cache"); 594 | 595 | DestinationIndex.LastSaveTime = DateTime.Now; 596 | 597 | if (File.Exists(cachedIndex + ".sbc") && !DestinationIndex.Modified) 598 | { 599 | return; 600 | } 601 | 602 | string jsonString = JsonSerializer.Serialize(DestinationIndex, new JsonSerializerOptions() { IncludeFields = true }); 603 | byte[] binary = Zip(jsonString); 604 | 605 | for (int retry = 0; retry < 10; retry++) 606 | { 607 | try 608 | { 609 | if (File.Exists(cachedIndex + "_new.sbc")) 610 | { 611 | File.Delete(cachedIndex + "_new.sbc"); 612 | } 613 | //File.WriteAllText(cachedIndex + "_new.sbc", jsonString); 614 | File.WriteAllBytes(cachedIndex + "_new.sbc", binary); 615 | 616 | if (File.Exists(cachedIndex + "_bak.sbc")) 617 | { 618 | File.Delete(cachedIndex + "_bak.sbc"); 619 | } 620 | if (File.Exists(cachedIndex + ".sbc")) 621 | { 622 | File.Move(cachedIndex + ".sbc", cachedIndex + "_bak.sbc"); 623 | } 624 | File.Move(cachedIndex + "_new.sbc", cachedIndex + ".sbc"); 625 | break; 626 | } 627 | catch (Exception ex) 628 | { 629 | Thread.Sleep(1000); 630 | } 631 | } 632 | 633 | DestinationIndex.Modified = false; 634 | } 635 | 636 | 637 | private void SaveCacheCheck() 638 | { 639 | if ((DateTime.Now - DestinationIndex.LastSaveTime).TotalSeconds > AutoSaveTime) 640 | { 641 | lock (DestinationIndex) 642 | { 643 | if (DestinationIndex.SaveThread == null) 644 | { 645 | DestinationIndex.SaveThread = new Thread(() => 646 | { 647 | try 648 | { 649 | SaveCache(); 650 | } 651 | catch (Exception ex) 652 | { 653 | } 654 | DestinationIndex.SaveThread = null; 655 | }); 656 | 657 | try 658 | { 659 | DestinationIndex.SaveThread.Start(); 660 | } 661 | catch (Exception ex) 662 | { 663 | } 664 | } 665 | } 666 | } 667 | } 668 | 669 | private void TreeSetAttributes(TreeInfo root, string dst, eType fileType, DateTime lastChange, long length) 670 | { 671 | DestinationIndex.Modified = true; 672 | 673 | string path = dst.Substring(root.Root.Name.Length); 674 | var elems = path.Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries); 675 | 676 | TreeNode node = root.Root; 677 | 678 | for (int pos = 0; pos < elems.Length; pos++) 679 | { 680 | var elem = elems[pos]; 681 | TreeNode next = node.Children.Where(c => c.Name == elem).FirstOrDefault(); 682 | 683 | if (next == null) 684 | { 685 | next = new TreeNode() 686 | { 687 | Name = elem, 688 | Type = (pos == elems.Length - 1) ? fileType : eType.Directory 689 | }; 690 | var list = node.Children.ToList(); 691 | list.Add(next); 692 | node.Children = list.ToArray(); 693 | } 694 | node = next; 695 | } 696 | 697 | node.LastChange = ToUnixTime(lastChange); 698 | node.Length = length; 699 | 700 | SaveCacheCheck(); 701 | } 702 | 703 | private long ToUnixTime(DateTime lastChange) 704 | { 705 | long unixTimestamp = (long)(lastChange - UnixEpoch).TotalSeconds; 706 | return unixTimestamp; 707 | } 708 | 709 | private void TreeDeleteEntry(TreeInfo root, string dst) 710 | { 711 | DestinationIndex.Modified = true; 712 | 713 | string path = dst.Substring(root.Root.Name.Length); 714 | var elems = path.Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries); 715 | 716 | TreeNode node = root.Root; 717 | 718 | for (int pos = 0; pos < elems.Length; pos++) 719 | { 720 | var elem = elems[pos]; 721 | 722 | if (pos == elems.Length - 1) 723 | { 724 | node.Children = node.Children.Where(c => c.Name != elem).ToArray(); 725 | return; 726 | } 727 | 728 | TreeNode next = node.Children.Where(c => c.Name == elem).FirstOrDefault(); 729 | 730 | if (next == null) 731 | { 732 | return; 733 | } 734 | node = next; 735 | } 736 | 737 | SaveCacheCheck(); 738 | } 739 | 740 | private void DeleteFile(TreeNode ent) 741 | { 742 | var fi = new FileInfo(ent.FullPath); 743 | 744 | AddMessage("[VERBOSE] DELETE '" + ent.FullPath + "' (" + FormatSize(fi.Length) + ")"); 745 | 746 | if (!fi.FullName.StartsWith(DestinationFolder)) 747 | { 748 | AddMessage("[ERROR]: Deleting a file that is not within destination folder"); 749 | return; 750 | } 751 | 752 | bool success = false; 753 | 754 | for (int retry = 0; retry < 5; retry++) 755 | { 756 | try 757 | { 758 | if (File.Exists(fi.FullName)) 759 | { 760 | File.Delete(fi.FullName); 761 | } 762 | success = true; 763 | break; 764 | } 765 | catch (Exception ex) 766 | { 767 | Thread.Sleep(10); 768 | } 769 | } 770 | 771 | if (success) 772 | { 773 | } 774 | else 775 | { 776 | AddMessage("[ERROR]: failed to delete"); 777 | } 778 | } 779 | 780 | private void DeleteDirectory(TreeNode ent) 781 | { 782 | var di = new DirectoryInfo(ent.FullPath); 783 | AddMessage("DELETE Dir: " + di.FullName); 784 | 785 | if (!di.FullName.StartsWith(DestinationFolder)) 786 | { 787 | AddMessage("[ERROR]: Deleting a directory that is not within destination folder"); 788 | return; 789 | } 790 | 791 | bool success = false; 792 | 793 | for (int retry = 0; retry < 5; retry++) 794 | { 795 | try 796 | { 797 | if (Directory.Exists(di.FullName)) 798 | { 799 | Directory.Delete(di.FullName, true); 800 | } 801 | success = true; 802 | break; 803 | } 804 | catch (Exception ex) 805 | { 806 | Thread.Sleep(10); 807 | } 808 | } 809 | 810 | if (success) 811 | { 812 | } 813 | else 814 | { 815 | AddMessage("[ERROR] failed to delete"); 816 | } 817 | } 818 | 819 | private bool CopyFile(string src, string dst) 820 | { 821 | try 822 | { 823 | SourceIndex.CurrentEntity = src; 824 | DestinationIndex.CurrentEntity = dst; 825 | 826 | var srcInfo = new FileInfo(src); 827 | var dstInfo = new FileInfo(dst); 828 | 829 | /* first delete the existing file to make help any defect file gets removed before */ 830 | if (File.Exists(dst)) 831 | { 832 | File.SetAttributes(dst, FileAttributes.Normal); 833 | File.Delete(dst); 834 | } 835 | /* also make sure there is no directory with that name */ 836 | if (Directory.Exists(dst)) 837 | { 838 | Directory.Delete(dst, true); 839 | } 840 | 841 | /* make sure destination path exists */ 842 | if (!Directory.Exists(dstInfo.Directory.FullName)) 843 | { 844 | string dirs = ""; 845 | foreach (string dir in dstInfo.Directory.FullName.Split(Path.DirectorySeparatorChar)) 846 | { 847 | dirs += dir + Path.DirectorySeparatorChar; 848 | 849 | if (!Directory.Exists(dirs)) 850 | { 851 | for (int retry = 0; retry < 10; retry++) 852 | { 853 | try 854 | { 855 | /* if part of that path is a file, delete it */ 856 | if(File.Exists(dirs.TrimEnd(Path.DirectorySeparatorChar))) 857 | { 858 | File.Delete(dirs.TrimEnd(Path.DirectorySeparatorChar)); 859 | } 860 | if (!Directory.Exists(dirs)) 861 | { 862 | Directory.CreateDirectory(dirs); 863 | } 864 | Directory.SetCreationTime(dirs, srcInfo.Directory.CreationTime); 865 | Directory.SetLastWriteTime(dirs, srcInfo.Directory.LastWriteTime); 866 | Directory.SetLastAccessTime(dirs, srcInfo.Directory.LastAccessTime); 867 | break; 868 | } 869 | catch (Exception e) 870 | { 871 | if (retry == 10) 872 | { 873 | AddMessage("[ERROR] COPY/ATTRIBUTES: " + src + " -> " + dst + " -> " + e.Message); 874 | } 875 | Thread.Sleep(10); 876 | } 877 | } 878 | } 879 | } 880 | } 881 | 882 | srcInfo.CopyTo(dst, true); 883 | //File.Copy(src, dst, true); 884 | 885 | if (!File.Exists(dst)) 886 | { 887 | return false; 888 | } 889 | 890 | for (int retry = 0; retry < 10; retry++) 891 | { 892 | try 893 | { 894 | /* update file access times. seen to crash */ 895 | File.SetAttributes(dst, FileAttributes.Normal); 896 | File.SetLastWriteTime(dst, srcInfo.LastWriteTime); 897 | File.SetCreationTime(dst, srcInfo.CreationTime); 898 | File.SetLastAccessTime(dst, srcInfo.LastAccessTime); 899 | File.SetAttributes(dst, srcInfo.Attributes); 900 | 901 | /* update parent directoy access times */ 902 | Directory.SetCreationTime(dstInfo.Directory.FullName, srcInfo.Directory.CreationTime); 903 | Directory.SetLastWriteTime(dstInfo.Directory.FullName, srcInfo.Directory.LastWriteTime); 904 | Directory.SetLastAccessTime(dstInfo.Directory.FullName, srcInfo.Directory.LastAccessTime); 905 | break; 906 | } 907 | catch (Exception e) 908 | { 909 | if (retry == 10) 910 | { 911 | AddMessage("[ERROR] COPY/ATTRIBUTES: " + src + " -> " + dst + " -> " + e.Message); 912 | } 913 | Thread.Sleep(10); 914 | } 915 | } 916 | 917 | TreeSetAttributes(DestinationIndex, dst, eType.File, srcInfo.LastWriteTime, srcInfo.Length); 918 | return true; 919 | } 920 | catch (IOException ex) 921 | { 922 | /* E_SHARING_VIOLATION */ 923 | if((uint)ex.HResult == 0x80070020) 924 | { 925 | AddMessage("[BUSY] COPY: " + src + " -> " + dst + " -> " + ex.Message); 926 | } 927 | else 928 | { 929 | AddMessage("[ERROR] COPY: " + src + " -> " + dst + " -> " + ex.Message); 930 | } 931 | } 932 | catch (Exception ex) 933 | { 934 | AddMessage("[ERROR] COPY: " + src + " -> " + dst + " -> " + ex.Message); 935 | } 936 | return false; 937 | } 938 | 939 | private void CopyDirectory(string src, string dst) 940 | { 941 | try 942 | { 943 | SourceIndex.CurrentEntity = src; 944 | DestinationIndex.CurrentEntity = dst; 945 | 946 | if (File.Exists(dst)) 947 | { 948 | File.SetAttributes(dst, FileAttributes.Normal); 949 | File.Delete(dst); 950 | } 951 | if (Directory.Exists(dst)) 952 | { 953 | Directory.Delete(dst); 954 | } 955 | 956 | Directory.CreateDirectory(dst); 957 | 958 | foreach (var file in new DirectoryInfo(src).GetFiles()) 959 | { 960 | CopyFile(Path.Combine(src, file.Name), Path.Combine(dst, file.Name)); 961 | } 962 | foreach (var dir in new DirectoryInfo(src).GetDirectories()) 963 | { 964 | CopyDirectory(Path.Combine(src, dir.Name), Path.Combine(dst, dir.Name)); 965 | } 966 | 967 | var info = new DirectoryInfo(src); 968 | 969 | Directory.SetLastWriteTime(dst, info.LastWriteTime); 970 | Directory.SetCreationTime(dst, info.CreationTime); 971 | Directory.SetLastAccessTime(dst, info.LastAccessTime); 972 | 973 | TreeSetAttributes(DestinationIndex, dst, eType.Directory, info.LastWriteTime, 0); 974 | DirectoriesCopied++; 975 | } 976 | catch (Exception ex) 977 | { 978 | AddMessage("[ERROR] COPY: " + src + " -> " + dst + " -> (" + ex.HResult.ToString("X8") + ") " + ex.Message); 979 | } 980 | } 981 | 982 | 983 | private void IndexDirectory(TreeInfo info, string path, TreeNode node, bool filterActive = false, int level = 0) 984 | { 985 | info.CurrentEntity = path; 986 | info.Modified = true; 987 | 988 | EnumerationOptions opts = new EnumerationOptions 989 | { 990 | AttributesToSkip = 0, 991 | IgnoreInaccessible = false 992 | }; 993 | 994 | List entries = new List(); 995 | IEnumerable filesAll = new DirectoryInfo(path).EnumerateFiles("*", opts); 996 | IEnumerable files = filesAll; 997 | if (filterActive) 998 | { 999 | files = filesAll.Where(f => !IsIgnored(f)); 1000 | } 1001 | if (level == 0) 1002 | { 1003 | files = files.Where(f => !f.Name.EndsWith(".sbc")); 1004 | } 1005 | 1006 | foreach (var fileInfo in files) 1007 | { 1008 | TreeNode n = new TreeNode 1009 | { 1010 | Name = fileInfo.Name, 1011 | Length = fileInfo.Length, 1012 | Type = eType.File, 1013 | LastChange = ToUnixTime(fileInfo.LastWriteTime) 1014 | }; 1015 | 1016 | entries.Add(n); 1017 | info.IndexedFiles++; 1018 | info.IndexedSize += fileInfo.Length; 1019 | } 1020 | 1021 | IEnumerable dirsAll = new DirectoryInfo(path).EnumerateDirectories("*", opts); 1022 | IEnumerable dirs = dirsAll; 1023 | 1024 | if (filterActive) 1025 | { 1026 | dirs = dirsAll.Where(d => !IsIgnored(d)); 1027 | } 1028 | 1029 | foreach (var dirInfo in dirs) 1030 | { 1031 | TreeNode n = new TreeNode 1032 | { 1033 | Name = dirInfo.Name, 1034 | Type = eType.Directory, 1035 | LastChange = ToUnixTime(dirInfo.LastWriteTime) 1036 | }; 1037 | 1038 | entries.Add(n); 1039 | } 1040 | 1041 | node.Children = entries.ToArray(); 1042 | 1043 | foreach (TreeNode dirNode in entries.Where(e => e.Type == eType.Directory)) 1044 | { 1045 | try 1046 | { 1047 | IndexDirectory(info, Path.Combine(path, dirNode.Name), dirNode, filterActive, level + 1); 1048 | } 1049 | catch (Exception ex) 1050 | { 1051 | } 1052 | } 1053 | 1054 | info.IndexedDirectories++; 1055 | } 1056 | 1057 | private bool IsIgnored(FileInfo f) 1058 | { 1059 | bool ret = IgnoreList.Where(i => f.FullName.Contains(i)).Any(); 1060 | ret |= IgnoreList.Where(i => i.StartsWith("^") && i.EndsWith("$")).Any(i => ("^" + f.Name + "$") == i); 1061 | ret |= IgnoreList.Where(i => i.StartsWith("^")).Any(i => ("^" + f.Name) == i); 1062 | ret |= IgnoreList.Where(i => i.EndsWith("$")).Any(i => (f.Name + "$") == i); 1063 | 1064 | if (ret) 1065 | { 1066 | AddMessage("Ignored: " + f.FullName); 1067 | } 1068 | 1069 | return ret; 1070 | } 1071 | 1072 | 1073 | private bool IsIgnored(DirectoryInfo f) 1074 | { 1075 | bool ret = IgnoreList.Where(i => (f.FullName + Path.DirectorySeparatorChar).Contains(i)).Any(); 1076 | 1077 | if (ret) 1078 | { 1079 | AddMessage("Ignored: " + f.FullName); 1080 | } 1081 | 1082 | return ret; 1083 | } 1084 | 1085 | internal void CopyFiles() 1086 | { 1087 | State = eState.Copy; 1088 | 1089 | var ent = FileCopyQueue.ToArray(); 1090 | Parallel.ForEach(ent, ParallelOptions, pair => 1091 | //foreach (var pair in ent) 1092 | { 1093 | string src = pair.Key; 1094 | string dst = pair.Value; 1095 | long length = 0; 1096 | 1097 | try 1098 | { 1099 | var srcInfo = new FileInfo(src); 1100 | length = srcInfo.Length; 1101 | } 1102 | catch(Exception ex) 1103 | { 1104 | } 1105 | 1106 | if (Directory.Exists(src)) 1107 | { 1108 | if (File.Exists(dst)) 1109 | { 1110 | AddMessage("[ERROR] Could not create directory '" + dst + "' as there is already a file with that name."); 1111 | } 1112 | else if (!Directory.Exists(dst)) 1113 | { 1114 | try 1115 | { 1116 | Directory.CreateDirectory(dst); 1117 | } 1118 | catch (Exception ex) 1119 | { 1120 | AddMessage("[ERROR] Could not create directory '" + dst + "': " + ex.Message); 1121 | } 1122 | } 1123 | } 1124 | else if (File.Exists(src)) 1125 | { 1126 | AddMessage("[VERBOSE] Copy '" + src + "' (" + FormatSize(length) + ")"); 1127 | if (CopyFile(src, dst)) 1128 | { 1129 | FilesCopied++; 1130 | SizeCopied += length; 1131 | DestinationIndex.IndexedSize += length; 1132 | } 1133 | } 1134 | else 1135 | { 1136 | AddMessage("[ERROR] File '" + src + "' was missing during copy. Maybe a temporary file?"); 1137 | } 1138 | 1139 | FileCopyQueue.Remove(src); 1140 | }); 1141 | } 1142 | 1143 | private void AddMessage(string v) 1144 | { 1145 | Log(v); 1146 | 1147 | lock (Messages) 1148 | { 1149 | Messages.Add(v); 1150 | } 1151 | } 1152 | 1153 | internal void DeleteFiles() 1154 | { 1155 | State = eState.Delete; 1156 | 1157 | var entities = FileDeleteQueue.ToArray(); 1158 | Parallel.ForEach(entities, ParallelOptions, ent => 1159 | //foreach (TreeNode ent in entities) 1160 | { 1161 | long files = ent.FilesRecursive; 1162 | long size = ent.SizeRecursive; 1163 | if (Directory.Exists(ent.FullPath)) 1164 | { 1165 | DeleteDirectory(ent); 1166 | } 1167 | else if (File.Exists(ent.FullPath)) 1168 | { 1169 | DeleteFile(ent); 1170 | } 1171 | 1172 | FilesDeleted += files; 1173 | SizeDeleted += size; 1174 | DestinationIndex.IndexedSize -= size; 1175 | 1176 | TreeDeleteEntry(DestinationIndex, ent.FullPath); 1177 | 1178 | FileDeleteQueue.Remove(ent); 1179 | }); 1180 | } 1181 | 1182 | internal void UpdateFiles() 1183 | { 1184 | State = eState.Update; 1185 | 1186 | var files = FileUpdateQueue.ToArray(); 1187 | Parallel.ForEach(files, ParallelOptions, pair => 1188 | //foreach (var pair in files) 1189 | { 1190 | string src = pair.Key; 1191 | string dst = pair.Value; 1192 | 1193 | try 1194 | { 1195 | var srcInfo = new FileInfo(src); 1196 | var dstInfo = new FileInfo(dst); 1197 | var info = new FileInfo(src); 1198 | 1199 | SourceIndex.CurrentEntity = src; 1200 | DestinationIndex.CurrentEntity = dst; 1201 | 1202 | AddMessage("[VERBOSE] UPDATE '" + src + "' (" + FormatSize(srcInfo.Length) + ")"); 1203 | 1204 | if (srcInfo.Length != dstInfo.Length) 1205 | { 1206 | CopyFile(src, dst); 1207 | } 1208 | else if (FilesDiffer(src, dst)) 1209 | { 1210 | CopyFile(src, dst); 1211 | } 1212 | else 1213 | { 1214 | /* just an attribute update */ 1215 | File.SetAttributes(dst, FileAttributes.Normal); 1216 | File.SetLastWriteTime(dst, info.LastWriteTime); 1217 | File.SetCreationTime(dst, info.CreationTime); 1218 | File.SetLastAccessTime(dst, info.LastAccessTime); 1219 | } 1220 | 1221 | FilesUpdated++; 1222 | SizeUpdated += srcInfo.Length; 1223 | DestinationIndex.IndexedSize -= dstInfo.Length; 1224 | DestinationIndex.IndexedSize += srcInfo.Length; 1225 | 1226 | File.SetAttributes(dst, info.Attributes); 1227 | 1228 | TreeSetAttributes(DestinationIndex, dst, eType.File, info.LastWriteTime, info.Length); 1229 | } 1230 | catch (Exception ex) 1231 | { 1232 | AddMessage("[ERROR] UPDATE: " + src + " -> " + dst + " -> " + ex.Message); 1233 | } 1234 | 1235 | FileUpdateQueue.Remove(src); 1236 | }); 1237 | } 1238 | 1239 | private bool FilesDiffer(string src, string dst) 1240 | { 1241 | try 1242 | { 1243 | if (FileChecksum(src) != FileChecksum(dst)) 1244 | { 1245 | return true; 1246 | } 1247 | } 1248 | catch (Exception ex) 1249 | { 1250 | AddMessage("[ERROR] HASH: " + src + " -> " + dst + " -> " + ex.Message); 1251 | } 1252 | return false; 1253 | } 1254 | 1255 | 1256 | private static string FormatSize(decimal size) 1257 | { 1258 | string[] units = new[] { "Byte", "KiB", "MiB", "GiB", "TiB" }; 1259 | int unit = 0; 1260 | 1261 | while (size > 1024 && unit < units.Length - 1) 1262 | { 1263 | size /= 1024; 1264 | unit++; 1265 | } 1266 | 1267 | return size.ToString("0.00") + " " + units[unit]; 1268 | } 1269 | internal void Finish() 1270 | { 1271 | try 1272 | { 1273 | List messages = new(); 1274 | 1275 | AddMessage("Title: " + Title); 1276 | AddMessage(" Source size: " + (SourceIndex.IndexedFiles + " (" + FormatSize(SourceIndex.IndexedSize) + ")")); 1277 | AddMessage(" Destination size: " + (DestinationIndex.IndexedFiles + " (" + FormatSize(DestinationIndex.IndexedSize) + ")")); 1278 | AddMessage(" Copied: " + (FilesCopied + " (" + FormatSize(SizeCopied) + ")")); 1279 | AddMessage(" Deleted: " + (FilesDeleted + " (" + FormatSize(SizeDeleted) + ")")); 1280 | } 1281 | catch (Exception ex) 1282 | { 1283 | } 1284 | State = eState.Done; 1285 | } 1286 | 1287 | internal void Execute() 1288 | { 1289 | if (!LockBackupFolder()) 1290 | { 1291 | return; 1292 | } 1293 | try 1294 | { 1295 | BuildTree(); 1296 | MatchFiles(); 1297 | DeleteFiles(); 1298 | CopyFiles(); 1299 | UpdateFiles(); 1300 | } 1301 | catch (Exception e) 1302 | { 1303 | } 1304 | finally 1305 | { 1306 | UnlockBackupFolder(); 1307 | Finish(); 1308 | SaveCache(); 1309 | } 1310 | } 1311 | 1312 | private void UnlockBackupFolder() 1313 | { 1314 | LockFileHandle.Close(); 1315 | try 1316 | { 1317 | File.Delete(LockFileHandle.Name); 1318 | } 1319 | catch (Exception ex) 1320 | { 1321 | } 1322 | } 1323 | 1324 | private bool LockBackupFolder() 1325 | { 1326 | string lockFile = Path.Combine(DestinationFolder, "_dst_lock.sbc"); 1327 | 1328 | if (File.Exists(lockFile)) 1329 | { 1330 | try 1331 | { 1332 | File.Delete(lockFile); 1333 | } 1334 | catch (Exception ex) 1335 | { 1336 | return false; 1337 | } 1338 | } 1339 | 1340 | try 1341 | { 1342 | LockFileHandle = File.Open(lockFile, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None); 1343 | return true; 1344 | } 1345 | catch (Exception ex) 1346 | { 1347 | } 1348 | 1349 | return false; 1350 | } 1351 | } 1352 | } 1353 | --------------------------------------------------------------------------------