├── FastPersistentDictionary.Benchmark ├── Benchmarks.cs ├── FastPersistentDictionary.Benchmark.csproj └── Program.cs ├── FastPersistentDictionary.Console ├── FastPersistentDictionary.Console.csproj └── Program.cs ├── FastPersistentDictionary.Tests ├── AllTests.cs ├── ConstructorTests.cs ├── DisposeTests.cs ├── FastPersistentDictionary.Tests.csproj ├── IndexTests.cs └── Usings.cs ├── FastPersistentDictionary.sln ├── FastPersistentDictionary ├── FastPersistentDictionary.cs ├── FastPersistentDictionary.csproj └── Internals │ ├── Accessor │ ├── DictionaryAccessor.cs │ ├── DictionaryAccessor_RecoverMode.cs │ └── IDictionaryAccessor.cs │ ├── Compression │ ├── ICompressionHandler.cs │ ├── SerializerCompress.cs │ └── SerializerUnCompressed.cs │ ├── DictionaryIO.cs │ ├── DictionaryMathOperations.cs │ ├── DictionaryObject.cs │ ├── DictionaryOperations.cs │ ├── DictionaryQuery.cs │ ├── DictionaryStructs.cs │ └── KeyValueComparer.cs ├── LICENSE └── README.md /FastPersistentDictionary.Benchmark/Benchmarks.cs: -------------------------------------------------------------------------------- 1 | using BenchmarkDotNet.Attributes; 2 | using Microsoft.Isam.Esent.Collections.Generic; 3 | 4 | namespace FastPersistentDictionary.Benchmark 5 | { 6 | public class Benchmarks 7 | { 8 | private FastPersistentDictionary fastPersistentDictionary; 9 | private FastPersistentDictionary fastPersistentDictionaryRecover; 10 | private Dictionary standardDictionary; 11 | private PersistentDictionary esentPersistentDictionary; 12 | 13 | [Params(10_000, 100_000, 1_000_000)]//, 100_000, 1_000_000)] 14 | public int N; 15 | 16 | [GlobalSetup] 17 | public void GlobalSetup() 18 | { 19 | string directoryEsent = "C:\\Users\\Admin\\Documents\\Copiers\\New folder\\Esent"; 20 | string fastPersistentDictionary = "C:\\Users\\Admin\\Documents\\Copiers\\New folder\\dict.tt"; 21 | string fastPersistentDictionaryRec = "C:\\Users\\Admin\\Documents\\Copiers\\New folder\\dictRec.tt"; 22 | 23 | this.fastPersistentDictionary = new FastPersistentDictionary(path: fastPersistentDictionary); 24 | this.fastPersistentDictionaryRecover = new FastPersistentDictionary(path: fastPersistentDictionaryRec, crashRecovery: true); 25 | standardDictionary = new Dictionary(); 26 | esentPersistentDictionary = new PersistentDictionary(directoryEsent); 27 | esentPersistentDictionary.Clear(); 28 | this.fastPersistentDictionary.Clear(); 29 | this.fastPersistentDictionaryRecover.Clear(); 30 | 31 | for (int i = 0; i < N; i++) 32 | { 33 | string key = $"Key{i}"; 34 | string value = $"Value{i} {longSentences[i % longSentences.Length]}"; 35 | this.fastPersistentDictionary.Add(key, value); 36 | this.fastPersistentDictionaryRecover.Add(key, value); 37 | standardDictionary.Add(key, value); 38 | esentPersistentDictionary.Add(key, value); 39 | } 40 | } 41 | 42 | [IterationSetup(Target = nameof(FastPersistentDictionary_Add))] 43 | public void IterationSetupForAddPersistent() 44 | { 45 | fastPersistentDictionary.Clear(); 46 | } 47 | 48 | [IterationSetup(Target = nameof(FastPersistentDictionaryRecover_Add))] 49 | public void IterationSetupForAddPersistentRecover() 50 | { 51 | fastPersistentDictionaryRecover.Clear(); 52 | } 53 | 54 | [IterationSetup(Target = nameof(StandardDictionary_Add))] 55 | public void IterationSetupForAddStandard() 56 | { 57 | standardDictionary.Clear(); 58 | } 59 | 60 | [IterationSetup(Target = nameof(EsentPersistentDictionary_Add))] 61 | public void IterationSetupForAddEsent() 62 | { 63 | esentPersistentDictionary.Clear(); 64 | } 65 | 66 | [IterationSetup(Targets = new[] { nameof(FastPersistentDictionary_Get), nameof(FastPersistentDictionary_Remove) })] 67 | public void IterationSetupForGetOrRemovePersistent() 68 | { 69 | fastPersistentDictionary.Clear(); 70 | for (int i = 0; i < N; i++) 71 | { 72 | string key = $"Key{i}"; 73 | string value = $"Value{i} {longSentences[i % longSentences.Length]}"; 74 | fastPersistentDictionary.Add(key, value); 75 | } 76 | } 77 | 78 | [IterationSetup(Targets = new[] { nameof(FastPersistentDictionaryRecover_Get), nameof(FastPersistentDictionaryRecover_Remove) })] 79 | public void IterationSetupForGetOrRemovePersistentRecover() 80 | { 81 | fastPersistentDictionaryRecover.Clear(); 82 | for (int i = 0; i < N; i++) 83 | { 84 | string key = $"Key{i}"; 85 | string value = $"Value{i} {longSentences[i % longSentences.Length]}"; 86 | fastPersistentDictionaryRecover.Add(key, value); 87 | } 88 | } 89 | 90 | [IterationSetup(Targets = new[] { nameof(StandardDictionary_Get), nameof(StandardDictionary_Remove) })] 91 | public void IterationSetupForGetOrRemoveStandard() 92 | { 93 | standardDictionary.Clear(); 94 | for (int i = 0; i < N; i++) 95 | { 96 | string key = $"Key{i}"; 97 | string value = $"Value{i} {longSentences[i % longSentences.Length]}"; 98 | standardDictionary.Add(key, value); 99 | } 100 | } 101 | 102 | [IterationSetup(Targets = new[] { nameof(EsentPersistentDictionary_Get), nameof(EsentPersistentDictionary_Remove) })] 103 | public void IterationSetupForGetOrRemoveEsent() 104 | { 105 | esentPersistentDictionary.Clear(); 106 | for (int i = 0; i < N; i++) 107 | { 108 | string key = $"Key{i}"; 109 | string value = $"Value{i} {longSentences[i % longSentences.Length]}"; 110 | esentPersistentDictionary[key] = value; 111 | } 112 | } 113 | 114 | [Benchmark] 115 | public void FastPersistentDictionary_Add() 116 | { 117 | for (int i = 0; i < N; i++) 118 | { 119 | string key = $"Key{i}"; 120 | string value = $"Value{i} {longSentences[i % longSentences.Length]}"; 121 | fastPersistentDictionary.Add(key, value); 122 | } 123 | } 124 | 125 | [Benchmark] 126 | public void FastPersistentDictionaryRecover_Add() 127 | { 128 | for (int i = 0; i < N; i++) 129 | { 130 | string key = $"Key{i}"; 131 | string value = $"Value{i} {longSentences[i % longSentences.Length]}"; 132 | fastPersistentDictionaryRecover.Add(key, value); 133 | } 134 | } 135 | 136 | [Benchmark] 137 | public void StandardDictionary_Add() 138 | { 139 | for (int i = 0; i < N; i++) 140 | { 141 | string key = $"Key{i}"; 142 | string value = $"Value{i} {longSentences[i % longSentences.Length]}"; 143 | standardDictionary.Add(key, value); 144 | } 145 | } 146 | 147 | [Benchmark] 148 | public void EsentPersistentDictionary_Add() 149 | { 150 | for (int i = 0; i < N; i++) 151 | { 152 | string key = $"Key{i}"; 153 | string value = $"Value{i} {longSentences[i % longSentences.Length]}"; 154 | esentPersistentDictionary.Add(key, value); 155 | } 156 | } 157 | 158 | [Benchmark] 159 | public void FastPersistentDictionary_Get() 160 | { 161 | for (int i = 0; i < N; i++) 162 | { 163 | string key = $"Key{i}"; 164 | string value = fastPersistentDictionary[key]; 165 | } 166 | } 167 | 168 | [Benchmark] 169 | public void FastPersistentDictionaryRecover_Get() 170 | { 171 | for (int i = 0; i < N; i++) 172 | { 173 | string key = $"Key{i}"; 174 | string value = fastPersistentDictionaryRecover[key]; 175 | } 176 | } 177 | 178 | [Benchmark] 179 | public void StandardDictionary_Get() 180 | { 181 | for (int i = 0; i < N; i++) 182 | { 183 | string key = $"Key{i}"; 184 | string value = standardDictionary[key]; 185 | } 186 | } 187 | 188 | [Benchmark] 189 | public void EsentPersistentDictionary_Get() 190 | { 191 | for (int i = 0; i < N; i++) 192 | { 193 | string key = $"Key{i}"; 194 | string value = esentPersistentDictionary[key]; 195 | } 196 | } 197 | 198 | [Benchmark] 199 | public void FastPersistentDictionary_Remove() 200 | { 201 | for (int i = 0; i < N; i++) 202 | { 203 | string key = $"Key{i}"; 204 | fastPersistentDictionary.Remove(key); 205 | } 206 | } 207 | 208 | [Benchmark] 209 | public void FastPersistentDictionaryRecover_Remove() 210 | { 211 | for (int i = 0; i < N; i++) 212 | { 213 | string key = $"Key{i}"; 214 | fastPersistentDictionaryRecover.Remove(key); 215 | } 216 | } 217 | 218 | [Benchmark] 219 | public void StandardDictionary_Remove() 220 | { 221 | for (int i = 0; i < N; i++) 222 | { 223 | string key = $"Key{i}"; 224 | standardDictionary.Remove(key); 225 | } 226 | } 227 | 228 | [Benchmark] 229 | public void EsentPersistentDictionary_Remove() 230 | { 231 | for (int i = 0; i < N; i++) 232 | { 233 | string key = $"Key{i}"; 234 | esentPersistentDictionary.Remove(key); 235 | } 236 | } 237 | 238 | string[] longSentences = new string[] 239 | { 240 | "In an unprecedented turn of events, the local community rallied together to support small businesses, showcasing a remarkable spirit of unity and resilience that had not been seen in decades.", 241 | "The picturesque village nestled in the foothills of the mountain range was often described as a hidden gem, attracting tourists who were eager to escape the hustle and bustle of city life.", 242 | "Under the sprawling oak tree that had stood for centuries, the children listened intently to their grandfather's stories, each tale more fascinating than the last.", 243 | "As the sun set over the horizon, casting a golden hue across the landscape, the weary travelers found solace in the serene beauty of their surroundings.", 244 | "In a meticulously orchestrated plan, the team of scientists embarked on a groundbreaking mission to explore the uncharted depths of the ocean.", 245 | "With each passing day, the artist's masterpiece began to take shape, a testament to her unwavering dedication and unparalleled talent.", 246 | "Despite numerous challenges, the project was completed on time, a feat that many had deemed impossible at the outset.", 247 | "Standing at the edge of the cliff, he felt a surge of adrenaline as he prepared to take the leap, trusting that his parachute would deploy as intended.", 248 | "The aroma of freshly baked bread wafted through the air, drawing customers to the quaint bakery that had become a beloved fixture in the neighborhood.", 249 | "Through rigorous training and sheer determination, she had transformed herself from a novice into a formidable athlete, ready to compete at the highest level.", 250 | "In a heartfelt speech, the retiring professor reflected on his decades-long career, expressing gratitude for the students who had inspired him along the way.", 251 | "The ancient manuscript, filled with cryptic symbols and forgotten languages, held secrets that had eluded scholars for generations.", 252 | "As the CEO addressed the board of directors, he outlined a bold vision for the company's future, one that promised innovation and growth.", 253 | "Navigating the dense forest required both skill and intuition, as the path was often obscured by thick underbrush and towering trees.", 254 | "The state-of-the-art facility was designed to be both functional and aesthetically pleasing, setting a new standard in architectural excellence.", 255 | "Her eloquent speech captured the audience's attention, leaving a lasting impression on everyone who had the privilege to hear it.", 256 | "In the midst of the bustling city, the tranquil garden offered a refuge for those seeking a moment of peace and reflection.", 257 | "The innovative startup quickly gained traction, attracting investors who were eager to support its disruptive technology.", 258 | "His journey across the continent was filled with encounters that broadened his horizons and deepened his appreciation for diverse cultures.", 259 | "The sprawling estate, with its manicured lawns and opulent mansion, was a testament to the family's enduring legacy.", 260 | "The relentless pursuit of knowledge drove the scientist to push the boundaries of what was considered possible.", 261 | "In the aftermath of the storm, the community came together to rebuild, demonstrating a resilience and camaraderie that was truly inspiring.", 262 | "Her groundbreaking research challenged conventional wisdom, opening new avenues for exploration and discovery.", 263 | "The historical novel, rich with vivid descriptions and intricate plotlines, transported readers to a bygone era.", 264 | "Despite facing numerous setbacks, the entrepreneur's perseverance ultimately led to the creation of a successful enterprise.", 265 | "The charismatic leader's vision for the future resonated with many, igniting a movement that would leave a lasting impact.", 266 | "In the quiet hours of the morning, the artist found solace in her studio, where creativity flowed freely and boundaries ceased to exist.", 267 | "The grand ballroom, adorned with crystal chandeliers and lavish decorations, was the perfect setting for the evening's gala.", 268 | "Drawing inspiration from nature, the architect designed a structure that seamlessly blended with its surroundings.", 269 | "The seasoned detective pieced together the clues, unraveling a mystery that had perplexed many for years.", 270 | "The breathtaking view from the mountaintop was a reward well worth the arduous journey.", 271 | "In a rapidly evolving industry, staying ahead of the curve required constant innovation and adaptability." 272 | }; 273 | } 274 | } -------------------------------------------------------------------------------- /FastPersistentDictionary.Benchmark/FastPersistentDictionary.Benchmark.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Exe 5 | net8.0 6 | enable 7 | enable 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastPersistentDictionary.Benchmark/Program.cs: -------------------------------------------------------------------------------- 1 | using BenchmarkDotNet.Attributes; 2 | using BenchmarkDotNet.Running; 3 | using FastPersistentDictionary; 4 | using Microsoft.Isam.Esent.Collections.Generic; 5 | using System.Data.SqlTypes; 6 | 7 | namespace FastPersistentDictionary.Benchmark 8 | { 9 | public class BenchmarkPersistence 10 | { 11 | static void Main(string[] args) 12 | { 13 | var summary = BenchmarkRunner.Run(); 14 | } 15 | 16 | 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /FastPersistentDictionary.Console/FastPersistentDictionary.Console.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Exe 5 | net8.0 6 | enable 7 | enable 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /FastPersistentDictionary.Console/Program.cs: -------------------------------------------------------------------------------- 1 | using FastPersistentDictionary; 2 | using System.Diagnostics; 3 | 4 | internal class Program 5 | { 6 | static void Main(string[] args) 7 | { 8 | string strValueTest = "Star Wars (later retitled Star Wars: Episode IV – A New Hope) is a 1977 American epic space opera film written and directed by George Lucas."; 9 | string pathMain = Path.Combine(System.Environment.CurrentDirectory, "data.tt"); 10 | var fastPersistentDictionary = new FastPersistentDictionary(path: pathMain, crashRecovery: false); 11 | //path: pathMain, 12 | Stopwatch stopwatch = new Stopwatch(); 13 | stopwatch.Start(); 14 | for (int i = 0; i < 10000; i++) 15 | { 16 | fastPersistentDictionary.Add("name_" + i.ToString(), strValueTest); 17 | 18 | } 19 | for (int i = 0; i < 10000; i++) 20 | { 21 | fastPersistentDictionary.TryGetValue("name_" + i.ToString(), out var val); 22 | 23 | 24 | } 25 | 26 | 27 | for (int i = 0; i < 1000; i++) 28 | { 29 | fastPersistentDictionary.Remove("name_" + i.ToString()); 30 | 31 | 32 | } 33 | 34 | for (int i = 10001; i < 10100; i++) 35 | { 36 | fastPersistentDictionary.Add("name_" + i.ToString(), strValueTest); 37 | 38 | 39 | } 40 | 41 | 42 | 43 | fastPersistentDictionary.Dispose(); 44 | 45 | stopwatch.Stop(); 46 | Console.WriteLine(stopwatch.Elapsed.TotalMilliseconds); 47 | Console.ReadLine(); 48 | } 49 | } -------------------------------------------------------------------------------- /FastPersistentDictionary.Tests/ConstructorTests.cs: -------------------------------------------------------------------------------- 1 | namespace FastPersistentDictionary.Tests; 2 | 3 | public sealed partial class PersistentDictionaryProTests 4 | { 5 | [TestFixture] 6 | public class ConstructorTests 7 | { 8 | //[Test] 9 | //public void DefaultConstructor_SetsDefaultValues() 10 | //{ 11 | // var dictionary = new FastPersistentDictionary(); 12 | 13 | 14 | // Assert.AreEqual(0f, dictionary.PercentageChangeBeforeCompact); 15 | // Assert.IsNotNull(dictionary.DictionarySerializedLookup); 16 | // Assert.IsNotNull(dictionary.FileLocation); 17 | // Assert.IsNotNull(dictionary.FastPersistentDictionaryVersion); 18 | // Assert.IsNotNull(dictionary.FileStream); 19 | // Assert.IsNotNull(dictionary.DictionaryAccessor); 20 | // Assert.IsNotNull(dictionary.Comparer); 21 | // Assert.IsNotNull(dictionary.Count); 22 | // Assert.IsNotNull(dictionary.Keys); 23 | // Assert.IsNotNull(dictionary.Values); 24 | //} 25 | 26 | [Test] 27 | public void CustomConstructor_SetsCustomValues() 28 | { 29 | var path = "./tempdictSave.pcd"; 30 | var updateRate = 100; 31 | var percentageChangeBeforeCompact = 0.2f; 32 | var importSavedPersistentDictionaryPro = "savedDictionaryPath"; 33 | var equalityComparer = EqualityComparer.Default; 34 | 35 | 36 | var dictionary = new FastPersistentDictionary( 37 | path, 38 | false, 39 | updateRate, 40 | true, 41 | percentageChangeBeforeCompact, 42 | 8196, 43 | importSavedPersistentDictionaryPro, 44 | equalityComparer); 45 | 46 | 47 | Assert.AreEqual(percentageChangeBeforeCompact, dictionary.PercentageChangeBeforeCompact); 48 | // Assert.IsNotNull(dictionary.DictionarySerializedCache); 49 | Assert.IsNotNull(dictionary.DictionarySerializedLookup); 50 | // Assert.IsNotNull(dictionary.DictionaryChangedEntries); 51 | Assert.AreEqual(path, dictionary.FileLocation); 52 | Assert.IsNotNull(dictionary.FastPersistentDictionaryVersion); 53 | Assert.IsNotNull(dictionary.FileStream); 54 | Assert.IsNotNull(dictionary.DictionaryAccessor); 55 | Assert.AreEqual(equalityComparer, dictionary.Comparer); 56 | Assert.IsNotNull(dictionary.Count); 57 | Assert.IsNotNull(dictionary.Keys); 58 | Assert.IsNotNull(dictionary.Values); 59 | } 60 | } 61 | } -------------------------------------------------------------------------------- /FastPersistentDictionary.Tests/DisposeTests.cs: -------------------------------------------------------------------------------- 1 | namespace FastPersistentDictionary.Tests; 2 | 3 | public sealed partial class PersistentDictionaryProTests 4 | { 5 | [TestFixture] 6 | public class DisposeTests 7 | { 8 | [Test] 9 | public void Dispose_CallsDisposeMethod() 10 | { 11 | using (var dictionary = new FastPersistentDictionary()) 12 | { 13 | dictionary.Dispose(); 14 | } 15 | } 16 | 17 | [Test] 18 | public void Dispose_CleansUpResources() 19 | { 20 | var dictionary = new FastPersistentDictionary(); 21 | dictionary.Dispose(); 22 | Assert.IsNull(dictionary.FileStream); 23 | } 24 | 25 | [Test] 26 | public void Dispose_CanBeCalledMultipleTimes() 27 | { 28 | var dictionary = new FastPersistentDictionary(); 29 | 30 | dictionary.Dispose(); 31 | dictionary.Dispose(); 32 | } 33 | } 34 | } -------------------------------------------------------------------------------- /FastPersistentDictionary.Tests/FastPersistentDictionary.Tests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net7.0 5 | enable 6 | enable 7 | 8 | false 9 | true 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /FastPersistentDictionary.Tests/IndexTests.cs: -------------------------------------------------------------------------------- 1 | namespace FastPersistentDictionary.Tests; 2 | 3 | public sealed partial class PersistentDictionaryProTests 4 | { 5 | [TestFixture] 6 | public class IndexTests 7 | { 8 | [Test] 9 | public void Index_ReturnsDictionaryIndexedByKeys() 10 | { 11 | var dictionary = new FastPersistentDictionary(); 12 | dictionary.Add(1, "One"); 13 | dictionary.Add(2, "Two"); 14 | dictionary.Add(3, "Three"); 15 | 16 | var indexedDictionary = dictionary.Index(); 17 | 18 | Assert.AreEqual(3, indexedDictionary.Count); 19 | Assert.IsTrue(indexedDictionary.ContainsKey(1)); 20 | Assert.IsTrue(indexedDictionary.ContainsKey(2)); 21 | Assert.IsTrue(indexedDictionary.ContainsKey(3)); 22 | Assert.AreEqual(("One", 0), indexedDictionary[1]); 23 | Assert.AreEqual(("Two", 1), indexedDictionary[2]); 24 | Assert.AreEqual(("Three", 2), indexedDictionary[3]); 25 | } 26 | 27 | [Test] 28 | public void Index_ReturnsEmptyDictionaryForEmptyPersistentDictionaryPro() 29 | { 30 | var dictionary = new FastPersistentDictionary(); 31 | 32 | var indexedDictionary = dictionary.Index(); 33 | Assert.AreEqual(0, indexedDictionary.Count); 34 | } 35 | 36 | [Test] 37 | public void Index_ReturnsDictionaryIndexedByValues() 38 | { 39 | var dictionary = new FastPersistentDictionary(useCompression: false); 40 | //dictionary.Add(1, "Star Wars is an American epic space opera[1] media franchise created by George Lucas, which began with the eponymous 1977 film[a] and quickly became a worldwide pop culture phenomenon. The franchise has been expanded into various films and other media, including television series, video games, novels, comic books, theme park attractions, and themed areas, comprising an all-encompassing fictional universe.[b] Star Wars is one of the highest-grossing media franchises of all time.\r\n\r\nThe original 1977 film, retroactively subtitled Episode IV: A New Hope, was followed by the sequels Episode V: The Empire Strikes Back (1980) and Episode VI: Return of the Jedi (1983), forming the original Star Wars trilogy. Lucas later returned to the series to write and direct a prequel trilogy, consisting of Episode I: The Phantom Menace (1999), Episode II: Attack of the Clones (2002), and Episode III: Revenge of the Sith (2005). In 2012, Lucas sold his production company to Disney, relinquishing his ownership of the franchise. This led to a sequel trilogy, consisting of Episode VII: The Force Awakens (2015), Episode VIII: The Last Jedi (2017), and Episode IX: The Rise of Skywalker (2019).\r\n\r\nAll nine films, collectively referred to as the \"Skywalker Saga\", were nominated for Academy Awards, with wins going to the first two releases. Together with the theatrical live action \"anthology\" films Rogue One (2016) and Solo (2018), the combined box office revenue of the films equated to over US$10 billion, making Star Wars the third-highest-grossing film franchise of all time."); 41 | //dictionary.Add(2, "The Star Wars franchise depicts the adventures of characters \"A long time ago in a galaxy far, far away\"[3] across multiple fictional eras, in which humans and many species of aliens (often humanoid) co-exist with robots (typically referred to in the films as 'droids'), which may be programmed for personal assistance or battle.[4] Space travel between planets is common due to lightspeed hyperspace technology.[5][6][7] The planets range from wealthy, planet-wide cities to deserts scarcely populated by primitive tribes. Virtually any Earth biome, along with many fictional ones, has its counterpart as a Star Wars planet which, in most cases, teem with sentient and non-sentient alien life.[8] The franchise also makes use of other astronomical objects such as asteroid fields and nebulae.[9][10] Spacecraft range from small starfighters to large capital ships, such as the Star Destroyers, as well as space stations such as the moon-sized Death Stars.[11][12][13] Telecommunication includes two-way audio and audiovisual screens, holographic projections and hyperspace transmission.[14]"); 42 | //dictionary.Add(3, "The universe of Star Wars is generally similar to the real universe but its laws of physics are less strict allowing for more imaginative stories.[15] One result of that is a mystical power known as the Force which is described in the original film as \"an energy field created by all living things ... [that] binds the galaxy together\".[16] The field is depicted as a kind of pantheistic god.[17] Through training and meditation, those whom \"the Force is strong with\" exhibit various superpowers (such as telekinesis, precognition, telepathy, and manipulation of physical energy).[18] It is believed nothing is impossible for the Force.[19] The mentioned powers are wielded by two major knightly orders at conflict with each other: the Jedi, peacekeepers of the Galactic Republic who act on the light side of the Force through non-attachment and arbitration, and the Sith, who use the dark side by manipulating fear and aggression.[20][21] While Jedi Knights can be numerous, the Dark Lords of the Sith (or 'Darths') are intended to be limited to two: a master and their apprentice.[22]\r\n\r\nThe franchise is set against a backdrop of galactic conflict involving republics and empires, such as the evil Galactic Empire.[23] The Jedi and Sith prefer the use of a weapon called the lightsaber, a blade of plasma that can cut through virtually any surface and deflect energy bolts.[24] The rest of the population, as well as renegades and soldiers, use plasma-powered blaster firearms.[25] In the outer reaches of the galaxy, crime syndicates such as the Hutt cartel are dominant.[26] Bounty hunters are often employed by both gangsters and governments, while illicit activities include smuggling and slavery.[26]\r\n\r\nThe combination of science fiction and fantasy elements makes Star Wars a very universal franchise, capable of telling stories of various genres.[27]"); 43 | 44 | dictionary.Add(1, "One"); 45 | dictionary.Add(2, "Two"); 46 | dictionary.Add(3, "Three"); 47 | 48 | var indexedDictionary = dictionary.Index().ToDictionary(x => x.Value.value, x => x.Value.index); 49 | 50 | Assert.AreEqual(3, indexedDictionary.Count); 51 | Assert.IsTrue(indexedDictionary.ContainsKey("One")); 52 | Assert.IsTrue(indexedDictionary.ContainsKey("Two")); 53 | Assert.IsTrue(indexedDictionary.ContainsKey("Three")); 54 | Assert.AreEqual(0, indexedDictionary["One"]); 55 | Assert.AreEqual(1, indexedDictionary["Two"]); 56 | Assert.AreEqual(2, indexedDictionary["Three"]); 57 | } 58 | } 59 | } -------------------------------------------------------------------------------- /FastPersistentDictionary.Tests/Usings.cs: -------------------------------------------------------------------------------- 1 | global using NUnit.Framework; -------------------------------------------------------------------------------- /FastPersistentDictionary.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 17 4 | VisualStudioVersion = 17.5.33516.290 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FastPersistentDictionary", "FastPersistentDictionary\FastPersistentDictionary.csproj", "{A1611B46-9E2F-4901-ACD3-98C4A42A79D6}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FastPersistentDictionary.Console", "FastPersistentDictionary.Console\FastPersistentDictionary.Console.csproj", "{F69BCA7A-0501-4050-9DFC-DE5CB448ECAB}" 9 | EndProject 10 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FastPersistentDictionary.Benchmark", "FastPersistentDictionary.Benchmark\FastPersistentDictionary.Benchmark.csproj", "{28E405A5-D140-4BD5-9D6C-4EBEAE02FE79}" 11 | EndProject 12 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FastPersistentDictionary.Tests", "FastPersistentDictionary.Tests\FastPersistentDictionary.Tests.csproj", "{1BD44FE4-AD66-48F0-8695-BFD7EFF7E5A8}" 13 | EndProject 14 | Global 15 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 16 | Debug|Any CPU = Debug|Any CPU 17 | Release|Any CPU = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 20 | {A1611B46-9E2F-4901-ACD3-98C4A42A79D6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 21 | {A1611B46-9E2F-4901-ACD3-98C4A42A79D6}.Debug|Any CPU.Build.0 = Debug|Any CPU 22 | {A1611B46-9E2F-4901-ACD3-98C4A42A79D6}.Release|Any CPU.ActiveCfg = Release|Any CPU 23 | {A1611B46-9E2F-4901-ACD3-98C4A42A79D6}.Release|Any CPU.Build.0 = Release|Any CPU 24 | {F69BCA7A-0501-4050-9DFC-DE5CB448ECAB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 25 | {F69BCA7A-0501-4050-9DFC-DE5CB448ECAB}.Debug|Any CPU.Build.0 = Debug|Any CPU 26 | {F69BCA7A-0501-4050-9DFC-DE5CB448ECAB}.Release|Any CPU.ActiveCfg = Release|Any CPU 27 | {F69BCA7A-0501-4050-9DFC-DE5CB448ECAB}.Release|Any CPU.Build.0 = Release|Any CPU 28 | {28E405A5-D140-4BD5-9D6C-4EBEAE02FE79}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 29 | {28E405A5-D140-4BD5-9D6C-4EBEAE02FE79}.Debug|Any CPU.Build.0 = Debug|Any CPU 30 | {28E405A5-D140-4BD5-9D6C-4EBEAE02FE79}.Release|Any CPU.ActiveCfg = Release|Any CPU 31 | {28E405A5-D140-4BD5-9D6C-4EBEAE02FE79}.Release|Any CPU.Build.0 = Release|Any CPU 32 | {1BD44FE4-AD66-48F0-8695-BFD7EFF7E5A8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 33 | {1BD44FE4-AD66-48F0-8695-BFD7EFF7E5A8}.Debug|Any CPU.Build.0 = Debug|Any CPU 34 | {1BD44FE4-AD66-48F0-8695-BFD7EFF7E5A8}.Release|Any CPU.ActiveCfg = Release|Any CPU 35 | {1BD44FE4-AD66-48F0-8695-BFD7EFF7E5A8}.Release|Any CPU.Build.0 = Release|Any CPU 36 | EndGlobalSection 37 | GlobalSection(SolutionProperties) = preSolution 38 | HideSolutionNode = FALSE 39 | EndGlobalSection 40 | GlobalSection(ExtensibilityGlobals) = postSolution 41 | SolutionGuid = {35603F90-428E-4082-8D49-F4B38B97F1D1} 42 | EndGlobalSection 43 | EndGlobal 44 | -------------------------------------------------------------------------------- /FastPersistentDictionary/FastPersistentDictionary.cs: -------------------------------------------------------------------------------- 1 | using System.Collections; 2 | using System.Collections.Immutable; 3 | using System.Runtime.CompilerServices; 4 | using System.Runtime.Serialization; 5 | using System.Timers; 6 | using FastPersistentDictionary.Internals.Accessor; 7 | using GroBuf; 8 | using GroBuf.DataMembersExtracters; 9 | using FastPersistentDictionary.Internals; 10 | using FastPersistentDictionary.Internals.Compression; 11 | using Timer = System.Timers.Timer; 12 | 13 | [assembly: InternalsVisibleTo("FastPersistentDictionary.Tests")] 14 | namespace FastPersistentDictionary 15 | { 16 | /// 17 | /// Written by James Grice 2-8-2023 / 8-09-2024 18 | /// 19 | /// The `FastPersistentDictionary` class is a implementation of a persistent dictionary in C#. 20 | /// It is designed to provide a dictionary-like data structure that persists data to disk. 21 | /// It is designed to be used like a regular dictionary. 22 | /// The class provides a flexible and efficient data structure for storing and querying large amounts of data while 23 | /// minimizing disk space usage. 24 | /// 25 | /// 26 | /// 27 | [Serializable] 28 | 29 | public sealed class FastPersistentDictionary : IEnumerable>, 30 | ISerializable, IDisposable 31 | { 32 | internal Dictionary> DictionarySerializedLookup = new Dictionary>(); 33 | 34 | private readonly DictionaryIo _dictionaryIo; 35 | private readonly DictionaryMathOperations _dictionaryMathOperations; 36 | private readonly DictionaryObject _dictionaryObject; 37 | private readonly DictionaryOperations _dictionaryOperations; 38 | private readonly DictionaryQuery _dictionaryQuery; 39 | 40 | 41 | public bool CanCompact = false; 42 | public FileStream FileStream; 43 | public FileStream FileStream_Keys; 44 | 45 | private bool _disposed; 46 | private readonly object _lockObj = new object(); 47 | private readonly Serializer _serializer; 48 | private readonly ICompressionHandler _compressionHandler; 49 | 50 | private Timer _updateTimer; 51 | internal readonly IDictionaryAccessor DictionaryAccessor; 52 | internal readonly string FileLocation; 53 | internal readonly string FastPersistentDictionaryVersion = "1.0.0.7"; 54 | internal readonly bool CrashRecovery; 55 | 56 | internal bool DeleteDBOnClose; 57 | internal readonly bool UseCompression; 58 | internal float PercentageChangeBeforeCompact; 59 | 60 | //Main create or load a FastPersistentDictionary into memory 61 | public FastPersistentDictionary( 62 | string path = "", 63 | bool crashRecovery = false, 64 | int updateRate = 1000, 65 | bool deleteDBOnClose = false, 66 | float percentageChangeBeforeCompact = 50, 67 | int fileStreamBufferSize = 8196, 68 | string importSavedFastPersistentDictionary = "", 69 | IEqualityComparer? equalityComparer = null, 70 | bool useCompression = false) 71 | { 72 | CrashRecovery = crashRecovery; 73 | DeleteDBOnClose = deleteDBOnClose; 74 | 75 | if (path == "") 76 | path = Path.GetTempFileName(); 77 | 78 | FileLocation = path; 79 | 80 | _serializer = new Serializer(new AllPropertiesExtractor(), options: GroBufOptions.WriteEmptyObjects); 81 | Directory.CreateDirectory(Path.GetDirectoryName(FileLocation)); 82 | 83 | UseCompression = useCompression; 84 | 85 | 86 | var fileOptions = FileOptions.RandomAccess; 87 | 88 | FileStream = new FileStream(FileLocation, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite | FileShare.Delete, fileStreamBufferSize, fileOptions); //| FileOptions.DeleteOnClose); 89 | 90 | _updateTimer = new Timer(updateRate); 91 | _updateTimer.Elapsed += UpdateTimerElapsedEventHandler; 92 | 93 | if (UseCompression) 94 | _compressionHandler = new SerializerCompress(_serializer); 95 | else 96 | _compressionHandler = new SerializerUnCompressed(_serializer); 97 | 98 | 99 | if (CrashRecovery) 100 | { 101 | string location = Path.GetDirectoryName(FileLocation); 102 | string Fname = Path.GetFileNameWithoutExtension(FileLocation); 103 | string PathKeys = Path.Combine(location, Fname) + ".prec"; 104 | 105 | FileStream_Keys = new FileStream(PathKeys, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite, 8196, FileOptions.SequentialScan); 106 | DictionaryAccessor = new DictionaryAccessor_RecoverMode(this, _updateTimer, _compressionHandler, _lockObj, FileStream, FileStream_Keys); 107 | } 108 | else 109 | DictionaryAccessor = new DictionaryAccessor(this, _updateTimer, _compressionHandler, _lockObj, FileStream); 110 | 111 | 112 | 113 | PercentageChangeBeforeCompact = percentageChangeBeforeCompact; 114 | 115 | _dictionaryQuery = new DictionaryQuery(this, DictionaryAccessor, _updateTimer, _compressionHandler, PercentageChangeBeforeCompact, _lockObj, FileStream); 116 | _dictionaryIo = new DictionaryIo(this, _compressionHandler, _dictionaryQuery, _lockObj, FileStream, _serializer, UseCompression); 117 | _dictionaryObject = new DictionaryObject(DictionaryAccessor, _lockObj); 118 | _dictionaryOperations = new DictionaryOperations(this, DictionaryAccessor, _updateTimer, _compressionHandler, _lockObj, FileStream); 119 | _dictionaryMathOperations = new DictionaryMathOperations(this, DictionaryAccessor, _dictionaryOperations, _compressionHandler, _lockObj, FileStream); 120 | 121 | DictionaryAccessor.Comparer = equalityComparer ?? EqualityComparer.Default; 122 | 123 | CheckDBValidStartup(crashRecovery); 124 | 125 | 126 | if (importSavedFastPersistentDictionary != "" && File.Exists(importSavedFastPersistentDictionary)) 127 | _dictionaryIo.LoadDictionary(importSavedFastPersistentDictionary); 128 | } 129 | 130 | public void Dispose() 131 | { 132 | //todo fill this out 133 | if (_disposed) 134 | return; 135 | 136 | 137 | if (DeleteDBOnClose) 138 | { 139 | 140 | _updateTimer.Stop(); 141 | _updateTimer = null; 142 | FileStream.Dispose(); 143 | FileStream.Close(); 144 | FileStream = null; 145 | 146 | FileStream_Keys.Dispose(); 147 | FileStream_Keys.Close(); 148 | FileStream_Keys = null; 149 | 150 | if (File.Exists(FileLocation)) 151 | File.Delete(FileLocation); 152 | 153 | var fpath = Path.GetDirectoryName(FileLocation); 154 | var fname = Path.GetFileNameWithoutExtension(FileLocation); 155 | var recFullPath = Path.Combine(fpath, fname + ".prec"); 156 | 157 | if (File.Exists(recFullPath)) 158 | File.Delete(recFullPath); 159 | } 160 | else 161 | { 162 | //Write out save 163 | // 164 | var tempPath = Path.GetTempFileName(); 165 | SaveDictionary(tempPath, comment: "Time Saved: " + DateTime.Now.ToString()); 166 | 167 | _updateTimer.Stop(); 168 | _updateTimer = null; 169 | FileStream.Dispose(); 170 | FileStream.Close(); 171 | FileStream = null; 172 | 173 | if (File.Exists(FileLocation)) 174 | File.Delete(FileLocation); 175 | 176 | if (File.Exists(tempPath)) 177 | File.Move(tempPath, FileLocation); 178 | 179 | var fpath = Path.GetDirectoryName(FileLocation); 180 | var fname = Path.GetFileNameWithoutExtension(FileLocation); 181 | var recFullPath = Path.Combine(fpath, fname + ".prec"); 182 | 183 | if (File.Exists(recFullPath)) 184 | File.Delete(recFullPath); 185 | } 186 | 187 | 188 | 189 | try 190 | { 191 | GC.Collect(); 192 | GC.WaitForPendingFinalizers(); 193 | 194 | } 195 | catch 196 | { 197 | // ignored 198 | } 199 | 200 | _disposed = true; 201 | GC.SuppressFinalize(this); 202 | } 203 | 204 | ~FastPersistentDictionary() 205 | { 206 | Dispose(); 207 | } 208 | 209 | private void CheckDBValidStartup(bool crashRecovery) 210 | { 211 | bool successLoad = false; 212 | if (File.Exists(FileLocation) && FileStream.Length != 0) 213 | { 214 | //todo: 215 | //1. TRY LOAD 216 | try 217 | { 218 | var loadedDict = LoadDictionary(FileLocation); 219 | successLoad = true; 220 | } 221 | catch 222 | { 223 | successLoad = false; 224 | } 225 | 226 | if (successLoad == false && crashRecovery) 227 | { 228 | var fpath = Path.GetDirectoryName(FileLocation); 229 | var fname = Path.GetFileNameWithoutExtension(FileLocation); 230 | var recFullPath = Path.Combine(fpath, fname + ".prec"); 231 | if (File.Exists(recFullPath)) 232 | { 233 | DictionarySerializedLookup.Clear(); 234 | var buffer8 = new byte[8]; 235 | var buffer4 = new byte[4]; 236 | var buffer2 = new byte[2]; 237 | 238 | while (FileStream_Keys.Position < FileStream_Keys.Length) 239 | { 240 | // Read keySerializedLen 241 | FileStream_Keys.Read(buffer2, 0, 2); 242 | var keySerializedLen = BitConverter.ToUInt16(buffer2, 0); 243 | 244 | // Read keySerialized 245 | var keySerialized = new byte[keySerializedLen]; 246 | FileStream_Keys.Read(keySerialized, 0, keySerializedLen); 247 | 248 | // Read valuePos 249 | FileStream_Keys.Read(buffer8, 0, 8); 250 | var valuePos = BitConverter.ToInt64(buffer8, 0); 251 | 252 | // Read valueLen 253 | FileStream_Keys.Read(buffer4, 0, 4); 254 | var valueLen = BitConverter.ToInt32(buffer4, 0); 255 | 256 | var keyDeserialized = _compressionHandler.DeserializeKey(keySerialized); 257 | 258 | if (DictionarySerializedLookup.ContainsKey(keyDeserialized)) 259 | { 260 | if (valuePos == -1 || valueLen == -1) 261 | { 262 | DictionarySerializedLookup.Remove(keyDeserialized); 263 | } 264 | else 265 | { 266 | DictionarySerializedLookup[keyDeserialized] = new KeyValuePair(valuePos, valueLen); 267 | } 268 | } 269 | else 270 | { 271 | if (valuePos != -1 || valueLen != -1) 272 | DictionarySerializedLookup.Add(keyDeserialized, new KeyValuePair(valuePos, valueLen)); 273 | } 274 | } 275 | 276 | //WRITE CLEAN RECOVER FILE 277 | FileStream_Keys.SetLength(0); 278 | List<(TKey, KeyValuePair)> corruptedValues = new List<(TKey, KeyValuePair)>(); 279 | foreach (var item in DictionarySerializedLookup) 280 | { 281 | try 282 | { 283 | var value = this[item.Key]; 284 | } 285 | catch 286 | { 287 | corruptedValues.Add((item.Key, item.Value)); 288 | continue; 289 | } 290 | 291 | byte[] KeySerialized = _compressionHandler.SerializeNotCompressed(item.Key); 292 | byte[] len = BitConverter.GetBytes((UInt16)KeySerialized.Length); 293 | byte[] keyPosSerialized = new byte[12]; 294 | Buffer.BlockCopy(BitConverter.GetBytes(item.Value.Key), 0, keyPosSerialized, 0, 8); 295 | Buffer.BlockCopy(BitConverter.GetBytes(item.Value.Value), 0, keyPosSerialized, 8, 4); 296 | 297 | // Calculate total length 298 | int totalLength = len.Length + KeySerialized.Length + keyPosSerialized.Length; 299 | byte[] allData = new byte[totalLength]; 300 | 301 | // Copy all data into single array 302 | Buffer.BlockCopy(len, 0, allData, 0, len.Length); 303 | Buffer.BlockCopy(KeySerialized, 0, allData, len.Length, KeySerialized.Length); 304 | Buffer.BlockCopy(keyPosSerialized, 0, allData, len.Length + KeySerialized.Length, keyPosSerialized.Length); 305 | 306 | // Write to file 307 | FileStream_Keys.Write(allData); 308 | FileStream_Keys.Flush(); 309 | } 310 | 311 | //REMOVE ANY CORRUPTED VALUES 312 | foreach (var item in corruptedValues) 313 | { 314 | DictionarySerializedLookup.Remove(item.Item1); 315 | } 316 | } 317 | else 318 | { 319 | //CANNOT RECOVER 320 | FileStream.SetLength(0); 321 | } 322 | } 323 | else if (successLoad == false) 324 | { 325 | FileStream.SetLength(0); 326 | } 327 | } 328 | } 329 | 330 | 331 | public TValue this[TKey key] 332 | { 333 | get => DictionaryAccessor[key]; 334 | set => DictionaryAccessor[key] = value; 335 | } 336 | public IEqualityComparer Comparer => DictionaryAccessor.Comparer; 337 | public int Count => DictionaryAccessor.Count; 338 | public IEnumerable Keys => DictionaryAccessor.Keys; 339 | public IEnumerable Values => DictionaryAccessor.Values; 340 | 341 | public void Add(TKey key, TValue value) => DictionaryAccessor.Add(key, value); 342 | 343 | public void GetObjectData(SerializationInfo info, StreamingContext context) => DictionaryAccessor.GetObjectData(info, context); 344 | 345 | public IEnumerator> GetEnumerator() => DictionaryAccessor.GetEnumerator(); 346 | 347 | public Type GetKeyType() => DictionaryAccessor.GetKeyType(); 348 | 349 | public Type GetValueType() => DictionaryAccessor.GetValueType(); 350 | 351 | public bool Contains(KeyValuePair item) => DictionaryAccessor.Contains(item); 352 | 353 | public bool ContainsKey(TKey key) => DictionaryAccessor.ContainsKey(key); 354 | 355 | public bool ContainsValue(TValue value) => DictionaryAccessor.ContainsValue(value); 356 | 357 | public void Clear() => DictionaryAccessor.Clear(); 358 | 359 | public TValue GetOrCreate(TKey key, Func creator) => DictionaryAccessor.GetOrCreate(key, creator); 360 | 361 | public void UpdateValue(TKey key, Func updater) => DictionaryAccessor.UpdateValue(key, updater); 362 | 363 | public void UpdateDatabaseTick(bool forceCompact = false) => _dictionaryQuery.UpdateDatabaseTick(forceCompact); 364 | 365 | public TValue TryGetValueOrNull(TKey key) => DictionaryAccessor.TryGetValueOrNull(key); 366 | 367 | public bool HasKey(TKey key) => DictionaryAccessor.HasKey(key); 368 | 369 | public bool HasValue(TValue key) => DictionaryAccessor.HasValue(key); 370 | 371 | public void AddOrUpdate(TKey key, TValue value) => DictionaryAccessor.AddOrUpdate(key, value); 372 | 373 | public TValue Pop(TKey key) => DictionaryAccessor.Pop(key); 374 | 375 | public void Push(TKey key, TValue value) => DictionaryAccessor.Push(key, value); 376 | 377 | public KeyValuePair GetRandom() => DictionaryAccessor.GetRandom(); 378 | 379 | public void ForEach(Action action) => DictionaryAccessor.ForEach(action); 380 | 381 | public void RemoveAll(Func predicate) => DictionaryAccessor.RemoveAll(predicate); 382 | 383 | public TValue[] GetBulk(TKey[] keys) => DictionaryAccessor.GetBulk(keys); 384 | 385 | public List GetBulk(List keys) => DictionaryAccessor.GetBulk(keys); 386 | 387 | public void ClearWithValue(TValue val) => DictionaryAccessor.ClearWithValue(val); 388 | 389 | public void RenameKey(TKey oldKey, TKey newKey) => DictionaryAccessor.RenameKey(oldKey, newKey); 390 | 391 | public void Switch(TKey key1, TKey key2) => DictionaryAccessor.Switch(key1, key2); 392 | 393 | public KeyValuePair[] GetBulk(Func predicate) => DictionaryAccessor.GetBulk(predicate); 394 | 395 | public bool Remove(TKey key) => DictionaryAccessor.Remove(key); 396 | 397 | public bool TryGetValue(TKey key, out TValue value) => DictionaryAccessor.TryGetValue(key, out value); 398 | 399 | public bool TryGetValue(TKey key, out TValue value, Type customType) => DictionaryAccessor.TryGetValue(key, out value, customType); 400 | 401 | public TValue Get(TKey key) => DictionaryAccessor.Get(key); 402 | 403 | public long GetDatabaseSizeBytes() => _dictionaryIo.GetDatabaseSizeBytes(); 404 | 405 | IEnumerator IEnumerable.GetEnumerator() => _dictionaryObject.GetEnumerator(); 406 | 407 | public override bool Equals(object obj) => _dictionaryObject.Equals(obj); 408 | 409 | public override int GetHashCode() => _dictionaryObject.GetHashCode(); 410 | 411 | public IEnumerable Select(Func, TResult> selector) => 412 | _dictionaryQuery.Select(selector); 413 | 414 | public override string ToString() => _dictionaryIo.ToString(); 415 | 416 | public DictionaryStructs.DictionarySaveHeader SaveDictionary(string savePath, string name = "", string comment = "") => _dictionaryIo.SaveDictionary(savePath, name, comment); 417 | 418 | public DictionaryStructs.DictionarySaveHeader SaveDictionary(string savePath, DictionaryStructs.DictionarySaveHeader header) => _dictionaryIo.SaveDictionary(savePath, header); 419 | 420 | public DictionaryStructs.DictionarySaveHeader LoadDictionary(string loadPath) => _dictionaryIo.LoadDictionary(loadPath); 421 | 422 | public byte[] PackBuffer() => _dictionaryIo.PackBuffer(); 423 | 424 | public TValue Min() => _dictionaryMathOperations.Min(); 425 | 426 | public TValue Sum() => _dictionaryMathOperations.Sum(); 427 | 428 | public bool TryGetMax(out TValue max) => _dictionaryMathOperations.TryGetMax(out max); 429 | 430 | public bool TryGetMin(out TValue min) => _dictionaryMathOperations.TryGetMin(out min); 431 | 432 | public TValue Max() => _dictionaryMathOperations.Max(); 433 | 434 | public TKey MaxKey() => _dictionaryMathOperations.MaxKey(); 435 | 436 | public TKey MinKey() => _dictionaryMathOperations.MinKey(); 437 | 438 | public double Average(Func, double> selector) => _dictionaryMathOperations.Average(selector); 439 | 440 | public IOrderedEnumerable> OrderByDescending( 441 | Func, object> selector) => _dictionaryOperations.OrderByDescending(selector); 442 | 443 | public IOrderedEnumerable> 444 | OrderBy(Func, object> selector) => _dictionaryOperations.OrderBy(selector); 445 | 446 | public List> Reverse() => _dictionaryOperations.Reverse(); 447 | 448 | public KeyValuePair[] ToArray() => _dictionaryOperations.ToArray(); 449 | 450 | public List> ToList() => _dictionaryOperations.ToList(); 451 | 452 | public IEnumerable> Take(int count) => _dictionaryOperations.Take(count); 453 | 454 | public IEnumerable> TakeWhile(Func, bool> predicate) => 455 | _dictionaryOperations.TakeWhile(predicate); 456 | 457 | public IEnumerable> Skip(int number) => _dictionaryOperations.Skip(number); 458 | 459 | public IEnumerable> SkipWhile(Func, bool> predicate) => 460 | _dictionaryOperations.SkipWhile(predicate); 461 | 462 | public IEnumerable> DistinctByKey() => _dictionaryOperations.DistinctByKey(); 463 | 464 | public IEnumerable> Except(FastPersistentDictionary otherDict) => 465 | _dictionaryOperations.Except(otherDict); 466 | 467 | public void Concat(FastPersistentDictionary other) => _dictionaryOperations.Concat(other); 468 | 469 | public FastPersistentDictionary Intersect( 470 | FastPersistentDictionary other) => _dictionaryOperations.Intersect(other); 471 | 472 | public FastPersistentDictionary Union(FastPersistentDictionary other) => 473 | _dictionaryOperations.Union(other); 474 | 475 | public FastPersistentDictionary> Zip( 476 | FastPersistentDictionary second) => _dictionaryOperations.Zip(second); 477 | 478 | public ImmutableList> ToImmutableList() => _dictionaryOperations.ToImmutableList(); 479 | 480 | public ImmutableArray> ToImmutableArray() => 481 | _dictionaryOperations.ToImmutableArray(); 482 | 483 | public ImmutableDictionary ToImmutableDictionary() => 484 | _dictionaryOperations.ToImmutableDictionary(); 485 | 486 | public ImmutableHashSet> ToImmutableHashSet() => 487 | _dictionaryOperations.ToImmutableHashSet(); 488 | 489 | public SortedSet> ToSortedSet() => _dictionaryOperations.ToSortedSet(); 490 | 491 | public SortedDictionary ToSortedDictionary() => _dictionaryOperations.ToSortedDictionary(); 492 | 493 | public IQueryable> AsQueryable() => _dictionaryOperations.AsQueryable(); 494 | 495 | public ParallelQuery> AsParallel() => _dictionaryOperations.AsParallel(); 496 | 497 | public void Merge(Dictionary other, bool overwriteExistingKeys = true) => _dictionaryOperations.Merge(other, overwriteExistingKeys); 498 | 499 | public void Merge(FastPersistentDictionary other, bool overwriteExistingKeys = true) => _dictionaryOperations.Merge(other, overwriteExistingKeys); 500 | 501 | public void Join(Dictionary other, bool overwriteExistingKeys = false) => _dictionaryOperations.Join(other, overwriteExistingKeys); 502 | 503 | public void Join(FastPersistentDictionary other, bool overwriteExistingKeys = false) => _dictionaryOperations.Join(other, overwriteExistingKeys); 504 | 505 | public FastPersistentDictionary Copy(string path) => _dictionaryOperations.Copy(path); 506 | 507 | public FastPersistentDictionary Invert() => _dictionaryOperations.Invert(); 508 | 509 | public IEnumerable> TakeLast(int number) => _dictionaryOperations.TakeLast(number); 510 | 511 | public IEnumerable>> InBatchesOf(int batchSize) => _dictionaryOperations.InBatchesOf(batchSize); 512 | 513 | public void Shuffle() => _dictionaryOperations.Shuffle(); 514 | 515 | public IEnumerable> SkipLast(int number) => _dictionaryOperations.SkipLast(number); 516 | 517 | public KeyValuePair Single(Func, bool> predicate = null) => 518 | _dictionaryQuery.Single(predicate); 519 | 520 | private void UpdateTimerElapsedEventHandler(object sender, ElapsedEventArgs e) => _dictionaryQuery.UpdateDatabaseTick(); 521 | 522 | public void Peek(Action> action) => _dictionaryQuery.Peek(action); 523 | 524 | public void CompactDatabaseFile() => _dictionaryQuery.CompactDatabaseFile(); 525 | 526 | public KeyValuePair SingleOrDefault(Func, bool> predicate = null) => 527 | _dictionaryQuery.SingleOrDefault(predicate); 528 | 529 | public bool Any() => _dictionaryQuery.Any(); 530 | 531 | public bool Any(Func predicate) => _dictionaryQuery.Any(predicate); 532 | 533 | public IEnumerable> Where(Func predicate) => 534 | _dictionaryQuery.Where(predicate); 535 | 536 | public KeyValuePair? FirstOrDefault() => _dictionaryQuery.FirstOrDefault(); 537 | 538 | public KeyValuePair Last() => _dictionaryQuery.Last(); 539 | 540 | public KeyValuePair? LastOrDefault() => _dictionaryQuery.LastOrDefault(); 541 | 542 | public KeyValuePair First() => _dictionaryQuery.First(); 543 | 544 | public KeyValuePair ElementAt(int index) => _dictionaryQuery.ElementAt(index); 545 | 546 | public bool All(Func, bool> predicate) => _dictionaryQuery.All(predicate); 547 | 548 | public TKey FindKeyByValue(TValue value) => _dictionaryQuery.FindKeyByValue(value); 549 | 550 | public List FindAllKeysByValue(TValue value) => _dictionaryQuery.FindAllKeysByValue(value); 551 | 552 | public FastPersistentDictionary GetSubset(Func predicate) => _dictionaryQuery.GetSubset(predicate); 553 | 554 | public FastPersistentDictionary GetSubset(List keys) => _dictionaryQuery.GetSubset(keys); 555 | 556 | public int CountValues(TValue value) => _dictionaryQuery.CountValues(value); 557 | 558 | public FastPersistentDictionary FilteredWhere(Func predicate) => 559 | _dictionaryQuery.FilteredWhere(predicate); 560 | 561 | public IDictionary Index() => _dictionaryQuery.Index(); 562 | 563 | public (IEnumerable>, IEnumerable>) Partition(Func, bool> predicate) => 564 | _dictionaryQuery.Partition(predicate); 565 | 566 | public IEnumerable Flatten(IEnumerable> enumerable) 567 | { 568 | foreach (var innerEnumerable in enumerable) 569 | foreach (var item in innerEnumerable) 570 | yield return item; 571 | } 572 | } 573 | } -------------------------------------------------------------------------------- /FastPersistentDictionary/FastPersistentDictionary.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net7.0 5 | enable 6 | enable 7 | False 8 | True 9 | Fast Persistent Dictionary 10 | James Grice 2024 11 | https://github.com/jgric2/Fast-Persistent-Dictionary 12 | Limintel Solutions 13 | A fast KV pair dictionary storing values on disk. 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /FastPersistentDictionary/Internals/Accessor/DictionaryAccessor.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics.CodeAnalysis; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.Serialization; 4 | using FastPersistentDictionary.Internals.Compression; 5 | 6 | namespace FastPersistentDictionary.Internals.Accessor 7 | { 8 | [SuppressMessage("ReSharper", "MustUseReturnValue")] 9 | public sealed class DictionaryAccessor : IDictionaryAccessor 10 | { 11 | private readonly ICompressionHandler _compressionHandler; 12 | 13 | private readonly object _lockObj; 14 | private readonly FastPersistentDictionary _fastPersistentDictionary; 15 | private readonly System.Timers.Timer _updateTimer; 16 | internal FileStream FileStream; 17 | 18 | public DictionaryAccessor( 19 | FastPersistentDictionary dict, 20 | System.Timers.Timer updateTimer, 21 | ICompressionHandler compressionHandler, 22 | object lockObj, 23 | FileStream fileStream) 24 | { 25 | _lockObj = lockObj; 26 | FileStream = fileStream; 27 | _fastPersistentDictionary = dict; 28 | _compressionHandler = compressionHandler; 29 | _updateTimer = updateTimer; 30 | } 31 | 32 | public IEqualityComparer Comparer { get; set; } 33 | 34 | public int Count => _fastPersistentDictionary.DictionarySerializedLookup.Count; 35 | 36 | public IEnumerable Keys => _fastPersistentDictionary.DictionarySerializedLookup.Keys; 37 | 38 | public IEnumerable Values 39 | { 40 | get 41 | { 42 | lock (_lockObj) 43 | { 44 | foreach (var kvp in _fastPersistentDictionary.DictionarySerializedLookup.Values) 45 | { 46 | var data = new byte[kvp.Value]; 47 | FileStream.Seek(kvp.Key, SeekOrigin.Begin); 48 | FileStream.Read(data, 0, kvp.Value); 49 | 50 | yield return _compressionHandler.Deserialize(data); 51 | } 52 | } 53 | } 54 | } 55 | 56 | public TValue this[TKey key] 57 | { 58 | get => Get(key); 59 | set => Set(key, value); 60 | } 61 | 62 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 63 | public void Add(TKey key, TValue value) 64 | { 65 | lock (_lockObj) 66 | { 67 | if (_fastPersistentDictionary.ContainsKey(key)) 68 | throw new ArgumentException("Key already exists"); 69 | 70 | 71 | var data = _compressionHandler.Serialize(value); 72 | 73 | if (FileStream.Position != FileStream.Length) 74 | FileStream.Seek(0, SeekOrigin.End); 75 | 76 | _fastPersistentDictionary.DictionarySerializedLookup[key] = new KeyValuePair(FileStream.Position, data.Length); 77 | FileStream.Write(data, 0, data.Length); 78 | } 79 | _updateTimer.Stop(); 80 | _updateTimer.Start(); 81 | } 82 | 83 | 84 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 85 | public void Set(TKey key, TValue value) 86 | { 87 | var data = _compressionHandler.Serialize(value); 88 | lock (_lockObj) 89 | { 90 | FileStream.Seek(0, SeekOrigin.End); 91 | var kvpLookup = new KeyValuePair(FileStream.Position, data.Length); 92 | _fastPersistentDictionary.DictionarySerializedLookup[key] = kvpLookup; 93 | FileStream.Write(data, 0, data.Length); 94 | } 95 | 96 | _updateTimer.Stop(); 97 | _updateTimer.Start(); 98 | } 99 | 100 | /// 101 | /// `AddOrUpdate(Key key, Value value)`: If the key exists in the dictionary, update the value. If it doesn't exist, add the new key-value pair 102 | /// 103 | /// 104 | /// 105 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 106 | public void AddOrUpdate(TKey key, TValue value) 107 | { 108 | lock (_lockObj) 109 | if (_fastPersistentDictionary.DictionarySerializedLookup.ContainsKey(key)) 110 | _fastPersistentDictionary.DictionarySerializedLookup.Remove(key); 111 | 112 | var data = _compressionHandler.Serialize(value); 113 | lock (_lockObj) 114 | { 115 | FileStream.Seek(0, SeekOrigin.End); 116 | var kvpLookup = new KeyValuePair(FileStream.Position, data.Length); 117 | _fastPersistentDictionary.DictionarySerializedLookup[key] = kvpLookup; 118 | FileStream.Write(data, 0, data.Length); 119 | } 120 | 121 | _updateTimer.Stop(); 122 | _updateTimer.Start(); 123 | } 124 | 125 | /// 126 | /// UpdateValue(Key key, Func updater): Update the value for a key using a function that transforms the old value into a new one. 127 | /// 128 | /// 129 | /// 130 | /// 131 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 132 | public void UpdateValue(TKey key, Func updater) 133 | { 134 | KeyValuePair lookupCoordinates; 135 | lock (_lockObj) 136 | { 137 | if (_fastPersistentDictionary.DictionarySerializedLookup.TryGetValue(key, out lookupCoordinates) == false) 138 | throw new KeyNotFoundException($"The key {key} was not found in the dictionary."); 139 | 140 | var data = new byte[lookupCoordinates.Value]; 141 | 142 | FileStream.Seek(lookupCoordinates.Key, SeekOrigin.Begin); 143 | FileStream.Read(data, 0, lookupCoordinates.Value); 144 | 145 | var valueDeserialized = updater(_compressionHandler.Deserialize(data)); 146 | var newData = _compressionHandler.Serialize(valueDeserialized); 147 | 148 | if (newData.Length <= data.Length) 149 | { 150 | FileStream.Seek(lookupCoordinates.Key, SeekOrigin.Begin); 151 | FileStream.Write(newData, 0, newData.Length); 152 | _fastPersistentDictionary.DictionarySerializedLookup[key] = new KeyValuePair(lookupCoordinates.Key, newData.Length); 153 | } 154 | else 155 | { 156 | FileStream.Seek(0, SeekOrigin.End); 157 | var kvpLookup = new KeyValuePair(FileStream.Position, newData.Length); 158 | _fastPersistentDictionary.DictionarySerializedLookup[key] = kvpLookup; 159 | FileStream.Write(newData, 0, newData.Length); 160 | 161 | _updateTimer.Stop(); 162 | _updateTimer.Start(); 163 | } 164 | } 165 | } 166 | 167 | /// 168 | /// `TryGetValueOrNull(Key key)`: A variation on `TryGetValue` that simply returns null if the key is not in the dictionary, rather than requiring a separate 'out' variable. 169 | /// 170 | /// 171 | /// 172 | public TValue TryGetValueOrNull(TKey key) 173 | { 174 | return SerializedLookup_TryGetValue(key, out var deserializedData) == false ? default : deserializedData; 175 | } 176 | 177 | /// 178 | /// `RemoveAll(Func predicate)`: Removes all the key-value pairs that satisfy the provided condition. 179 | /// 180 | /// 181 | public void RemoveAll(Func predicate) 182 | { 183 | var keysToRemove = new List(); 184 | 185 | lock (_lockObj) 186 | { 187 | foreach (var kvp in _fastPersistentDictionary.DictionarySerializedLookup.Keys) 188 | if (predicate(kvp, Get(kvp))) 189 | keysToRemove.Add(kvp); 190 | 191 | foreach (var key in keysToRemove) 192 | _fastPersistentDictionary.DictionarySerializedLookup.Remove(key); 193 | 194 | _updateTimer.Stop(); 195 | _updateTimer.Start(); 196 | } 197 | } 198 | 199 | /// 200 | /// `GetOrCreate(Key key, Func creator)`: Method to get a value if it exists; otherwise, use the provided function to create a new value and insert it into the dictionary. 201 | /// 202 | /// 203 | /// 204 | /// 205 | public TValue GetOrCreate(TKey key, Func creator) 206 | { 207 | if (ContainsKey(key)) 208 | return Get(key); 209 | 210 | var newValue = creator(); 211 | Add(key, newValue); 212 | 213 | _updateTimer.Stop(); 214 | _updateTimer.Start(); 215 | 216 | return newValue; 217 | } 218 | 219 | public IEnumerator> GetEnumerator() 220 | { 221 | lock (_lockObj) 222 | foreach (var key in _fastPersistentDictionary.DictionarySerializedLookup.Keys) 223 | yield return new KeyValuePair(key, this[key]); 224 | } 225 | 226 | public void GetObjectData(SerializationInfo info, StreamingContext context) 227 | { 228 | var index = 0; 229 | lock (_lockObj) 230 | { 231 | var entries = new KeyValuePair[_fastPersistentDictionary.DictionarySerializedLookup.Count]; 232 | 233 | foreach (var keyValuePair in _fastPersistentDictionary.DictionarySerializedLookup.Keys) 234 | entries[index++] = new KeyValuePair(keyValuePair, Get(keyValuePair)); 235 | 236 | info.AddValue("entries", entries); 237 | } 238 | } 239 | 240 | public Type GetKeyType() 241 | { 242 | return typeof(TKey); 243 | } 244 | 245 | public Type GetValueType() 246 | { 247 | return typeof(TValue); 248 | } 249 | 250 | public void Switch(TKey key1, TKey key2) 251 | { 252 | if (TryGetValue(key1, out var value1) == false || TryGetValue(key2, out var value2) == false) 253 | throw new ArgumentException("One or both keys are not present in the dictionary."); 254 | 255 | Remove(key1); 256 | Remove(key2); 257 | 258 | Add(key1, value2); 259 | Add(key2, value1); 260 | } 261 | 262 | public bool Contains(KeyValuePair item) 263 | { 264 | lock (_lockObj) 265 | return _fastPersistentDictionary.DictionarySerializedLookup.ContainsKey(item.Key) && Get(item.Key).Equals(item.Value); 266 | } 267 | 268 | ///// 269 | ///// ClearWithValue(Value val) : Clears the dictionary but sets all keys to be associated with a specified default value. 270 | ///// 271 | ///// 272 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 273 | public void ClearWithValue(TValue val) 274 | { 275 | var defaultValue = _compressionHandler.Serialize(val); 276 | lock (_lockObj) 277 | { 278 | FileStream.SetLength(0); 279 | foreach (var key in _fastPersistentDictionary.DictionarySerializedLookup.Keys) 280 | { 281 | var location = new KeyValuePair(FileStream.Position, defaultValue.Length); 282 | FileStream.Write(defaultValue, 0, defaultValue.Length); 283 | 284 | _fastPersistentDictionary.DictionarySerializedLookup[key] = location; 285 | } 286 | } 287 | } 288 | 289 | /// 290 | /// RenameKey(Key oldKey, Key newKey): Changes the name of a key while keeping its associated value. 291 | /// 292 | /// 293 | /// 294 | /// 295 | /// 296 | public void RenameKey(TKey oldKey, TKey newKey) 297 | { 298 | if (ContainsKey(oldKey) == false) 299 | throw new KeyNotFoundException($"The key {oldKey} was not found in the dictionary."); 300 | 301 | var value = Get(oldKey); 302 | 303 | Remove(oldKey); 304 | Set(newKey, value); 305 | } 306 | 307 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 308 | public bool ContainsKey(TKey key) 309 | { 310 | lock (_lockObj) 311 | return _fastPersistentDictionary.DictionarySerializedLookup.ContainsKey(key); 312 | } 313 | 314 | public bool HasKey(TKey key) => ContainsKey(key); 315 | 316 | public bool HasValue(TValue key) => ContainsValue(key); 317 | 318 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 319 | public bool ContainsValue(TValue value) 320 | { 321 | lock (_lockObj) 322 | foreach (var key in _fastPersistentDictionary.DictionarySerializedLookup.Values) 323 | { 324 | var val = SeekReadAndDeserialize(key.Key, key.Value); 325 | if (val == null && value == null || val.Equals(value)) 326 | return true; 327 | } 328 | 329 | return false; 330 | } 331 | 332 | /// 333 | /// `Pop(Key key)`: Removes the item with the specified key and returns its value. 334 | /// 335 | /// 336 | /// 337 | public TValue Pop(TKey key) 338 | { 339 | var value = Get(key); 340 | Remove(key); 341 | return value; 342 | } 343 | 344 | public void Push(TKey key, TValue value) 345 | { 346 | Add(key, value); 347 | } 348 | 349 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 350 | public void Clear() 351 | { 352 | lock (_lockObj) 353 | { 354 | _fastPersistentDictionary.DictionarySerializedLookup.Clear(); 355 | FileStream.SetLength(0); 356 | //FileStream.Seek(0, SeekOrigin.Begin); 357 | } 358 | } 359 | 360 | 361 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 362 | private void ClearCacheAndLookup() 363 | { 364 | _fastPersistentDictionary.DictionarySerializedLookup.Clear(); 365 | } 366 | 367 | /// 368 | /// `GetRandom()`: Returns a random key-value pair from the dictionary. 369 | /// 370 | /// 371 | /// 372 | public KeyValuePair GetRandom() 373 | { 374 | var allKeys = Keys.Count(); 375 | 376 | if (allKeys == 0) 377 | throw new InvalidOperationException("Dictionary is empty"); 378 | 379 | var rng = new Random(); 380 | 381 | var randomKey = Keys.ElementAt(rng.Next(allKeys)); 382 | 383 | return new KeyValuePair(randomKey, Get(randomKey)); 384 | } 385 | 386 | 387 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 388 | public bool Remove(TKey key) 389 | { 390 | lock (_lockObj) 391 | { 392 | var removedFromLookup = _fastPersistentDictionary.DictionarySerializedLookup.Remove(key); 393 | if (removedFromLookup == false) 394 | return false; 395 | } 396 | 397 | _updateTimer.Stop(); 398 | _updateTimer.Start(); 399 | 400 | return true; 401 | } 402 | 403 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 404 | public bool TryGetValue(TKey key, out TValue value) 405 | { 406 | if (SerializedLookup_TryGetValue(key, out value)) 407 | return true; 408 | 409 | value = default; 410 | return false; 411 | } 412 | 413 | 414 | public bool TryGetValue(TKey key, out TValue value, Type custType) 415 | { 416 | if (SerializedLookup_TryGetValueCustomType(key, out value, custType)) 417 | return true; 418 | 419 | value = default; 420 | return false; 421 | } 422 | 423 | 424 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 425 | public TValue Get(TKey key) 426 | { 427 | if (SerializedLookup_TryGetValue(key, out var value) == false) 428 | throw new KeyNotFoundException($"The key {key} was not found in the dictionary."); 429 | 430 | return value; 431 | } 432 | 433 | 434 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 435 | public TValue GetReturnDefaultIfNotFound(TKey key) 436 | { 437 | lock (_lockObj) 438 | { 439 | if (SerializedLookup_TryGetValue(key, out var value) == false) 440 | return default; 441 | 442 | return value; 443 | } 444 | } 445 | 446 | 447 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 448 | public TValue[] GetBulk(TKey[] keys) 449 | { 450 | var returnValues = new TValue[keys.Length]; 451 | lock (_lockObj) 452 | { 453 | for (var index = 0; index < keys.Length; index++) 454 | { 455 | if (SerializedLookup_TryGetValue(keys[index], out returnValues[index]) == false) 456 | throw new KeyNotFoundException($"The key {keys[index]} was not found in the dictionary."); 457 | } 458 | } 459 | 460 | return returnValues; 461 | } 462 | 463 | 464 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 465 | public List GetBulk(List keys) => GetBulk(keys.ToArray()).ToList(); 466 | 467 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 468 | public KeyValuePair[] GetBulk(Func predicate) 469 | { 470 | var returnValuesList = new List>(); 471 | lock (_lockObj) 472 | { 473 | foreach (var key in Keys) 474 | { 475 | if (SerializedLookup_TryGetValue(key, out var value) == false) 476 | throw new KeyNotFoundException($"The key {key} was not found in the dictionary."); 477 | 478 | if (predicate(key, value)) 479 | returnValuesList.Add(new KeyValuePair(key, value)); 480 | } 481 | } 482 | 483 | return returnValuesList.ToArray(); 484 | } 485 | 486 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 487 | public void ForEach(Action action) 488 | { 489 | lock (_lockObj) 490 | { 491 | foreach (var item in _fastPersistentDictionary.DictionarySerializedLookup) 492 | action(item.Key, SeekReadAndDeserialize(item.Value.Key, item.Value.Value)); 493 | } 494 | } 495 | 496 | 497 | 498 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 499 | private bool SerializedLookup_TryGetValue(TKey key, out TValue value) 500 | { 501 | lock (_lockObj) 502 | { 503 | if (_fastPersistentDictionary.DictionarySerializedLookup.TryGetValue(key, out var lookupCoordinates)) 504 | { 505 | var data = new byte[lookupCoordinates.Value]; 506 | 507 | if (FileStream.Position != lookupCoordinates.Key) 508 | FileStream.Seek(lookupCoordinates.Key, SeekOrigin.Begin); 509 | 510 | FileStream.Read(data, 0, lookupCoordinates.Value); 511 | 512 | value = _compressionHandler.Deserialize(data); 513 | _updateTimer.Stop(); 514 | _updateTimer.Start(); 515 | 516 | return true; 517 | } 518 | } 519 | 520 | 521 | value = default; 522 | return false; 523 | } 524 | 525 | 526 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 527 | private bool SerializedLookup_TryGetValueCustomType(TKey key, out TValue value, Type custType) 528 | { 529 | lock (_lockObj) 530 | { 531 | if (_fastPersistentDictionary.DictionarySerializedLookup.TryGetValue(key, out var lookupCoordinates)) 532 | { 533 | var data = new byte[lookupCoordinates.Value]; 534 | 535 | FileStream.Seek(lookupCoordinates.Key, SeekOrigin.Begin); 536 | FileStream.Read(data, 0, lookupCoordinates.Value); 537 | 538 | _updateTimer.Stop(); 539 | _updateTimer.Start(); 540 | value = _compressionHandler.Deserialize(custType, data); 541 | return true; 542 | } 543 | } 544 | 545 | value = default; 546 | return false; 547 | } 548 | 549 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 550 | private TValue SeekReadAndDeserialize(long index, int length) 551 | { 552 | var data = new byte[length]; 553 | FileStream.Seek(index, SeekOrigin.Begin); 554 | FileStream.Read(data, 0, length); 555 | 556 | _updateTimer.Stop(); 557 | _updateTimer.Start(); 558 | 559 | return _compressionHandler.Deserialize(data); 560 | } 561 | 562 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 563 | private byte[] SeekRead(long index, int length) 564 | { 565 | var data = new byte[length]; 566 | FileStream.Seek(index, SeekOrigin.Begin); 567 | FileStream.Read(data, 0, length); 568 | 569 | return data; 570 | } 571 | } 572 | } -------------------------------------------------------------------------------- /FastPersistentDictionary/Internals/Accessor/DictionaryAccessor_RecoverMode.cs: -------------------------------------------------------------------------------- 1 | using FastPersistentDictionary.Internals.Compression; 2 | using FastPersistentDictionary; 3 | using System.Runtime.CompilerServices; 4 | using System.Runtime.Serialization; 5 | 6 | namespace FastPersistentDictionary.Internals.Accessor 7 | { 8 | public sealed class DictionaryAccessor_RecoverMode : IDictionaryAccessor 9 | { 10 | private readonly ICompressionHandler _compressionHandler; 11 | 12 | private readonly object _lockObj; 13 | private readonly FastPersistentDictionary _persistentDictionaryPro; 14 | private readonly System.Timers.Timer _updateTimer; 15 | internal FileStream FileStream; 16 | 17 | internal FileStream FileStream_Keys; 18 | 19 | public DictionaryAccessor_RecoverMode( 20 | FastPersistentDictionary dict, 21 | System.Timers.Timer updateTimer, 22 | ICompressionHandler compressionHandler, 23 | object lockObj, 24 | FileStream fileStream, 25 | FileStream fileStream_Keys) 26 | { 27 | _lockObj = lockObj; 28 | FileStream = fileStream; 29 | FileStream_Keys = fileStream_Keys; 30 | _persistentDictionaryPro = dict; 31 | _compressionHandler = compressionHandler; 32 | _updateTimer = updateTimer; 33 | } 34 | 35 | public IEqualityComparer Comparer { get; set; } 36 | 37 | public int Count => _persistentDictionaryPro.DictionarySerializedLookup.Count; 38 | 39 | public IEnumerable Keys => _persistentDictionaryPro.DictionarySerializedLookup.Keys; 40 | 41 | public IEnumerable Values 42 | { 43 | get 44 | { 45 | lock (_lockObj) 46 | { 47 | foreach (var kvp in _persistentDictionaryPro.DictionarySerializedLookup.Values) 48 | { 49 | var data = new byte[kvp.Value]; 50 | FileStream.Seek(kvp.Key, SeekOrigin.Begin); 51 | FileStream.Read(data, 0, kvp.Value); 52 | 53 | yield return _compressionHandler.Deserialize(data); 54 | } 55 | } 56 | } 57 | } 58 | 59 | public TValue this[TKey key] 60 | { 61 | get => Get(key); 62 | set => Set(key, value); 63 | } 64 | 65 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 66 | public void Add(TKey key, TValue value) 67 | { 68 | lock (_lockObj) 69 | { 70 | if (_persistentDictionaryPro.ContainsKey(key)) 71 | throw new ArgumentException("Key already exists"); 72 | 73 | 74 | var data = _compressionHandler.Serialize(value); 75 | 76 | FileStream.Seek(0, SeekOrigin.End); 77 | var kvpPair = new KeyValuePair(FileStream.Position, data.Length); 78 | _persistentDictionaryPro.DictionarySerializedLookup[key] = kvpPair; 79 | 80 | 81 | byte[] KeySerialized = _compressionHandler.SerializeNotCompressed(key); 82 | byte[] len = BitConverter.GetBytes((UInt16)KeySerialized.Length); 83 | byte[] keyPosSerialized = new byte[12]; 84 | Buffer.BlockCopy(BitConverter.GetBytes(kvpPair.Key), 0, keyPosSerialized, 0, 8); 85 | Buffer.BlockCopy(BitConverter.GetBytes(kvpPair.Value), 0, keyPosSerialized, 8, 4); 86 | 87 | // Calculate total length 88 | int totalLength = len.Length + KeySerialized.Length + keyPosSerialized.Length; 89 | byte[] allData = new byte[totalLength]; 90 | 91 | // Copy all data into single array 92 | Buffer.BlockCopy(len, 0, allData, 0, len.Length); 93 | Buffer.BlockCopy(KeySerialized, 0, allData, len.Length, KeySerialized.Length); 94 | Buffer.BlockCopy(keyPosSerialized, 0, allData, len.Length + KeySerialized.Length, keyPosSerialized.Length); 95 | 96 | // Write to file 97 | FileStream_Keys.Write(allData); 98 | FileStream_Keys.Flush(); 99 | 100 | FileStream.Write(data, 0, data.Length); 101 | FileStream.Flush(); 102 | } 103 | _updateTimer.Start(); 104 | } 105 | 106 | 107 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 108 | public void Set(TKey key, TValue value) 109 | { 110 | var data = _compressionHandler.Serialize(value); 111 | lock (_lockObj) 112 | { 113 | FileStream.Seek(0, SeekOrigin.End); 114 | var kvpLookup = new KeyValuePair(FileStream.Position, data.Length); 115 | _persistentDictionaryPro.DictionarySerializedLookup[key] = kvpLookup; 116 | 117 | byte[] KeySerialized = _compressionHandler.SerializeNotCompressed(key); 118 | byte[] len = BitConverter.GetBytes((UInt16)KeySerialized.Length); 119 | byte[] keyPosSerialized = new byte[12]; 120 | Buffer.BlockCopy(BitConverter.GetBytes(kvpLookup.Key), 0, keyPosSerialized, 0, 8); 121 | Buffer.BlockCopy(BitConverter.GetBytes(kvpLookup.Value), 0, keyPosSerialized, 8, 4); 122 | 123 | // Calculate total length 124 | int totalLength = len.Length + KeySerialized.Length + keyPosSerialized.Length; 125 | byte[] allData = new byte[totalLength]; 126 | 127 | // Copy all data into single array 128 | Buffer.BlockCopy(len, 0, allData, 0, len.Length); 129 | Buffer.BlockCopy(KeySerialized, 0, allData, len.Length, KeySerialized.Length); 130 | Buffer.BlockCopy(keyPosSerialized, 0, allData, len.Length + KeySerialized.Length, keyPosSerialized.Length); 131 | 132 | // Write to file 133 | FileStream_Keys.Write(allData); 134 | FileStream_Keys.Flush(); 135 | 136 | FileStream.Write(data, 0, data.Length); 137 | FileStream.Flush(); 138 | } 139 | 140 | _updateTimer.Start(); 141 | } 142 | 143 | /// 144 | /// `AddOrUpdate(Key key, Value value)`: If the key exists in the dictionary, update the value. If it doesn't exist, add the new key-value pair 145 | /// 146 | /// 147 | /// 148 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 149 | public void AddOrUpdate(TKey key, TValue value) 150 | { 151 | lock (_lockObj) 152 | if (_persistentDictionaryPro.DictionarySerializedLookup.ContainsKey(key)) 153 | _persistentDictionaryPro.DictionarySerializedLookup.Remove(key); 154 | 155 | var data = _compressionHandler.Serialize(value); 156 | lock (_lockObj) 157 | { 158 | FileStream.Seek(0, SeekOrigin.End); 159 | var kvpLookup = new KeyValuePair(FileStream.Position, data.Length); 160 | _persistentDictionaryPro.DictionarySerializedLookup[key] = kvpLookup; 161 | 162 | byte[] KeySerialized = _compressionHandler.SerializeNotCompressed(key); 163 | byte[] len = BitConverter.GetBytes((UInt16)KeySerialized.Length); 164 | byte[] keyPosSerialized = new byte[12]; 165 | Buffer.BlockCopy(BitConverter.GetBytes(kvpLookup.Key), 0, keyPosSerialized, 0, 8); 166 | Buffer.BlockCopy(BitConverter.GetBytes(kvpLookup.Value), 0, keyPosSerialized, 8, 4); 167 | 168 | // Calculate total length 169 | int totalLength = len.Length + KeySerialized.Length + keyPosSerialized.Length; 170 | byte[] allData = new byte[totalLength]; 171 | 172 | // Copy all data into single array 173 | Buffer.BlockCopy(len, 0, allData, 0, len.Length); 174 | Buffer.BlockCopy(KeySerialized, 0, allData, len.Length, KeySerialized.Length); 175 | Buffer.BlockCopy(keyPosSerialized, 0, allData, len.Length + KeySerialized.Length, keyPosSerialized.Length); 176 | 177 | // Write to file 178 | FileStream_Keys.Write(allData); 179 | FileStream_Keys.Flush(); 180 | 181 | FileStream.Write(data, 0, data.Length); 182 | FileStream.Flush(); 183 | } 184 | 185 | _updateTimer.Start(); 186 | } 187 | 188 | /// 189 | /// UpdateValue(Key key, Func updater): Update the value for a key using a function that transforms the old value into a new one. 190 | /// 191 | /// 192 | /// 193 | /// 194 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 195 | public void UpdateValue(TKey key, Func updater) 196 | { 197 | KeyValuePair lookupCoordinates; 198 | lock (_lockObj) 199 | { 200 | if (_persistentDictionaryPro.DictionarySerializedLookup.TryGetValue(key, out lookupCoordinates) == false) 201 | throw new KeyNotFoundException($"The key {key} was not found in the dictionary."); 202 | 203 | var data = new byte[lookupCoordinates.Value]; 204 | 205 | FileStream.Seek(lookupCoordinates.Key, SeekOrigin.Begin); 206 | FileStream.Read(data, 0, lookupCoordinates.Value); 207 | 208 | var valueDeserialized = updater(_compressionHandler.Deserialize(data)); 209 | var newData = _compressionHandler.Serialize(valueDeserialized); 210 | 211 | if (newData.Length <= data.Length) 212 | { 213 | 214 | var kvpLookup = new KeyValuePair(lookupCoordinates.Key, newData.Length); 215 | _persistentDictionaryPro.DictionarySerializedLookup[key] = kvpLookup; 216 | 217 | byte[] KeySerialized = _compressionHandler.SerializeNotCompressed(key); 218 | byte[] len = BitConverter.GetBytes((UInt16)KeySerialized.Length); 219 | byte[] keyPosSerialized = new byte[12]; 220 | Buffer.BlockCopy(BitConverter.GetBytes(kvpLookup.Key), 0, keyPosSerialized, 0, 8); 221 | Buffer.BlockCopy(BitConverter.GetBytes(kvpLookup.Value), 0, keyPosSerialized, 8, 4); 222 | 223 | // Calculate total length 224 | int totalLength = len.Length + KeySerialized.Length + keyPosSerialized.Length; 225 | byte[] allData = new byte[totalLength]; 226 | 227 | // Copy all data into single array 228 | Buffer.BlockCopy(len, 0, allData, 0, len.Length); 229 | Buffer.BlockCopy(KeySerialized, 0, allData, len.Length, KeySerialized.Length); 230 | Buffer.BlockCopy(keyPosSerialized, 0, allData, len.Length + KeySerialized.Length, keyPosSerialized.Length); 231 | 232 | // Write to file 233 | FileStream_Keys.Write(allData); 234 | FileStream_Keys.Flush(); 235 | 236 | FileStream.Seek(lookupCoordinates.Key, SeekOrigin.Begin); 237 | FileStream.Write(newData, 0, newData.Length); 238 | FileStream.Flush(); 239 | } 240 | else 241 | { 242 | 243 | var kvpLookup = new KeyValuePair(FileStream.Position, newData.Length); 244 | _persistentDictionaryPro.DictionarySerializedLookup[key] = kvpLookup; 245 | 246 | byte[] KeySerialized = _compressionHandler.SerializeNotCompressed(key); 247 | byte[] len = BitConverter.GetBytes((UInt16)KeySerialized.Length); 248 | byte[] keyPosSerialized = new byte[12]; 249 | Buffer.BlockCopy(BitConverter.GetBytes(kvpLookup.Key), 0, keyPosSerialized, 0, 8); 250 | Buffer.BlockCopy(BitConverter.GetBytes(kvpLookup.Value), 0, keyPosSerialized, 8, 4); 251 | 252 | // Calculate total length 253 | int totalLength = len.Length + KeySerialized.Length + keyPosSerialized.Length; 254 | byte[] allData = new byte[totalLength]; 255 | 256 | // Copy all data into single array 257 | Buffer.BlockCopy(len, 0, allData, 0, len.Length); 258 | Buffer.BlockCopy(KeySerialized, 0, allData, len.Length, KeySerialized.Length); 259 | Buffer.BlockCopy(keyPosSerialized, 0, allData, len.Length + KeySerialized.Length, keyPosSerialized.Length); 260 | 261 | // Write to file 262 | FileStream_Keys.Write(allData); 263 | FileStream_Keys.Flush(); 264 | 265 | FileStream.Seek(0, SeekOrigin.End); 266 | FileStream.Write(newData, 0, newData.Length); 267 | FileStream.Flush(); 268 | 269 | _updateTimer.Start(); 270 | } 271 | } 272 | } 273 | 274 | /// 275 | /// `TryGetValueOrNull(Key key)`: A variation on `TryGetValue` that simply returns null if the key is not in the dictionary, rather than requiring a separate 'out' variable. 276 | /// 277 | /// 278 | /// 279 | public TValue TryGetValueOrNull(TKey key) 280 | { 281 | return SerializedLookup_TryGetValue(key, out var deserializedData) == false ? default : deserializedData; 282 | } 283 | 284 | /// 285 | /// `RemoveAll(Func predicate)`: Removes all the key-value pairs that satisfy the provided condition. 286 | /// 287 | /// 288 | public void RemoveAll(Func predicate) 289 | { 290 | var keysToRemove = new List(); 291 | 292 | lock (_lockObj) 293 | { 294 | foreach (var kvp in _persistentDictionaryPro.DictionarySerializedLookup.Keys) 295 | if (predicate(kvp, Get(kvp))) 296 | keysToRemove.Add(kvp); 297 | 298 | foreach (var key in keysToRemove) 299 | _persistentDictionaryPro.DictionarySerializedLookup.Remove(key); 300 | 301 | _updateTimer.Start(); 302 | } 303 | } 304 | 305 | /// 306 | /// `GetOrCreate(Key key, Func creator)`: Method to get a value if it exists; otherwise, use the provided function to create a new value and insert it into the dictionary. 307 | /// 308 | /// 309 | /// 310 | /// 311 | public TValue GetOrCreate(TKey key, Func creator) 312 | { 313 | if (ContainsKey(key)) 314 | return Get(key); 315 | 316 | var newValue = creator(); 317 | Add(key, newValue); 318 | 319 | _updateTimer.Start(); 320 | 321 | return newValue; 322 | } 323 | 324 | public IEnumerator> GetEnumerator() 325 | { 326 | lock (_lockObj) 327 | foreach (var key in _persistentDictionaryPro.DictionarySerializedLookup.Keys) 328 | yield return new KeyValuePair(key, this[key]); 329 | } 330 | 331 | public void GetObjectData(SerializationInfo info, StreamingContext context) 332 | { 333 | var index = 0; 334 | lock (_lockObj) 335 | { 336 | var entries = new KeyValuePair[_persistentDictionaryPro.DictionarySerializedLookup.Count]; 337 | 338 | foreach (var keyValuePair in _persistentDictionaryPro.DictionarySerializedLookup.Keys) 339 | entries[index++] = new KeyValuePair(keyValuePair, Get(keyValuePair)); 340 | 341 | info.AddValue("entries", entries); 342 | } 343 | } 344 | 345 | public Type GetKeyType() 346 | { 347 | return typeof(TKey); 348 | } 349 | 350 | public Type GetValueType() 351 | { 352 | return typeof(TValue); 353 | } 354 | 355 | public void Switch(TKey key1, TKey key2) 356 | { 357 | if (TryGetValue(key1, out var value1) == false || TryGetValue(key2, out var value2) == false) 358 | throw new ArgumentException("One or both keys are not present in the dictionary."); 359 | 360 | Remove(key1); 361 | Remove(key2); 362 | 363 | Add(key1, value2); 364 | Add(key2, value1); 365 | } 366 | 367 | public bool Contains(KeyValuePair item) 368 | { 369 | lock (_lockObj) 370 | return _persistentDictionaryPro.DictionarySerializedLookup.ContainsKey(item.Key) && Get(item.Key).Equals(item.Value); 371 | } 372 | 373 | ///// 374 | ///// ClearWithValue(Value val) : Clears the dictionary but sets all keys to be associated with a specified default value. 375 | ///// 376 | ///// 377 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 378 | public void ClearWithValue(TValue val) 379 | { 380 | var defaultValue = _compressionHandler.Serialize(val); 381 | lock (_lockObj) 382 | { 383 | FileStream.SetLength(0); 384 | FileStream_Keys.SetLength(0); 385 | FileStream_Keys.Flush(); 386 | FileStream.Flush(); 387 | foreach (var key in _persistentDictionaryPro.DictionarySerializedLookup.Keys) 388 | { 389 | var location = new KeyValuePair(FileStream.Position, defaultValue.Length); 390 | FileStream.Write(defaultValue, 0, defaultValue.Length); 391 | 392 | _persistentDictionaryPro.DictionarySerializedLookup[key] = location; 393 | 394 | byte[] KeySerialized = _compressionHandler.SerializeNotCompressed(key); 395 | byte[] len = BitConverter.GetBytes((UInt16)KeySerialized.Length); 396 | 397 | byte[] keyPosSerialized = new byte[12]; 398 | Buffer.BlockCopy(BitConverter.GetBytes(location.Key), 0, keyPosSerialized, 0, 8); 399 | Buffer.BlockCopy(BitConverter.GetBytes(location.Value), 0, keyPosSerialized, 8, 4); 400 | 401 | // Calculate total length 402 | int totalLength = len.Length + KeySerialized.Length + keyPosSerialized.Length; 403 | byte[] allData = new byte[totalLength]; 404 | 405 | // Copy all data into single array 406 | Buffer.BlockCopy(len, 0, allData, 0, len.Length); 407 | Buffer.BlockCopy(KeySerialized, 0, allData, len.Length, KeySerialized.Length); 408 | Buffer.BlockCopy(keyPosSerialized, 0, allData, len.Length + KeySerialized.Length, keyPosSerialized.Length); 409 | 410 | // Write to file 411 | FileStream_Keys.Write(allData); 412 | 413 | } 414 | FileStream_Keys.Flush(); 415 | FileStream.Flush(); 416 | } 417 | } 418 | 419 | /// 420 | /// RenameKey(Key oldKey, Key newKey): Changes the name of a key while keeping its associated value. 421 | /// 422 | /// 423 | /// 424 | /// 425 | /// 426 | public void RenameKey(TKey oldKey, TKey newKey) 427 | { 428 | if (ContainsKey(oldKey) == false) 429 | throw new KeyNotFoundException($"The key {oldKey} was not found in the dictionary."); 430 | 431 | var value = Get(oldKey); 432 | 433 | Remove(oldKey); 434 | Set(newKey, value); 435 | } 436 | 437 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 438 | public bool ContainsKey(TKey key) 439 | { 440 | lock (_lockObj) 441 | return _persistentDictionaryPro.DictionarySerializedLookup.ContainsKey(key); 442 | } 443 | 444 | public bool HasKey(TKey key) => ContainsKey(key); 445 | 446 | public bool HasValue(TValue key) => ContainsValue(key); 447 | 448 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 449 | public bool ContainsValue(TValue value) 450 | { 451 | lock (_lockObj) 452 | foreach (var key in _persistentDictionaryPro.DictionarySerializedLookup.Values) 453 | { 454 | var val = SeekReadAndDeserialize(key.Key, key.Value); 455 | if (val == null && value == null || val.Equals(value)) 456 | return true; 457 | } 458 | 459 | return false; 460 | } 461 | 462 | /// 463 | /// `Pop(Key key)`: Removes the item with the specified key and returns its value. 464 | /// 465 | /// 466 | /// 467 | public TValue Pop(TKey key) 468 | { 469 | var value = Get(key); 470 | Remove(key); 471 | return value; 472 | } 473 | 474 | public void Push(TKey key, TValue value) 475 | { 476 | Add(key, value); 477 | } 478 | 479 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 480 | public void Clear() 481 | { 482 | lock (_lockObj) 483 | { 484 | _persistentDictionaryPro.DictionarySerializedLookup.Clear(); 485 | FileStream.SetLength(0); 486 | FileStream_Keys.SetLength(0); 487 | FileStream_Keys.Flush(); 488 | FileStream.Flush(); 489 | 490 | //FileStream.Seek(0, SeekOrigin.Begin); 491 | } 492 | } 493 | 494 | 495 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 496 | private void ClearCacheAndLookup() 497 | { 498 | _persistentDictionaryPro.DictionarySerializedLookup.Clear(); 499 | } 500 | 501 | /// 502 | /// `GetRandom()`: Returns a random key-value pair from the dictionary. 503 | /// 504 | /// 505 | /// 506 | public KeyValuePair GetRandom() 507 | { 508 | var allKeys = Keys.Count(); 509 | 510 | if (allKeys == 0) 511 | throw new InvalidOperationException("Dictionary is empty"); 512 | 513 | var rng = new Random(); 514 | 515 | var randomKey = Keys.ElementAt(rng.Next(allKeys)); 516 | 517 | return new KeyValuePair(randomKey, Get(randomKey)); 518 | } 519 | 520 | 521 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 522 | public bool Remove(TKey key) 523 | { 524 | lock (_lockObj) 525 | { 526 | //var tempKey = _persistentDictionaryPro.DictionarySerializedLookup[key]; 527 | 528 | 529 | var removedFromLookup = _persistentDictionaryPro.DictionarySerializedLookup.Remove(key); 530 | 531 | 532 | 533 | 534 | if (removedFromLookup == false) 535 | return false; 536 | else 537 | { 538 | byte[] KeySerialized = _compressionHandler.SerializeNotCompressed(key); 539 | byte[] len = BitConverter.GetBytes((UInt16)KeySerialized.Length); 540 | 541 | byte[] keyPosSerialized = new byte[12]; 542 | var tempKey = new KeyValuePair(-1,-1); 543 | 544 | Buffer.BlockCopy(BitConverter.GetBytes(tempKey.Key), 0, keyPosSerialized, 0, 8); 545 | Buffer.BlockCopy(BitConverter.GetBytes(tempKey.Value), 0, keyPosSerialized, 8, 4); 546 | 547 | // Calculate total length 548 | int totalLength = len.Length + KeySerialized.Length + keyPosSerialized.Length; 549 | byte[] allData = new byte[totalLength]; 550 | 551 | // Copy all data into single array 552 | Buffer.BlockCopy(len, 0, allData, 0, len.Length); 553 | Buffer.BlockCopy(KeySerialized, 0, allData, len.Length, KeySerialized.Length); 554 | Buffer.BlockCopy(keyPosSerialized, 0, allData, len.Length + KeySerialized.Length, keyPosSerialized.Length); 555 | 556 | // Write to file 557 | FileStream_Keys.Write(allData); 558 | FileStream_Keys.Flush(); 559 | FileStream.Flush(); 560 | } 561 | } 562 | 563 | _updateTimer.Start(); 564 | 565 | return true; 566 | } 567 | 568 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 569 | public bool TryGetValue(TKey key, out TValue value) 570 | { 571 | if (SerializedLookup_TryGetValue(key, out value)) 572 | return true; 573 | 574 | value = default; 575 | return false; 576 | } 577 | 578 | 579 | public bool TryGetValue(TKey key, out TValue value, Type custType) 580 | { 581 | if (SerializedLookup_TryGetValueCustomType(key, out value, custType)) 582 | return true; 583 | 584 | value = default; 585 | return false; 586 | } 587 | 588 | 589 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 590 | public TValue Get(TKey key) 591 | { 592 | if (SerializedLookup_TryGetValue(key, out var value) == false) 593 | throw new KeyNotFoundException($"The key {key} was not found in the dictionary."); 594 | 595 | return value; 596 | } 597 | 598 | 599 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 600 | public TValue GetReturnDefaultIfNotFound(TKey key) 601 | { 602 | lock (_lockObj) 603 | { 604 | if (SerializedLookup_TryGetValue(key, out var value) == false) 605 | return default; 606 | 607 | return value; 608 | } 609 | } 610 | 611 | 612 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 613 | public TValue[] GetBulk(TKey[] keys) 614 | { 615 | var returnValues = new TValue[keys.Length]; 616 | lock (_lockObj) 617 | { 618 | for (var index = 0; index < keys.Length; index++) 619 | { 620 | if (SerializedLookup_TryGetValue(keys[index], out returnValues[index]) == false) 621 | throw new KeyNotFoundException($"The key {keys[index]} was not found in the dictionary."); 622 | } 623 | } 624 | 625 | return returnValues; 626 | } 627 | 628 | 629 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 630 | public List GetBulk(List keys) => GetBulk(keys.ToArray()).ToList(); 631 | 632 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 633 | public KeyValuePair[] GetBulk(Func predicate) 634 | { 635 | var returnValuesList = new List>(); 636 | lock (_lockObj) 637 | { 638 | foreach (var key in Keys) 639 | { 640 | if (SerializedLookup_TryGetValue(key, out var value) == false) 641 | throw new KeyNotFoundException($"The key {key} was not found in the dictionary."); 642 | 643 | if (predicate(key, value)) 644 | returnValuesList.Add(new KeyValuePair(key, value)); 645 | } 646 | } 647 | 648 | return returnValuesList.ToArray(); 649 | } 650 | 651 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 652 | public void ForEach(Action action) 653 | { 654 | lock (_lockObj) 655 | { 656 | foreach (var item in _persistentDictionaryPro.DictionarySerializedLookup) 657 | action(item.Key, SeekReadAndDeserialize(item.Value.Key, item.Value.Value)); 658 | } 659 | } 660 | 661 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 662 | private bool SerializedLookup_TryGetValue(TKey key, out TValue value) 663 | { 664 | lock (_lockObj) 665 | { 666 | if (_persistentDictionaryPro.DictionarySerializedLookup.TryGetValue(key, out var lookupCoordinates)) 667 | { 668 | var data = new byte[lookupCoordinates.Value]; 669 | 670 | FileStream.Seek(lookupCoordinates.Key, SeekOrigin.Begin); 671 | FileStream.Read(data, 0, lookupCoordinates.Value); 672 | 673 | value = _compressionHandler.Deserialize(data); 674 | return true; 675 | } 676 | } 677 | 678 | value = default; 679 | return false; 680 | } 681 | 682 | 683 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 684 | private bool SerializedLookup_TryGetValueCustomType(TKey key, out TValue value, Type custType) 685 | { 686 | lock (_lockObj) 687 | { 688 | if (_persistentDictionaryPro.DictionarySerializedLookup.TryGetValue(key, out var lookupCoordinates)) 689 | { 690 | var data = new byte[lookupCoordinates.Value]; 691 | 692 | FileStream.Seek(lookupCoordinates.Key, SeekOrigin.Begin); 693 | FileStream.Read(data, 0, lookupCoordinates.Value); 694 | 695 | value = _compressionHandler.Deserialize(custType, data); 696 | return true; 697 | } 698 | } 699 | 700 | value = default; 701 | return false; 702 | } 703 | 704 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 705 | private TValue SeekReadAndDeserialize(long index, int length) 706 | { 707 | var data = new byte[length]; 708 | FileStream.Seek(index, SeekOrigin.Begin); 709 | FileStream.Read(data, 0, length); 710 | 711 | return _compressionHandler.Deserialize(data); 712 | } 713 | 714 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 715 | private byte[] SeekRead(long index, int length) 716 | { 717 | var data = new byte[length]; 718 | FileStream.Seek(index, SeekOrigin.Begin); 719 | FileStream.Read(data, 0, length); 720 | 721 | return data; 722 | } 723 | 724 | } 725 | } 726 | -------------------------------------------------------------------------------- /FastPersistentDictionary/Internals/Accessor/IDictionaryAccessor.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Runtime.Serialization; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | 8 | namespace FastPersistentDictionary.Internals.Accessor 9 | { 10 | public interface IDictionaryAccessor 11 | { 12 | IEqualityComparer Comparer { get; set; } 13 | int Count { get; } 14 | IEnumerable Keys { get; } 15 | IEnumerable Values { get; } 16 | TValue this[TKey key] { get; set; } 17 | 18 | void Add(TKey key, TValue value); 19 | void Set(TKey key, TValue value); 20 | void AddOrUpdate(TKey key, TValue value); 21 | void UpdateValue(TKey key, Func updater); 22 | TValue TryGetValueOrNull(TKey key); 23 | void RemoveAll(Func predicate); 24 | TValue GetOrCreate(TKey key, Func creator); 25 | IEnumerator> GetEnumerator(); 26 | void GetObjectData(SerializationInfo info, StreamingContext context); 27 | Type GetKeyType(); 28 | Type GetValueType(); 29 | void Switch(TKey key1, TKey key2); 30 | bool Contains(KeyValuePair item); 31 | void ClearWithValue(TValue val); 32 | void RenameKey(TKey oldKey, TKey newKey); 33 | bool ContainsKey(TKey key); 34 | bool HasKey(TKey key); 35 | bool HasValue(TValue key); 36 | bool ContainsValue(TValue value); 37 | TValue Pop(TKey key); 38 | void Push(TKey key, TValue value); 39 | void Clear(); 40 | KeyValuePair GetRandom(); 41 | bool Remove(TKey key); 42 | bool TryGetValue(TKey key, out TValue value); 43 | bool TryGetValue(TKey key, out TValue value, Type custType); 44 | TValue Get(TKey key); 45 | TValue GetReturnDefaultIfNotFound(TKey key); 46 | TValue[] GetBulk(TKey[] keys); 47 | List GetBulk(List keys); 48 | KeyValuePair[] GetBulk(Func predicate); 49 | void ForEach(Action action); 50 | 51 | //public IEqualityComparer Comparer; 52 | 53 | 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /FastPersistentDictionary/Internals/Compression/ICompressionHandler.cs: -------------------------------------------------------------------------------- 1 | using GroBuf; 2 | 3 | namespace FastPersistentDictionary.Internals.Compression 4 | { 5 | public interface ICompressionHandler 6 | { 7 | //todo: clean this up, compression added as an afterthought 8 | public byte[] Serialize(TValue data); 9 | public byte[] Serialize(T obj); 10 | 11 | public TValue Deserialize(byte[] compressedData); 12 | public TValue Deserialize(byte[] compressedData); 13 | 14 | public TValue Deserialize(Type type, byte[] data); 15 | 16 | 17 | //Generic 18 | public byte[] SerializeNotCompressed(T obj); 19 | public TValue DeserializeNotCompressed(byte[] compressedData); 20 | public TValue DeserializeNotCompressed(byte[] compressedData); 21 | 22 | 23 | public TKey DeserializeKey(byte[] compressedData); 24 | 25 | 26 | 27 | 28 | public byte[] SerializeCompressed(T obj); 29 | public TValue DeserializeCompressed(byte[] compressedData); 30 | public TValue DeserializeCompressed(byte[] compressedData); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /FastPersistentDictionary/Internals/Compression/SerializerCompress.cs: -------------------------------------------------------------------------------- 1 | using GroBuf; 2 | 3 | namespace FastPersistentDictionary.Internals.Compression 4 | { 5 | public class SerializerCompress : ICompressionHandler 6 | { 7 | public readonly Serializer serializer; 8 | public byte[] Serialize(TValue data) 9 | { 10 | byte[] byteData = serializer.Serialize(data); 11 | 12 | return K4os.Compression.LZ4.LZ4Pickler.Pickle(byteData); 13 | } 14 | 15 | public byte[] Serialize(TValue obj) 16 | { 17 | byte[] byteData = serializer.Serialize(obj); 18 | 19 | return K4os.Compression.LZ4.LZ4Pickler.Pickle(byteData); 20 | } 21 | 22 | public TValue Deserialize(byte[] compressedData) 23 | { 24 | compressedData = K4os.Compression.LZ4.LZ4Pickler.Unpickle(compressedData); 25 | 26 | return serializer.Deserialize(compressedData); 27 | } 28 | 29 | public TValue Deserialize(byte[] compressedData) 30 | { 31 | compressedData = K4os.Compression.LZ4.LZ4Pickler.Unpickle(compressedData); 32 | 33 | return serializer.Deserialize(compressedData); 34 | } 35 | 36 | public TValue Deserialize(Type type, byte[] data) 37 | { 38 | 39 | return (TValue)serializer.Deserialize(type,data); 40 | } 41 | 42 | public byte[] SerializeNotCompressed(T obj) 43 | { 44 | return serializer.Serialize(obj); 45 | } 46 | 47 | public TValue DeserializeNotCompressed(byte[] compressedData) 48 | { 49 | return serializer.Deserialize(compressedData); 50 | } 51 | 52 | 53 | public TKey DeserializeKey(byte[] compressedData) 54 | { 55 | return serializer.Deserialize(compressedData); 56 | } 57 | 58 | public byte[] SerializeCompressed(T obj) 59 | { 60 | byte[] data = serializer.Serialize(obj); 61 | return K4os.Compression.LZ4.LZ4Pickler.Pickle(data); 62 | } 63 | 64 | public TValue DeserializeCompressed(byte[] compressedData) 65 | { 66 | compressedData = K4os.Compression.LZ4.LZ4Pickler.Unpickle(compressedData); 67 | return serializer.Deserialize(compressedData); 68 | } 69 | 70 | public TValue DeserializeCompressed(byte[] compressedData) 71 | { 72 | compressedData = K4os.Compression.LZ4.LZ4Pickler.Unpickle(compressedData); 73 | return serializer.Deserialize(compressedData); 74 | } 75 | 76 | public TValue DeserializeNotCompressed(byte[] compressedData) 77 | { 78 | return serializer.Deserialize(compressedData); 79 | } 80 | 81 | public SerializerCompress(Serializer serializer) 82 | { 83 | this.serializer = serializer; 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /FastPersistentDictionary/Internals/Compression/SerializerUnCompressed.cs: -------------------------------------------------------------------------------- 1 |  2 | using GroBuf; 3 | 4 | namespace FastPersistentDictionary.Internals.Compression 5 | { 6 | public class SerializerUnCompressed : ICompressionHandler 7 | { 8 | public readonly Serializer serializer; 9 | public byte[] Serialize(TValue data) 10 | { 11 | return serializer.Serialize(data); 12 | } 13 | 14 | public byte[] Serialize(TValue obj) 15 | { 16 | return serializer.Serialize(obj); 17 | } 18 | 19 | public TValue Deserialize(byte[] compressedData) 20 | { 21 | return serializer.Deserialize(compressedData); 22 | } 23 | 24 | 25 | public TValue Deserialize(byte[] compressedData) 26 | { 27 | return serializer.Deserialize(compressedData); 28 | } 29 | 30 | public TValue Deserialize(Type type, byte[] data) 31 | { 32 | return (TValue)serializer.Deserialize(type, data); 33 | } 34 | 35 | public byte[] SerializeNotCompressed(T obj) 36 | { 37 | return serializer.Serialize(obj); 38 | } 39 | 40 | public TValue DeserializeNotCompressed(byte[] compressedData) 41 | { 42 | return serializer.Deserialize(compressedData); 43 | } 44 | 45 | public TKey DeserializeKey(byte[] compressedData) 46 | { 47 | return serializer.Deserialize(compressedData); 48 | } 49 | 50 | public byte[] SerializeCompressed(T obj) 51 | { 52 | byte[] data = serializer.Serialize(obj); 53 | return K4os.Compression.LZ4.LZ4Pickler.Pickle(data); 54 | } 55 | 56 | public TValue DeserializeCompressed(byte[] compressedData) 57 | { 58 | compressedData = K4os.Compression.LZ4.LZ4Pickler.Unpickle(compressedData); 59 | return serializer.Deserialize(compressedData); 60 | } 61 | 62 | public TValue DeserializeCompressed(byte[] compressedData) 63 | { 64 | compressedData = K4os.Compression.LZ4.LZ4Pickler.Unpickle(compressedData); 65 | return serializer.Deserialize(compressedData); 66 | } 67 | 68 | public TValue DeserializeNotCompressed(byte[] compressedData) 69 | { 70 | return serializer.Deserialize(compressedData); 71 | } 72 | 73 | public SerializerUnCompressed(Serializer serializer) 74 | { 75 | this.serializer = serializer; 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /FastPersistentDictionary/Internals/DictionaryIO.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics.CodeAnalysis; 2 | using System.Text; 3 | using GroBuf; 4 | using FastPersistentDictionary.Internals.Compression; 5 | using System.IO; 6 | 7 | namespace FastPersistentDictionary.Internals 8 | { 9 | [SuppressMessage("ReSharper", "MustUseReturnValue")] 10 | public sealed class DictionaryIo 11 | { 12 | private ICompressionHandler _compressionHandler; 13 | private readonly object _lockObj; 14 | private readonly Serializer _serializer; 15 | private readonly DictionaryQuery _dictionaryQuery; 16 | private DictionaryStructs.DictionarySaveHeader _currentHeader; 17 | internal FileStream FileStream; 18 | internal bool UseCompression; 19 | 20 | public DictionaryIo( 21 | FastPersistentDictionary dict, 22 | ICompressionHandler compressionHandler, 23 | DictionaryQuery dictionaryQuery, 24 | object lockObj, 25 | FileStream fileStream, 26 | Serializer serializer, 27 | bool useCompression) 28 | { 29 | FileStream = fileStream; 30 | _fastPersistentDictionary = dict; 31 | _compressionHandler = compressionHandler; 32 | _serializer = serializer; 33 | _dictionaryQuery = dictionaryQuery; 34 | UseCompression = useCompression; 35 | _currentHeader = new DictionaryStructs.DictionarySaveHeader() 36 | { 37 | UseCompression = UseCompression, //LZ4 Fastest compression option 38 | CreationDate = DateTime.Now, 39 | LastModifiedDate = DateTime.Now, 40 | Version = _fastPersistentDictionary.FastPersistentDictionaryVersion, 41 | DictionaryCount = 0, 42 | KeyType = typeof(TKey).ToString(), 43 | KeyValue = typeof(TValue).ToString(), 44 | ByteLengthOfLookupTable = 0, 45 | PercentageChangeBeforeCompact = _fastPersistentDictionary.PercentageChangeBeforeCompact 46 | }; 47 | 48 | _lockObj = lockObj; 49 | } 50 | 51 | private readonly FastPersistentDictionary _fastPersistentDictionary; 52 | 53 | public override string ToString() 54 | { 55 | var stringBuilder = new StringBuilder(); 56 | 57 | lock (_lockObj) 58 | { 59 | foreach (var keyValPair in _fastPersistentDictionary.DictionaryAccessor) 60 | stringBuilder.Append($"{keyValPair.Key + ": " + keyValPair.Value}\n"); 61 | } 62 | 63 | return stringBuilder.ToString(); 64 | } 65 | 66 | public long GetDatabaseSizeBytes() 67 | { 68 | return FileStream.Length; 69 | } 70 | 71 | public DictionaryStructs.DictionarySaveHeader SaveDictionary(string savePath, string name = "", string comment = "") 72 | { 73 | lock (_lockObj) 74 | { 75 | _dictionaryQuery.UpdateDatabaseTick(true); 76 | 77 | var header = new DictionaryStructs.DictionarySaveHeader() 78 | { 79 | UseCompression = UseCompression, 80 | CreationDate = _currentHeader.CreationDate, 81 | LastModifiedDate = DateTime.Now, 82 | Version = _fastPersistentDictionary.FastPersistentDictionaryVersion, 83 | DictionaryCount = _fastPersistentDictionary.DictionarySerializedLookup.Count, 84 | KeyType = typeof(TKey).ToString(), 85 | KeyValue = typeof(TValue).ToString(), 86 | ByteLengthOfLookupTable = 0, 87 | Comment = comment, 88 | Name = name, 89 | PercentageChangeBeforeCompact = _fastPersistentDictionary.PercentageChangeBeforeCompact 90 | }; 91 | 92 | return SaveDictionary(savePath, header); 93 | } 94 | } 95 | 96 | public DictionaryStructs.DictionarySaveHeader SaveDictionary(string savePath, DictionaryStructs.DictionarySaveHeader header = null) 97 | { 98 | var path = Path.GetDirectoryName(savePath); 99 | 100 | if (Directory.Exists(path) == false) 101 | Directory.CreateDirectory(path); 102 | 103 | lock (_lockObj) 104 | { 105 | var serializedLookup = _compressionHandler.SerializeCompressed(_fastPersistentDictionary.DictionarySerializedLookup); 106 | if (header == null) 107 | { 108 | _dictionaryQuery.UpdateDatabaseTick(true); 109 | header = new DictionaryStructs.DictionarySaveHeader() 110 | { 111 | UseCompression = UseCompression, 112 | CreationDate = _currentHeader.CreationDate, 113 | LastModifiedDate = DateTime.Now, 114 | Version = _fastPersistentDictionary.FastPersistentDictionaryVersion, 115 | DictionaryCount = _fastPersistentDictionary.DictionarySerializedLookup.Count, 116 | KeyType = typeof(TKey).ToString(), 117 | KeyValue = typeof(TValue).ToString(), 118 | ByteLengthOfLookupTable = serializedLookup.Length, 119 | Comment = "", 120 | Name = _fastPersistentDictionary.GetType().Name, 121 | PercentageChangeBeforeCompact = _fastPersistentDictionary.PercentageChangeBeforeCompact 122 | }; 123 | } 124 | 125 | //ensure in case not null from other overload 126 | header.ByteLengthOfLookupTable = serializedLookup.Length; 127 | 128 | var headerBytes = _compressionHandler.SerializeNotCompressed(header); 129 | 130 | using (var origFs = new FileStream(savePath, FileMode.Create, FileAccess.Write)) 131 | { 132 | //1. write out length of compressed database data as long 133 | //2. write database data 134 | //3. write out length of lookup table as int 135 | //4. write out lookup table 136 | //5. write out length of header 137 | //6. write out header 138 | var lookupLength = BitConverter.GetBytes(serializedLookup.Length); 139 | var headerLength = BitConverter.GetBytes(headerBytes.Length); 140 | 141 | FileStream.Seek(0, SeekOrigin.Begin); 142 | var dbLength = BitConverter.GetBytes(FileStream.Length); 143 | 144 | origFs.Write(dbLength, 0, dbLength.Length); //1 145 | FileStream.CopyTo(origFs); //2 146 | 147 | 148 | origFs.Write(lookupLength, 0, lookupLength.Length); //3 149 | origFs.Write(serializedLookup, 0, serializedLookup.Length); //4 150 | origFs.Write(headerLength, 0, headerLength.Length); //5 151 | origFs.Write(headerBytes, 0, headerBytes.Length); //6 152 | 153 | return header; 154 | } 155 | } 156 | } 157 | 158 | 159 | public DictionaryStructs.DictionarySaveHeader LoadDictionary(string loadPath) 160 | { 161 | if (File.Exists(loadPath) == false) 162 | throw new InvalidDataException("File Path does not exist."); 163 | //1. read in length of compressed database data as long 164 | //2. skip database data 165 | //3. read in length of lookup table as int 166 | //4. read in lookup table 167 | //5. read in length of header as int 168 | //6. read in header 169 | 170 | 171 | // FileStream = new FileStream(_fastPersistentDictionary.FileLocation, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite | FileShare.Delete, _fastPersistentDictionary.fileStreamBufferSize, _fastPersistentDictionary.fileOptions); //| FileOptions.DeleteOnClose); 172 | 173 | if (loadPath == FileStream.Name) 174 | { 175 | //FileStream.Close(); 176 | //FileStream = new FileStream(_fastPersistentDictionary.FileLocation, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite | FileShare.Delete, _fastPersistentDictionary.fileStreamBufferSize, _fastPersistentDictionary.fileOptions); //| FileOptions.DeleteOnClose); 177 | var tempPath = Path.GetTempFileName(); 178 | using (var newFileStream = new FileStream(tempPath, FileMode.Open, FileAccess.ReadWrite)) 179 | { 180 | FileStream.Seek(0, SeekOrigin.Begin); 181 | var lengthCompressedDataBaseBytes = new byte[8]; //long 182 | var lengthLookupBytes = new byte[4]; //int //CONSIDER long, doubt file size would ever get that size but only 4 more bytes 183 | var lengthHeaderBytes = new byte[4]; //int 184 | 185 | FileStream.Read(lengthCompressedDataBaseBytes, 0, 8); 186 | var lengthCompressedDataBase = BitConverter.ToInt64(lengthCompressedDataBaseBytes, 0); 187 | FileStream.Position += lengthCompressedDataBase; 188 | 189 | FileStream.Read(lengthLookupBytes, 0, 4); 190 | var lengthLookup = BitConverter.ToInt32(lengthLookupBytes, 0); 191 | var lookupBytes = new byte[lengthLookup]; 192 | FileStream.Read(lookupBytes, 0, lengthLookup); 193 | 194 | FileStream.Read(lengthHeaderBytes, 0, 4); 195 | var lengthHeader = BitConverter.ToInt32(lengthHeaderBytes, 0); 196 | var lookupHeader = new byte[lengthHeader]; 197 | FileStream.Read(lookupHeader, 0, lengthHeader); 198 | 199 | var header = _compressionHandler.DeserializeNotCompressed(lookupHeader); 200 | _currentHeader = header; 201 | 202 | UseCompression = header.UseCompression; 203 | 204 | if (UseCompression) 205 | _compressionHandler = new SerializerCompress(_serializer); 206 | else 207 | _compressionHandler = new SerializerUnCompressed(_serializer); 208 | 209 | var loadedLookup = _compressionHandler.DeserializeCompressed>>(lookupBytes); 210 | lock (_lockObj) 211 | { 212 | _fastPersistentDictionary.PercentageChangeBeforeCompact = header.PercentageChangeBeforeCompact; 213 | 214 | _fastPersistentDictionary.DictionarySerializedLookup = loadedLookup; 215 | 216 | FileStream.Seek(8, SeekOrigin.Begin); // initial long for length on file //huh 217 | var buffer = new byte[8192]; 218 | 219 | int count; 220 | newFileStream.SetLength(0); 221 | 222 | while ((count = FileStream.Read(buffer, 0, buffer.Length)) != 0) 223 | newFileStream.Write(buffer, 0, count); 224 | 225 | FileStream.SetLength(0); 226 | FileStream.Seek(0, SeekOrigin.Begin); 227 | newFileStream.Seek(0, SeekOrigin.Begin); 228 | newFileStream.CopyTo(FileStream); 229 | 230 | _dictionaryQuery.LastFileSizeCompact = FileStream.Length; 231 | _dictionaryQuery.NextFileSizeCompact = _dictionaryQuery.LastFileSizeCompact + (_dictionaryQuery.LastFileSizeCompact * (long)(_fastPersistentDictionary.PercentageChangeBeforeCompact / 100)); 232 | } 233 | 234 | 235 | return header; 236 | } 237 | } 238 | else 239 | { 240 | using (var newFileStream = new FileStream(loadPath, FileMode.Open, FileAccess.ReadWrite)) 241 | { 242 | newFileStream.Seek(0, SeekOrigin.Begin); 243 | var lengthCompressedDataBaseBytes = new byte[8]; //long 244 | var lengthLookupBytes = new byte[4]; //int //CONSIDER long, doubt file size would ever get that size but only 4 more bytes 245 | var lengthHeaderBytes = new byte[4]; //int 246 | 247 | newFileStream.Read(lengthCompressedDataBaseBytes, 0, 8); 248 | var lengthCompressedDataBase = BitConverter.ToInt64(lengthCompressedDataBaseBytes, 0); 249 | newFileStream.Position += lengthCompressedDataBase; 250 | 251 | newFileStream.Read(lengthLookupBytes, 0, 4); 252 | var lengthLookup = BitConverter.ToInt32(lengthLookupBytes, 0); 253 | var lookupBytes = new byte[lengthLookup]; 254 | newFileStream.Read(lookupBytes, 0, lengthLookup); 255 | 256 | newFileStream.Read(lengthHeaderBytes, 0, 4); 257 | var lengthHeader = BitConverter.ToInt32(lengthHeaderBytes, 0); 258 | var lookupHeader = new byte[lengthHeader]; 259 | newFileStream.Read(lookupHeader, 0, lengthHeader); 260 | 261 | var header = _compressionHandler.DeserializeNotCompressed(lookupHeader); 262 | _currentHeader = header; 263 | 264 | UseCompression = header.UseCompression; 265 | 266 | if (UseCompression) 267 | _compressionHandler = new SerializerCompress(_serializer); 268 | else 269 | _compressionHandler = new SerializerUnCompressed(_serializer); 270 | 271 | var loadedLookup = _compressionHandler.DeserializeCompressed>>(lookupBytes); 272 | lock (_lockObj) 273 | { 274 | _fastPersistentDictionary.PercentageChangeBeforeCompact = header.PercentageChangeBeforeCompact; 275 | _fastPersistentDictionary.DictionarySerializedLookup.Clear(); 276 | foreach (var kvp in loadedLookup) 277 | _fastPersistentDictionary.DictionarySerializedLookup[kvp.Key] = kvp.Value; 278 | 279 | newFileStream.Seek(8, SeekOrigin.Begin); // initial long for length on file //huh 280 | var buffer = new byte[8192]; 281 | 282 | int count; 283 | FileStream.SetLength(0); 284 | 285 | while ((count = newFileStream.Read(buffer, 0, buffer.Length)) != 0) 286 | FileStream.Write(buffer, 0, count); 287 | 288 | _dictionaryQuery.LastFileSizeCompact = FileStream.Length; 289 | _dictionaryQuery.NextFileSizeCompact = _dictionaryQuery.LastFileSizeCompact + (_dictionaryQuery.LastFileSizeCompact * (long)(_fastPersistentDictionary.PercentageChangeBeforeCompact / 100)); 290 | } 291 | return header; 292 | } 293 | 294 | } 295 | 296 | 297 | } 298 | 299 | public byte[] PackBuffer() 300 | { 301 | var allData = new Dictionary(); 302 | lock (_lockObj) 303 | { 304 | foreach (var kvp in _fastPersistentDictionary.DictionarySerializedLookup) 305 | { 306 | var data = new byte[kvp.Value.Value]; 307 | FileStream.Seek(kvp.Value.Key, SeekOrigin.Begin); 308 | FileStream.Read(data, 0, kvp.Value.Value); 309 | 310 | allData[kvp.Key] = _compressionHandler.Deserialize(data); 311 | } 312 | } 313 | 314 | return _compressionHandler.Serialize(allData); 315 | } 316 | } 317 | } -------------------------------------------------------------------------------- /FastPersistentDictionary/Internals/DictionaryMathOperations.cs: -------------------------------------------------------------------------------- 1 | using FastPersistentDictionary.Internals.Accessor; 2 | using FastPersistentDictionary.Internals.Compression; 3 | 4 | namespace FastPersistentDictionary.Internals 5 | { 6 | public sealed class DictionaryMathOperations 7 | { 8 | private readonly object _lockObj; 9 | private readonly ICompressionHandler _compressionHandler; 10 | internal FileStream FileStream; 11 | 12 | public DictionaryMathOperations( 13 | FastPersistentDictionary dict, 14 | IDictionaryAccessor dictionaryAccessor, 15 | DictionaryOperations dictionaryOperations, 16 | ICompressionHandler compressionHandler, 17 | object lockObj, 18 | FileStream fileStream) 19 | { 20 | FileStream = fileStream; 21 | _persistentDictionaryPro = dict; 22 | _dictionaryAccessor = dictionaryAccessor; 23 | _dictionaryOperations = dictionaryOperations; 24 | _compressionHandler = compressionHandler; 25 | _lockObj = lockObj; 26 | } 27 | 28 | private readonly FastPersistentDictionary _persistentDictionaryPro; 29 | private readonly IDictionaryAccessor _dictionaryAccessor; 30 | private readonly DictionaryOperations _dictionaryOperations; 31 | 32 | public TValue Min() 33 | { 34 | lock (_lockObj) 35 | { 36 | var comparer = Comparer.Default; 37 | var min = default(TValue); 38 | 39 | var isFirstValue = true; 40 | 41 | foreach (var key in _persistentDictionaryPro.DictionarySerializedLookup.Keys) 42 | { 43 | var value = _dictionaryAccessor.Get(key); 44 | if (isFirstValue == false && comparer.Compare(value, min) >= 0) 45 | continue; 46 | 47 | min = value; 48 | isFirstValue = false; 49 | } 50 | 51 | if (isFirstValue) 52 | throw new InvalidOperationException("No elements in the dictionary."); 53 | 54 | return min; 55 | } 56 | } 57 | 58 | public TValue Sum() 59 | { 60 | if (typeof(TValue).IsPrimitive == false && typeof(TValue) != typeof(decimal)) 61 | throw new InvalidOperationException("Sum method can only be used for TValue being a primitive type."); 62 | 63 | dynamic sumVal = 0; 64 | lock (_lockObj) 65 | { 66 | foreach (var kvp in _persistentDictionaryPro.DictionarySerializedLookup.Values) 67 | { 68 | var data = new byte[kvp.Value]; 69 | FileStream.Seek(kvp.Key, SeekOrigin.Begin); 70 | FileStream.Read(data, 0, kvp.Value); 71 | 72 | sumVal += _compressionHandler.Deserialize(data); 73 | } 74 | } 75 | return (TValue)sumVal; 76 | } 77 | 78 | public bool TryGetMax(out TValue max) 79 | { 80 | lock (_lockObj) 81 | { 82 | if (_persistentDictionaryPro.DictionarySerializedLookup.Count == 0) 83 | { 84 | max = default; 85 | return false; 86 | } 87 | 88 | var comparer = Comparer.Default; 89 | max = default; 90 | var firstRound = true; 91 | 92 | foreach (var key in _persistentDictionaryPro.DictionarySerializedLookup.Keys) 93 | { 94 | var value = _dictionaryAccessor.Get(key); 95 | 96 | if (value == null) 97 | continue; 98 | if (firstRound == false && comparer.Compare(value, max) <= 0) 99 | continue; 100 | 101 | max = value; 102 | firstRound = false; 103 | } 104 | } 105 | 106 | return true; 107 | } 108 | 109 | public bool TryGetMin(out TValue min) 110 | { 111 | var comparer = Comparer.Default; 112 | min = default; 113 | 114 | var isFirstValue = true; 115 | lock (_lockObj) 116 | { 117 | foreach (var key in _persistentDictionaryPro.DictionarySerializedLookup.Keys) 118 | { 119 | var value = _dictionaryAccessor.Get(key); 120 | if (value == null) 121 | continue; 122 | 123 | if (isFirstValue == false && comparer.Compare(value, min) >= 0) 124 | continue; 125 | 126 | min = value; 127 | isFirstValue = false; 128 | } 129 | } 130 | 131 | if (min == null) 132 | return false; 133 | 134 | return isFirstValue == false; 135 | } 136 | 137 | public TValue Max() 138 | { 139 | lock (_lockObj) 140 | { 141 | if (_persistentDictionaryPro.Count == 0) 142 | throw new InvalidOperationException("No items in dictionary"); 143 | 144 | var comparer = Comparer.Default; 145 | var max = default(TValue); 146 | 147 | foreach (var key in _persistentDictionaryPro.DictionarySerializedLookup.Keys) 148 | { 149 | var value = _dictionaryAccessor.Get(key); 150 | if (comparer.Compare(value, max) > 0) 151 | max = value; 152 | } 153 | 154 | return max; 155 | } 156 | } 157 | 158 | public double Average(Func, double> selector) 159 | { 160 | lock (_lockObj) 161 | return _dictionaryOperations.ToList().Average(selector); 162 | } 163 | 164 | public TKey MaxKey() 165 | { 166 | lock (_lockObj) 167 | { 168 | var totalKeys = _persistentDictionaryPro.Keys; 169 | if (totalKeys.Any() == false) 170 | throw new InvalidOperationException("The Dictionary is empty."); 171 | 172 | return totalKeys.Max(); 173 | } 174 | } 175 | 176 | public TKey MinKey() 177 | { 178 | lock (_lockObj) 179 | { 180 | var totalKeys = _persistentDictionaryPro.Keys; 181 | if (totalKeys.Any() == false) 182 | throw new InvalidOperationException("The Dictionary is empty."); 183 | 184 | return totalKeys.Min(); 185 | } 186 | } 187 | } 188 | } -------------------------------------------------------------------------------- /FastPersistentDictionary/Internals/DictionaryObject.cs: -------------------------------------------------------------------------------- 1 | using System.Collections; 2 | using System.Runtime.CompilerServices; 3 | using FastPersistentDictionary.Internals.Accessor; 4 | 5 | namespace FastPersistentDictionary.Internals 6 | { 7 | public sealed class DictionaryObject : IEnumerable 8 | { 9 | private readonly object _lockObj; 10 | public DictionaryObject(IDictionaryAccessor dictionaryAccessor, object lockObj) 11 | { 12 | _dictionaryAccessor = dictionaryAccessor; 13 | _lockObj = lockObj; 14 | } 15 | 16 | private readonly IDictionaryAccessor _dictionaryAccessor; 17 | 18 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 19 | IEnumerator IEnumerable.GetEnumerator() 20 | { 21 | return _dictionaryAccessor.GetEnumerator(); 22 | } 23 | 24 | public IEnumerator> GetEnumerator() 25 | { 26 | return _dictionaryAccessor.GetEnumerator(); 27 | } 28 | 29 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 30 | public override bool Equals(object obj) 31 | { 32 | lock (_lockObj) 33 | { 34 | if (this.GetHashCode() == obj.GetHashCode()) 35 | return true; 36 | 37 | if (ReferenceEquals(this, obj)) 38 | return true; 39 | 40 | lock (_lockObj) 41 | { 42 | if (obj is FastPersistentDictionary other == false) 43 | return false; 44 | 45 | if (other.DictionaryAccessor.Count != _dictionaryAccessor.Count) 46 | return false; 47 | 48 | foreach (var pair in this) 49 | { 50 | if (other.DictionaryAccessor.TryGetValue(pair.Key, out var otherVal) == false) 51 | return false; 52 | 53 | if (Equals(pair.Value, otherVal) == false) 54 | return false; 55 | } 56 | } 57 | 58 | return true; 59 | } 60 | } 61 | } 62 | } -------------------------------------------------------------------------------- /FastPersistentDictionary/Internals/DictionaryOperations.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Immutable; 2 | using System.Runtime.CompilerServices; 3 | using FastPersistentDictionary.Internals.Accessor; 4 | using FastPersistentDictionary.Internals.Compression; 5 | 6 | namespace FastPersistentDictionary.Internals 7 | { 8 | public sealed class DictionaryOperations 9 | { 10 | private readonly object _lockObj; 11 | private readonly System.Timers.Timer _updateTimer; 12 | private readonly ICompressionHandler _compressionHandler; 13 | internal FileStream FileStream; 14 | 15 | public DictionaryOperations( 16 | FastPersistentDictionary dict, 17 | IDictionaryAccessor dictionaryAccessor, 18 | System.Timers.Timer updateTimer, 19 | ICompressionHandler compressionHandler, 20 | object lockObj, 21 | FileStream fileStream) 22 | { 23 | _persistentDictionaryPro = dict; 24 | _dictionaryAccessor = dictionaryAccessor; 25 | _updateTimer = updateTimer; 26 | _compressionHandler = compressionHandler; 27 | _lockObj = lockObj; 28 | FileStream = fileStream; 29 | } 30 | 31 | private readonly FastPersistentDictionary _persistentDictionaryPro; 32 | private readonly IDictionaryAccessor _dictionaryAccessor; 33 | 34 | public IOrderedEnumerable> OrderByDescending(Func, object> selector) 35 | { 36 | var allEntries = new List>(); 37 | lock (_lockObj) 38 | foreach (var key in _persistentDictionaryPro.DictionarySerializedLookup.Keys) 39 | allEntries.Add(new KeyValuePair(key, _dictionaryAccessor.Get(key))); 40 | 41 | return allEntries.OrderByDescending(selector); 42 | } 43 | 44 | public IOrderedEnumerable> OrderBy(Func, object> selector) 45 | { 46 | var allEntries = new List>(); 47 | lock (_lockObj) 48 | foreach (var key in _persistentDictionaryPro.DictionarySerializedLookup.Keys) 49 | allEntries.Add(new KeyValuePair(key, _dictionaryAccessor.Get(key))); 50 | 51 | return allEntries.OrderBy(selector); 52 | } 53 | 54 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 55 | public List> Reverse() 56 | { 57 | var allEntries = new List>(); 58 | lock (_lockObj) 59 | foreach (var key in _persistentDictionaryPro.DictionarySerializedLookup.Keys) 60 | allEntries.Add(new KeyValuePair(key, _dictionaryAccessor.Get(key))); 61 | 62 | allEntries.Reverse(); 63 | return allEntries; 64 | } 65 | 66 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 67 | public KeyValuePair[] ToArray() 68 | { 69 | lock (_lockObj) 70 | { 71 | var array = new KeyValuePair[_persistentDictionaryPro.DictionarySerializedLookup.Count]; 72 | var index = 0; 73 | 74 | foreach (var item in _persistentDictionaryPro.DictionarySerializedLookup) 75 | { 76 | var data = new byte[item.Value.Value]; 77 | FileStream.Seek(item.Value.Key, SeekOrigin.Begin); 78 | FileStream.Read(data, 0, item.Value.Value); 79 | 80 | array[index++] = new KeyValuePair(item.Key, _compressionHandler.Deserialize(data)); 81 | } 82 | return array; 83 | } 84 | } 85 | 86 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 87 | public List> ToList() 88 | { 89 | return ToArray().ToList(); 90 | } 91 | 92 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 93 | public IEnumerable> Take(int count) 94 | { 95 | var startIndex = 0; 96 | foreach (var item in _dictionaryAccessor) 97 | { 98 | if (startIndex++ == count) 99 | break; 100 | 101 | yield return item; 102 | } 103 | } 104 | 105 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 106 | public IEnumerable> TakeWhile(Func, bool> predicate) 107 | { 108 | foreach (var item in _dictionaryAccessor) 109 | { 110 | if (predicate(item) == false) 111 | break; 112 | 113 | yield return item; 114 | } 115 | } 116 | 117 | /// 118 | /// `Copy()`: Creates a shallow copy of the dictionary. 119 | /// 120 | /// 121 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 122 | public FastPersistentDictionary Copy(string path) 123 | { 124 | lock (_lockObj) 125 | { 126 | var copiedDictionary = new FastPersistentDictionary( 127 | path, 128 | _persistentDictionaryPro.CrashRecovery, 129 | (int)_updateTimer.Interval, 130 | _persistentDictionaryPro.DeleteDBOnClose, 131 | _persistentDictionaryPro.PercentageChangeBeforeCompact, 132 | 8196, 133 | "", 134 | _persistentDictionaryPro.Comparer); 135 | 136 | _updateTimer.Stop(); 137 | 138 | 139 | foreach (var kvp in _persistentDictionaryPro.DictionarySerializedLookup.Keys) 140 | copiedDictionary.Add(kvp, _persistentDictionaryPro.DictionaryAccessor.Get(kvp)); 141 | 142 | _updateTimer.Start(); 143 | 144 | return copiedDictionary; 145 | } 146 | } 147 | 148 | /// 149 | /// SkipLast: This skips the last `n` items in the sequence 150 | /// 151 | /// 152 | /// 153 | public IEnumerable> SkipLast(int number) 154 | { 155 | lock (_lockObj) 156 | { 157 | var itemCount = _persistentDictionaryPro.DictionarySerializedLookup.Count; 158 | 159 | if (itemCount <= number) 160 | return Enumerable.Empty>(); 161 | 162 | var results = new List>(itemCount - number); 163 | var index = 0; 164 | var len = _persistentDictionaryPro.Count - number; 165 | 166 | foreach (var entry in _persistentDictionaryPro) 167 | { 168 | if (index < len) 169 | { 170 | ++index; 171 | results.Add(new KeyValuePair(entry.Key, _persistentDictionaryPro.DictionaryAccessor.Get(entry.Key))); 172 | } 173 | else 174 | break; 175 | } 176 | 177 | return results; 178 | } 179 | } 180 | 181 | 182 | public void Join(Dictionary other, bool overwriteExistingKeys = true) => Merge(other, overwriteExistingKeys); 183 | public void Join(FastPersistentDictionary other, bool overwriteExistingKeys = true) => Merge(other, overwriteExistingKeys); 184 | 185 | /// 186 | /// `Merge(Dictionary other)`: Import all entries from another Dictionary, has an optional parameter for overwriting existing keys. 187 | /// 188 | /// 189 | /// 190 | public void Merge(Dictionary other, bool overwriteExistingKeys = true) 191 | { 192 | lock (_lockObj) 193 | foreach (var entry in other) 194 | { 195 | if (overwriteExistingKeys == false && _persistentDictionaryPro.ContainsKey(entry.Key)) 196 | continue; 197 | 198 | _persistentDictionaryPro.DictionaryAccessor.Set(entry.Key, entry.Value); 199 | } 200 | } 201 | 202 | /// 203 | /// `Merge(PersistentDictionaryPro other)`: Import all entries from another PersistentDictionary, has an optional parameter for overwriting existing keys. 204 | /// 205 | /// 206 | /// 207 | public void Merge(FastPersistentDictionary other, bool overwriteExistingKeys = true) 208 | { 209 | lock (_lockObj) 210 | foreach (var entry in other) 211 | { 212 | if (overwriteExistingKeys == false && _persistentDictionaryPro.ContainsKey(entry.Key)) 213 | continue; 214 | 215 | _persistentDictionaryPro.DictionaryAccessor.Set(entry.Key, entry.Value); 216 | } 217 | } 218 | 219 | /// 220 | /// `Invert()`: If all values are unique, invert the dictionary so that the keys become values and vice versa. 221 | /// 222 | /// 223 | /// 224 | public FastPersistentDictionary Invert() 225 | { 226 | lock (_lockObj) 227 | { 228 | var invertedDictionary = new FastPersistentDictionary(_persistentDictionaryPro.FileLocation); 229 | 230 | var valuesSet = new HashSet(_persistentDictionaryPro.Values); 231 | 232 | if (_persistentDictionaryPro.Values.Count() != valuesSet.Count) 233 | throw new InvalidOperationException( 234 | "Cannot invert PersistentDictionaryPro: not all values are unique."); 235 | 236 | foreach (var key in _persistentDictionaryPro.Keys) 237 | invertedDictionary.Add(_persistentDictionaryPro.DictionaryAccessor.Get(key), key); 238 | 239 | return invertedDictionary; 240 | } 241 | } 242 | 243 | public IEnumerable> Skip(int number) 244 | { 245 | if (number < 0) 246 | throw new ArgumentOutOfRangeException(nameof(number), "Number of elements to skip cannot be negative."); 247 | 248 | var count = 0; 249 | lock (_lockObj) 250 | foreach (var key in _persistentDictionaryPro.Keys) 251 | { 252 | if (count++ < number) 253 | continue; 254 | 255 | yield return new KeyValuePair(key, _dictionaryAccessor.Get(key)); 256 | } 257 | } 258 | 259 | public IEnumerable> SkipWhile(Func, bool> predicate) 260 | { 261 | lock (_lockObj) 262 | { 263 | var skipping = true; 264 | 265 | foreach (var key in _persistentDictionaryPro.Keys) 266 | { 267 | var entry = new KeyValuePair(key, _dictionaryAccessor.Get(key)); 268 | switch (skipping) 269 | { 270 | case true when predicate(entry): 271 | continue; 272 | case true: 273 | skipping = false; 274 | break; 275 | } 276 | 277 | yield return entry; 278 | } 279 | } 280 | } 281 | 282 | public IEnumerable> DistinctByKey() 283 | { 284 | var distinctKeys = new HashSet(); 285 | lock (_lockObj) 286 | foreach (var pair in _dictionaryAccessor) 287 | if (distinctKeys.Add(pair.Key)) 288 | yield return pair; 289 | } 290 | 291 | public IEnumerable> Except(FastPersistentDictionary otherDict) 292 | { 293 | var otherDictAsRegularDict = new Dictionary(); 294 | lock (_lockObj) 295 | { 296 | foreach (var pair in otherDict) 297 | otherDictAsRegularDict.Add(pair.Key, pair.Value); 298 | 299 | foreach (var pair in _dictionaryAccessor) 300 | if (otherDictAsRegularDict.ContainsKey(pair.Key) == false) 301 | yield return pair; 302 | } 303 | } 304 | 305 | public void Concat(FastPersistentDictionary other) 306 | { 307 | lock (_lockObj) 308 | foreach (var keyValuePair in _persistentDictionaryPro) 309 | { 310 | var key = keyValuePair.Key; 311 | _dictionaryAccessor.Add(key, other.DictionaryAccessor.Get(key)); 312 | } 313 | } 314 | 315 | public FastPersistentDictionary Intersect(FastPersistentDictionary other) 316 | { 317 | lock (_lockObj) 318 | { 319 | var intersection = 320 | new FastPersistentDictionary(_persistentDictionaryPro.FileLocation); 321 | 322 | foreach (var pair in _dictionaryAccessor) 323 | if (other.DictionaryAccessor.ContainsKey(pair.Key)) 324 | intersection.DictionaryAccessor.Add(pair.Key, pair.Value); 325 | 326 | return intersection; 327 | } 328 | } 329 | 330 | public FastPersistentDictionary Union(FastPersistentDictionary other) 331 | { 332 | lock (_lockObj) 333 | { 334 | var unionDictionary = 335 | new FastPersistentDictionary(_persistentDictionaryPro.FileLocation); 336 | 337 | 338 | foreach (var kvp in _persistentDictionaryPro.DictionarySerializedLookup.Keys) 339 | unionDictionary.DictionaryAccessor.Add(kvp, _persistentDictionaryPro.DictionaryAccessor.Get(kvp)); 340 | 341 | foreach (var kvp in _persistentDictionaryPro.DictionarySerializedLookup.Keys) 342 | if (unionDictionary.DictionaryAccessor.ContainsKey(kvp) == false) 343 | unionDictionary.DictionaryAccessor.Add(kvp, other.DictionaryAccessor.Get(kvp)); 344 | 345 | return unionDictionary; 346 | } 347 | } 348 | 349 | /// 350 | /// InBatchesOf: This method can process your sequence in batches of specified size. 351 | /// 352 | /// 353 | /// 354 | public IEnumerable>> InBatchesOf(int batchSize) 355 | { 356 | var batch = new List>(); 357 | lock (_lockObj) 358 | { 359 | foreach (var key in _persistentDictionaryPro.Keys) 360 | { 361 | batch.Add(new KeyValuePair(key, _persistentDictionaryPro[key])); 362 | if (batch.Count != batchSize) 363 | continue; 364 | 365 | yield return batch; 366 | batch = new List>(); 367 | } 368 | 369 | if (batch.Any()) 370 | yield return batch; 371 | 372 | } 373 | } 374 | 375 | //Todo: this can be optimized more, got lazy 376 | /// 377 | /// TakeLast: This method returns the last `n` items in the sequence. 378 | /// 379 | /// 380 | /// 381 | public IEnumerable> TakeLast(int number) 382 | { 383 | lock (_lockObj) 384 | { 385 | if (number > _persistentDictionaryPro.Count) 386 | number = _persistentDictionaryPro.Count; 387 | 388 | foreach (var key in _persistentDictionaryPro.DictionarySerializedLookup.Keys.Skip(Math.Max(0, _persistentDictionaryPro.DictionarySerializedLookup.Keys.Count() - number))) 389 | yield return new KeyValuePair(key, _dictionaryAccessor.Get(key)); 390 | } 391 | } 392 | 393 | public FastPersistentDictionary> Zip(FastPersistentDictionary second) 394 | { 395 | lock (_lockObj) 396 | { 397 | var result = new FastPersistentDictionary>(_persistentDictionaryPro.FileLocation, _persistentDictionaryPro.CrashRecovery, (int)_updateTimer.Interval); 398 | 399 | foreach (var key in _dictionaryAccessor.Keys) 400 | if (second.DictionaryAccessor.TryGetValue(key, out var secondValue)) 401 | result.DictionaryAccessor.Add(key, Tuple.Create(_dictionaryAccessor.Get(key), secondValue)); 402 | 403 | return result; 404 | } 405 | } 406 | 407 | public ImmutableList> ToImmutableList() => ToArray().ToImmutableList(); 408 | 409 | public ImmutableArray> ToImmutableArray() => ToArray().ToImmutableArray(); 410 | 411 | public ImmutableDictionary ToImmutableDictionary() => ToArray().ToImmutableDictionary(); 412 | 413 | public ImmutableHashSet> ToImmutableHashSet() => ToArray().ToImmutableHashSet(); 414 | 415 | public SortedSet> ToSortedSet() 416 | { 417 | lock (_lockObj) 418 | return new SortedSet>(ToArray(), new KeyValuePairComparer()); 419 | } 420 | 421 | public SortedDictionary ToSortedDictionary() 422 | { 423 | var sortedDictionary = new SortedDictionary(); 424 | foreach (var keyValuePair in _dictionaryAccessor) 425 | sortedDictionary.Add(keyValuePair.Key, keyValuePair.Value); 426 | 427 | return sortedDictionary; 428 | } 429 | 430 | public IQueryable> AsQueryable() => ToArray().AsQueryable(); 431 | 432 | public ParallelQuery> AsParallel() 433 | { 434 | lock (_lockObj) 435 | { 436 | var lookup = _persistentDictionaryPro.DictionarySerializedLookup.AsParallel().Select(kv => new KeyValuePair(kv.Key, _dictionaryAccessor.Get(kv.Key))); 437 | return lookup; 438 | } 439 | } 440 | 441 | public void Shuffle() 442 | { 443 | var list = new List>(); 444 | lock (_lockObj) 445 | { 446 | foreach (var kvp in _persistentDictionaryPro.DictionarySerializedLookup.Keys) 447 | list.Add(new KeyValuePair(kvp, _persistentDictionaryPro.DictionaryAccessor.Get(kvp))); 448 | 449 | var shuffledList = list.OrderBy(x => Guid.NewGuid()).ToList(); 450 | 451 | _persistentDictionaryPro.DictionarySerializedLookup.Clear(); 452 | FileStream.SetLength(0); 453 | FileStream.Seek(0, SeekOrigin.Begin); 454 | 455 | foreach (var kvp in shuffledList) 456 | { 457 | var compressedData = _compressionHandler.Serialize(kvp.Value); 458 | var location = new KeyValuePair(FileStream.Position, compressedData.Length); 459 | FileStream.Write(compressedData, 0, compressedData.Length); 460 | 461 | _persistentDictionaryPro.DictionarySerializedLookup[kvp.Key] = location; 462 | } 463 | } 464 | } 465 | } 466 | } -------------------------------------------------------------------------------- /FastPersistentDictionary/Internals/DictionaryQuery.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.CompilerServices; 2 | using System.Timers; 3 | using FastPersistentDictionary.Internals.Accessor; 4 | using FastPersistentDictionary.Internals.Compression; 5 | using Timer = System.Timers.Timer; 6 | 7 | namespace FastPersistentDictionary.Internals 8 | { 9 | public sealed class DictionaryQuery 10 | { 11 | private readonly object _lockObj; 12 | private readonly Timer _updateTimer; 13 | private readonly Timer _idleCompactTimer; 14 | 15 | private readonly ICompressionHandler _compressionHandler; 16 | private const int IdleCompactTime = 1000; 17 | private readonly float _percentageDiffBeforeCompact; 18 | public long LastFileSizeCompact { get; set; } 19 | public long NextFileSizeCompact { get; set; } 20 | // private bool _canCompact; 21 | 22 | internal FileStream FileStream; 23 | public DictionaryQuery( 24 | FastPersistentDictionary dict, 25 | IDictionaryAccessor dictionaryAccessor, 26 | Timer updateTimer, 27 | ICompressionHandler compressionHandler, 28 | float percentageDiffBeforeCompact, 29 | object lockObj, 30 | FileStream fileStream) 31 | { 32 | FileStream = fileStream; 33 | _fastPersistentDictionary = dict; 34 | _dictionaryAccessor = dictionaryAccessor; 35 | _updateTimer = updateTimer; 36 | _compressionHandler = compressionHandler; 37 | _lockObj = lockObj; 38 | _percentageDiffBeforeCompact = percentageDiffBeforeCompact; 39 | LastFileSizeCompact = fileStream.Length; 40 | 41 | 42 | NextFileSizeCompact = LastFileSizeCompact + (long)(LastFileSizeCompact * (_percentageDiffBeforeCompact / 100f)); 43 | 44 | _idleCompactTimer = new Timer(IdleCompactTime + _updateTimer.Interval); 45 | _idleCompactTimer.Elapsed += UpdateCompactTimerElapsedEventHandler; 46 | _idleCompactTimer.Enabled = false; 47 | } 48 | 49 | private readonly FastPersistentDictionary _fastPersistentDictionary; 50 | private readonly IDictionaryAccessor _dictionaryAccessor; 51 | 52 | private void UpdateCompactTimerElapsedEventHandler(object sender, ElapsedEventArgs e) 53 | { 54 | IdleCompactTimerEvent(); 55 | } 56 | 57 | public bool All(Func, bool> predicate) 58 | { 59 | lock (_lockObj) 60 | foreach (var entry in _fastPersistentDictionary.DictionarySerializedLookup.Keys) 61 | if (predicate(new KeyValuePair(entry, _dictionaryAccessor.Get(entry))) == false) 62 | return false; 63 | 64 | return true; 65 | } 66 | 67 | public bool Any() 68 | { 69 | lock (_lockObj) 70 | return _fastPersistentDictionary.DictionarySerializedLookup.Count > 0; 71 | } 72 | 73 | public bool Any(Func predicate) 74 | { 75 | lock (_lockObj) 76 | foreach (var key in _fastPersistentDictionary.DictionarySerializedLookup.Keys) 77 | if (predicate(key, _dictionaryAccessor.Get(key))) 78 | return true; 79 | 80 | 81 | return false; 82 | } 83 | 84 | public void IdleCompactTimerEvent() 85 | { 86 | _idleCompactTimer.Stop(); 87 | //if (_canCompact) 88 | CompactDatabaseFile(); 89 | } 90 | 91 | 92 | //[MethodImpl(MethodImplOptions.AggressiveInlining)] 93 | //public async Task CompactDatabaseFile()//Async() 94 | //{ 95 | // Stopwatch stpw = Stopwatch.StartNew(); 96 | // Console.WriteLine("compact"); 97 | // var tempFilePath = Path.GetTempFileName(); 98 | 99 | // _fastPersistentDictionary.CanCompact = false; 100 | // _idleCompactTimer.Stop(); 101 | 102 | // var tempDict = new Dictionary>(); 103 | 104 | // using (var tempFileStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite, bufferSize: 1024 * 1024, useAsync: true)) 105 | // { 106 | // List tasks = new(); 107 | // //lock (_lockObj) 108 | // //{ 109 | // foreach (var kvp in _fastPersistentDictionary.DictionarySerializedLookup) 110 | // { 111 | // tasks.Add(Task.Run(async () => 112 | // { 113 | // byte[] buffer = new byte[kvp.Value.Value]; 114 | 115 | // lock (_lockObj) 116 | // { 117 | // FileStream.Seek(kvp.Value.Key, SeekOrigin.Begin); 118 | // } 119 | 120 | // int bytesRead = await FileStream.ReadAsync(buffer, 0, kvp.Value.Value).ConfigureAwait(false); 121 | // await tempFileStream.WriteAsync(buffer, 0, bytesRead).ConfigureAwait(false); 122 | 123 | // lock (_lockObj) 124 | // { 125 | // tempDict[kvp.Key] = new KeyValuePair(tempFileStream.Position - kvp.Value.Value, kvp.Value.Value); 126 | // } 127 | // })); 128 | // } 129 | // //} 130 | 131 | // Task.WhenAll(tasks); 132 | 133 | // // Replace the old dictionary with the updated one 134 | // _fastPersistentDictionary.DictionarySerializedLookup = tempDict; 135 | 136 | // FileStream.SetLength(0); 137 | // FileStream.Seek(0, SeekOrigin.Begin); 138 | // tempFileStream.Seek(0, SeekOrigin.Begin); 139 | // tempFileStream.CopyTo(FileStream); 140 | 141 | // LastFileSizeCompact = FileStream.Length; 142 | // NextFileSizeCompact = LastFileSizeCompact + (long)(LastFileSizeCompact * (_percentageDiffBeforeCompact / 100f)); 143 | // } 144 | 145 | 146 | // if (File.Exists(tempFilePath)) 147 | // File.Delete(tempFilePath); 148 | 149 | // stpw.Stop(); 150 | // Console.WriteLine(stpw.ElapsedMilliseconds); 151 | //} 152 | 153 | 154 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 155 | public void CompactDatabaseFile() 156 | { 157 | //Stopwatch stpw = Stopwatch.StartNew(); 158 | //Console.WriteLine("compact"); 159 | var tempFilePath = Path.GetTempFileName(); 160 | lock (_lockObj) 161 | { 162 | _fastPersistentDictionary.CanCompact = false; 163 | // _canCompact = false; 164 | _idleCompactTimer.Stop(); 165 | 166 | using (var tempFileStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.None, bufferSize: 1024 * 1024, FileOptions.SequentialScan | FileOptions.DeleteOnClose)) 167 | { 168 | Span buffer = new Span(); 169 | 170 | var tempDict = new Dictionary>(_fastPersistentDictionary.DictionarySerializedLookup.Count); 171 | foreach (var kvp in _fastPersistentDictionary.DictionarySerializedLookup) 172 | { 173 | FileStream.Seek(kvp.Value.Key, SeekOrigin.Begin); 174 | 175 | var bytesLeft = kvp.Value.Value; 176 | buffer = new byte[bytesLeft]; 177 | 178 | 179 | var bytesRead = FileStream.Read(buffer); 180 | tempFileStream.Write(buffer); 181 | tempDict[kvp.Key] = new KeyValuePair(tempFileStream.Position - kvp.Value.Value, kvp.Value.Value); 182 | } 183 | 184 | // Replace the old dictionary with the updated one 185 | _fastPersistentDictionary.DictionarySerializedLookup = tempDict; 186 | 187 | FileStream.SetLength(0); 188 | FileStream.Seek(0, SeekOrigin.Begin); 189 | tempFileStream.Seek(0, SeekOrigin.Begin); 190 | tempFileStream.CopyTo(FileStream); 191 | 192 | LastFileSizeCompact = FileStream.Length; 193 | NextFileSizeCompact = LastFileSizeCompact + (long)(LastFileSizeCompact * (_percentageDiffBeforeCompact / 100f)); 194 | } 195 | } 196 | 197 | //if (File.Exists(tempFilePath)) 198 | // File.Delete(tempFilePath); 199 | 200 | //stpw.Stop(); 201 | //Console.WriteLine(stpw.ElapsedMilliseconds); 202 | } 203 | 204 | 205 | /// 206 | /// `CountValues(Value value)`: Counts the number of keys associated with a particular value. 207 | /// 208 | /// 209 | /// 210 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 211 | public int CountValues(TValue value) 212 | { 213 | var count = 0; 214 | lock (_lockObj) 215 | foreach (var kvp in _fastPersistentDictionary.DictionarySerializedLookup.Values) 216 | { 217 | var data = new byte[kvp.Value]; 218 | FileStream.Seek(kvp.Key, SeekOrigin.Begin); 219 | FileStream.Read(data, 0, kvp.Value); 220 | 221 | if (_compressionHandler.Deserialize(data).Equals(value)) 222 | ++count; 223 | } 224 | 225 | return count; 226 | } 227 | 228 | public KeyValuePair ElementAt(int index) 229 | { 230 | lock (_lockObj) 231 | { 232 | if (index < 0 || index >= _fastPersistentDictionary.DictionarySerializedLookup.Count) 233 | throw new ArgumentOutOfRangeException(nameof(index)); 234 | 235 | var valuePair = _fastPersistentDictionary.DictionarySerializedLookup.ElementAt(index); 236 | 237 | FileStream.Seek(valuePair.Value.Key, SeekOrigin.Begin); 238 | var data = new byte[valuePair.Value.Value]; 239 | FileStream.Read(data, 0, valuePair.Value.Value); 240 | var value = _compressionHandler.Deserialize(data); 241 | 242 | return new KeyValuePair(valuePair.Key, value); 243 | } 244 | } 245 | 246 | /// 247 | /// `FilteredWhere(Func predicate)`: Query the dictionary, returning a new dictionary containing only key/value pairs that match the predicate. 248 | /// 249 | /// 250 | /// 251 | public FastPersistentDictionary FilteredWhere(Func predicate) 252 | { 253 | lock (_lockObj) 254 | { 255 | var result = new FastPersistentDictionary(_fastPersistentDictionary.FileLocation, equalityComparer: _fastPersistentDictionary.Comparer); 256 | foreach (var kvp in _fastPersistentDictionary) 257 | if (predicate(kvp.Key, kvp.Value)) 258 | result.Add(kvp.Key, kvp.Value); 259 | 260 | return result; 261 | } 262 | } 263 | 264 | public KeyValuePair First() 265 | { 266 | lock (_lockObj) 267 | { 268 | UpdateDatabaseTick(); 269 | 270 | var key = _fastPersistentDictionary.DictionarySerializedLookup.Keys.First(); 271 | var value = _dictionaryAccessor.Get(key); 272 | return new KeyValuePair(key, value); 273 | } 274 | } 275 | 276 | public KeyValuePair? FirstOrDefault() 277 | { 278 | lock (_lockObj) 279 | { 280 | UpdateDatabaseTick(); 281 | 282 | var key = _fastPersistentDictionary.DictionarySerializedLookup.Keys.FirstOrDefault(); 283 | if (key == null) 284 | return null; 285 | 286 | var value = _dictionaryAccessor.GetReturnDefaultIfNotFound(key); 287 | 288 | return new KeyValuePair(key, value); 289 | } 290 | } 291 | 292 | /// 293 | /// `FindAllKeysByValue(Value value)`: Finds all keys in the dictionary by its value Returning a list. 294 | /// 295 | /// 296 | /// 297 | public List FindAllKeysByValue(TValue value) 298 | { 299 | lock (_lockObj) 300 | { 301 | var keys = new List(); 302 | // Check the file database 303 | foreach (var entry in _fastPersistentDictionary.DictionarySerializedLookup.Keys) 304 | { 305 | var serializedKey = entry; 306 | if (EqualityComparer.Default.Equals(_dictionaryAccessor.Get(serializedKey), value)) 307 | keys.Add(serializedKey); 308 | } 309 | 310 | return keys; 311 | } 312 | } 313 | 314 | /// 315 | /// `FindKeyByValue(Value value)`: Finds a key in the dictionary by its value. 316 | /// 317 | /// 318 | /// 319 | /// 320 | public TKey FindKeyByValue(TValue value) 321 | { 322 | lock (_lockObj) 323 | { 324 | foreach (var entry in _fastPersistentDictionary.DictionarySerializedLookup.Keys) 325 | { 326 | var serializedKey = entry; 327 | if (EqualityComparer.Default.Equals(_dictionaryAccessor.Get(serializedKey), value)) 328 | return serializedKey; 329 | } 330 | 331 | throw new KeyNotFoundException("The key for the given value was not found in the dictionary."); 332 | } 333 | } 334 | 335 | /// 336 | /// `GetSubset(List keys)`: Returns a new Dictionary that is a subset of the original Dictionary and only includes the keys provided. 337 | /// 338 | /// 339 | /// 340 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 341 | public FastPersistentDictionary GetSubset(Func predicate) 342 | { 343 | lock (_lockObj) 344 | { 345 | var newDict = new FastPersistentDictionary(_fastPersistentDictionary.FileLocation); 346 | foreach (var kvp in _fastPersistentDictionary) 347 | { 348 | if (predicate(kvp.Key, kvp.Value)) 349 | newDict.Add(kvp.Key, kvp.Value); 350 | } 351 | 352 | return newDict; 353 | } 354 | } 355 | 356 | /// 357 | /// `GetSubset(List keys)`: Returns a new Dictionary that is a subset of the original Dictionary and only includes the keys provided. 358 | /// 359 | /// 360 | /// 361 | public FastPersistentDictionary GetSubset(List keys) 362 | { 363 | lock (_lockObj) 364 | { 365 | var newDict = new FastPersistentDictionary(_fastPersistentDictionary.FileLocation); 366 | 367 | foreach (var key in keys) 368 | if (_fastPersistentDictionary.ContainsKey(key)) 369 | newDict.Add(key, _dictionaryAccessor.Get(key)); 370 | 371 | return newDict; 372 | } 373 | } 374 | 375 | public IDictionary Index() 376 | { 377 | lock (_lockObj) 378 | { 379 | IDictionary indexDictionary = new Dictionary(); 380 | var index = 0; 381 | 382 | foreach (var key in _fastPersistentDictionary.DictionarySerializedLookup.Keys) 383 | indexDictionary.Add(key, (_dictionaryAccessor.Get(key), index++)); 384 | 385 | return indexDictionary; 386 | } 387 | } 388 | 389 | public KeyValuePair Last() 390 | { 391 | lock (_lockObj) 392 | { 393 | UpdateDatabaseTick(); 394 | 395 | var lastKey = _fastPersistentDictionary.DictionarySerializedLookup.Keys.LastOrDefault(); 396 | if (lastKey != null) 397 | return new KeyValuePair(lastKey, _dictionaryAccessor.Get(lastKey)); 398 | 399 | throw new InvalidOperationException("No elements in the dictionary."); 400 | } 401 | } 402 | 403 | public KeyValuePair? LastOrDefault() 404 | { 405 | lock (_lockObj) 406 | { 407 | UpdateDatabaseTick(); 408 | 409 | var lastKey = _fastPersistentDictionary.DictionarySerializedLookup.Keys.LastOrDefault(); 410 | if (lastKey != null) 411 | return new KeyValuePair(lastKey, _dictionaryAccessor.GetReturnDefaultIfNotFound(lastKey)); 412 | 413 | return null; 414 | } 415 | } 416 | 417 | public (IEnumerable>, IEnumerable>) Partition(Func, bool> predicate) 418 | { 419 | var trueList = new List>(); 420 | var falseList = new List>(); 421 | 422 | lock (_lockObj) 423 | { 424 | foreach (var key in _fastPersistentDictionary.DictionarySerializedLookup.Keys) 425 | { 426 | var pair = new KeyValuePair(key, _dictionaryAccessor.Get(key)); 427 | if (predicate(pair)) 428 | trueList.Add(pair); 429 | else 430 | falseList.Add(pair); 431 | } 432 | } 433 | return (trueList, falseList); 434 | } 435 | 436 | /// 437 | /// Peek: This method can be used to perform an action on each item in the sequence while iterating without changing the elements in the collection. 438 | /// 439 | /// 440 | public void Peek(Action> action) 441 | { 442 | lock (_lockObj) 443 | { 444 | foreach (var key in _fastPersistentDictionary.DictionarySerializedLookup.Keys) 445 | action(new KeyValuePair(key, _dictionaryAccessor.Get(key))); 446 | } 447 | } 448 | 449 | public IEnumerable Select(Func, TResult> selector) 450 | { 451 | lock (_lockObj) 452 | { 453 | var results = new List(); 454 | 455 | foreach (var key in _fastPersistentDictionary.DictionarySerializedLookup.Keys) 456 | results.Add(selector(new KeyValuePair(key, _dictionaryAccessor.Get(key)))); 457 | 458 | return results; 459 | } 460 | } 461 | 462 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 463 | public KeyValuePair Single(Func, bool> predicate) 464 | { 465 | lock (_lockObj) 466 | { 467 | KeyValuePair foundPair = default; 468 | var found = false; 469 | 470 | foreach (var entry in _fastPersistentDictionary.DictionarySerializedLookup.Keys) 471 | { 472 | var pair = new KeyValuePair(entry, _dictionaryAccessor.Get(entry)); 473 | 474 | if (predicate != null && predicate(pair) == false) 475 | continue; 476 | 477 | if (found) 478 | throw new InvalidOperationException("More than one element satisfies the condition."); 479 | 480 | foundPair = pair; 481 | found = true; 482 | } 483 | 484 | if (found == false) 485 | throw new InvalidOperationException("The source sequence is empty."); 486 | 487 | return foundPair; 488 | } 489 | } 490 | 491 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 492 | public KeyValuePair SingleOrDefault(Func, bool> predicate) 493 | { 494 | lock (_lockObj) 495 | { 496 | KeyValuePair foundPair = default; 497 | var found = false; 498 | 499 | foreach (var entry in _fastPersistentDictionary.DictionarySerializedLookup.Keys) 500 | { 501 | var pair = new KeyValuePair(entry, _dictionaryAccessor.Get(entry)); 502 | 503 | if (predicate != null && predicate(pair) == false) 504 | continue; 505 | 506 | if (found) 507 | return default; 508 | 509 | foundPair = pair; 510 | found = true; 511 | } 512 | 513 | return found ? foundPair : default; 514 | } 515 | } 516 | 517 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 518 | public void UpdateDatabaseTick(bool forceCompact = false) 519 | { 520 | lock (_lockObj) 521 | { 522 | _updateTimer.Stop(); 523 | } 524 | 525 | if (forceCompact) 526 | { 527 | CompactDatabaseFile(); 528 | } 529 | 530 | //if (forceCompact == false && (_canCompact == false || FileStream.Length < NextFileSizeCompact)) 531 | if (FileStream.Length > NextFileSizeCompact) 532 | { 533 | 534 | //Task.Run(async () => 535 | //{ 536 | CompactDatabaseFile(); 537 | // }); 538 | 539 | //_idleCompactTimer.Stop(); 540 | //_idleCompactTimer.Start(); 541 | // return; 542 | } 543 | 544 | 545 | //lock (_lockObj) 546 | //{ 547 | // _updateTimer.Start(); 548 | //} 549 | //if zero percent this means we ONLY compact on the idle timer. 550 | //if (_percentageDiffBeforeCompact != 0) 551 | // CompactDatabaseFile(); 552 | } 553 | 554 | public IEnumerable> Where(Func predicate) 555 | { 556 | lock (_lockObj) 557 | { 558 | foreach (var key in _fastPersistentDictionary.DictionarySerializedLookup.Keys) 559 | { 560 | var value = _dictionaryAccessor.Get(key); 561 | if (predicate(key, value)) 562 | yield return new KeyValuePair(key, value); 563 | } 564 | } 565 | } 566 | } 567 | } -------------------------------------------------------------------------------- /FastPersistentDictionary/Internals/DictionaryStructs.cs: -------------------------------------------------------------------------------- 1 | namespace FastPersistentDictionary.Internals 2 | { 3 | public static class DictionaryStructs 4 | { 5 | public class DictionarySaveHeader 6 | { 7 | public bool UseCompression { get; set; } 8 | public DateTime CreationDate { get; set; } 9 | public DateTime LastModifiedDate { get; set; } 10 | public string Version { get; set; } 11 | public long DictionaryCount { get; set; } 12 | public string KeyType { get; set; } 13 | public string KeyValue { get; set; } 14 | public int ByteLengthOfLookupTable { get; set; } 15 | public string Comment { get; set; } 16 | public string Name { get; set; } 17 | public float PercentageChangeBeforeCompact { get; set; } 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /FastPersistentDictionary/Internals/KeyValueComparer.cs: -------------------------------------------------------------------------------- 1 | namespace FastPersistentDictionary.Internals 2 | { 3 | public class KeyValuePairComparer : IComparer> 4 | { 5 | public int Compare(KeyValuePair x, KeyValuePair y) 6 | { 7 | int result = Comparer.Default.Compare(x.Key, y.Key); 8 | if (result == 0) 9 | { 10 | // If keys are equal, compare by values. 11 | return Comparer.Default.Compare(x.Value, y.Value); 12 | } 13 | else 14 | { 15 | // If keys are not equal, return result. 16 | return result; 17 | } 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 jgric2 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | # Fast Persistent Dictionary 5 | 6 | 7 | 8 | Fast Persistent Dictionary, is an implementation of a persistent dictionary in C#. This dictionary-like data structure is designed to persist data to disk, serving as a reliable workhorse for cases where the application requires storing large amounts of data, but also needs to operate with high efficiency and minimize disk space usage. 9 | 10 | The dictionary comes with a plethora of features and methods, giving the developer the flexibility to manipulate the data as per their requirements. Some of the key features include options to add or update a record, clear all records, load or save the dictionary, or perform common mathematical operations across the dataset. The dictionary also supports compression to help manage disk usage and provides a GROBUF Serializer for data serialization, facilitating efficient storage of complex objects. 11 | 12 | This dictionary makes use of the .NET `IEnumerable` and `ISerializable` interfaces, allowing it to be used in various scenarios where these interfaces are required, for example, in .NET collections or serialization operations. 13 | 14 | The project also provides exception handling and recovery features. A dispose mechanism is incorporated ensuring that all resources are released when the instance is no longer required, enhancing the project’s stability and resilience. 15 | 16 | Fast Persistent Dictionary is an open-source project crafted with care, providing a robust and efficient persistent dictionary in C#. It's an invaluable tool when building applications requiring efficient, reliable, and persistent data storage. 17 | 18 | 19 | ## Fast Persistent Dictionary Features 20 | * **Single File Database**: the In use database and the saved and loadable format is all compiled in a single file. 21 | * **Performance**: Fast Persistent Dictionary supports a high rate of updates and retrieves. Typically surpassing ESENT by a wide margin. 22 | * **Simplicity**: a FastPersistentDictionary looks and behaves like the .NET Dictionary class. No extra method calls are required. 23 | * **Concurrency**: each data structure can be accessed by multiple threads. 24 | * **Scale**: Values can be up to 2gb in size. 25 | * **No Serialization Flags**: Any key or value can be used as long as it is serializable by Grobuf. 26 | 27 | 28 | 29 | 30 | # Benchmarks 31 | ``` 32 | 33 | BenchmarkDotNet v0.14.0, Windows 11 (10.0.22631.4037/23H2/2023Update/SunValley3) 34 | Intel Core i7-10875H CPU 2.30GHz, 1 CPU, 16 logical and 8 physical cores 35 | .NET SDK 8.0.200 36 | [Host] : .NET 8.0.2 (8.0.224.6711), X64 RyuJIT AVX2 [AttachedDebugger] 37 | Job-URLKUQ : .NET 8.0.2 (8.0.224.6711), X64 RyuJIT AVX2 38 | 39 | InvocationCount=1 UnrollFactor=1 40 | 41 | ``` 42 | | Method | N | Mean | Error | StdDev | Median | 43 | |-------------------------------- |-------- |------------------:|---------------:|---------------:|------------------:| 44 | | **FastPersistentDictionary_Add** | **10** | **35.712 μs** | **1.0418 μs** | **2.8869 μs** | **34.700 μs** | 45 | | StandardDictionary_Add | 10 | 4.615 μs | 0.0963 μs | 0.1350 μs | 4.600 μs | 46 | | EsentPersistentDictionary_Add | 10 | 169.886 μs | 5.2269 μs | 14.5704 μs | 164.950 μs | 47 | | FastPersistentDictionary_Get | 10 | 29.895 μs | 0.9572 μs | 2.5550 μs | 30.300 μs | 48 | | StandardDictionary_Get | 10 | 3.429 μs | 0.0810 μs | 0.2312 μs | 3.350 μs | 49 | | EsentPersistentDictionary_Get | 10 | 49.512 μs | 1.2037 μs | 3.2544 μs | 49.300 μs | 50 | | FastPersistentDictionary_Remove | 10 | 4.667 μs | 0.0940 μs | 0.1742 μs | 4.700 μs | 51 | | StandardDictionary_Remove | 10 | 3.397 μs | 0.1454 μs | 0.3981 μs | 3.300 μs | 52 | | EsentPersistentDictionary_Remove | 10 | 85.437 μs | 1.6982 μs | 3.5821 μs | 85.650 μs | 53 | | **FastPersistentDictionary_Add** | **100** | **246.110 μs** | **4.8914 μs** | **7.1697 μs** | **245.000 μs** | 54 | | StandardDictionary_Add | 100 | 20.167 μs | 0.3757 μs | 0.2934 μs | 20.200 μs | 55 | | EsentPersistentDictionary_Add | 100 | 1,387.178 μs | 24.7951 μs | 41.4270 μs | 1,383.100 μs | 56 | | FastPersistentDictionary_Get | 100 | 103.832 μs | 6.6457 μs | 19.1743 μs | 107.050 μs | 57 | | StandardDictionary_Get | 100 | 10.550 μs | 0.2117 μs | 0.3037 μs | 10.500 μs | 58 | | EsentPersistentDictionary_Get | 100 | 376.640 μs | 7.4826 μs | 20.4835 μs | 375.100 μs | 59 | | FastPersistentDictionary_Remove | 100 | 17.328 μs | 0.4785 μs | 1.2689 μs | 17.200 μs | 60 | | StandardDictionary_Remove | 100 | 10.529 μs | 0.2106 μs | 0.3217 μs | 10.600 μs | 61 | | EsentPersistentDictionary_Remove | 100 | 840.953 μs | 16.4776 μs | 30.5424 μs | 839.100 μs | 62 | | **FastPersistentDictionary_Add** | **1000** | **2,308.243 μs** | **44.5800 μs** | **53.0693 μs** | **2,306.800 μs** | 63 | | StandardDictionary_Add | 1000 | 177.095 μs | 3.4542 μs | 3.9779 μs | 175.400 μs | 64 | | EsentPersistentDictionary_Add | 1000 | 12,976.556 μs | 491.4843 μs | 1,441.4385 μs | 12,218.100 μs | 65 | | FastPersistentDictionary_Get | 1000 | 790.672 μs | 32.5535 μs | 94.4435 μs | 814.500 μs | 66 | | StandardDictionary_Get | 1000 | 87.707 μs | 1.2218 μs | 1.0831 μs | 87.500 μs | 67 | | EsentPersistentDictionary_Get | 1000 | 3,355.380 μs | 92.6984 μs | 264.4738 μs | 3,264.600 μs | 68 | | FastPersistentDictionary_Remove | 1000 | 200.779 μs | 5.6056 μs | 16.0834 μs | 196.950 μs | 69 | | StandardDictionary_Remove | 1000 | 106.619 μs | 5.9200 μs | 15.4915 μs | 106.600 μs | 70 | | EsentPersistentDictionary_Remove | 1000 | 7,758.348 μs | 203.6357 μs | 580.9842 μs | 7,583.750 μs | 71 | | **FastPersistentDictionary_Add** | **10000** | **22,342.905 μs** | **910.8584 μs** | **2,671.3902 μs** | **21,158.300 μs** | 72 | | StandardDictionary_Add | 10000 | 1,814.397 μs | 120.8519 μs | 356.3347 μs | 1,636.200 μs | 73 | | EsentPersistentDictionary_Add | 10000 | 122,277.346 μs | 2,166.8262 μs | 4,570.5723 μs | 122,271.200 μs | 74 | | FastPersistentDictionary_Get | 10000 | 4,522.615 μs | 242.8226 μs | 668.8040 μs | 4,239.550 μs | 75 | | StandardDictionary_Get | 10000 | 710.010 μs | 50.2010 μs | 140.7690 μs | 662.100 μs | 76 | | EsentPersistentDictionary_Get | 10000 | 32,145.620 μs | 631.8258 μs | 591.0102 μs | 31,998.800 μs | 77 | | FastPersistentDictionary_Remove | 10000 | 876.048 μs | 73.5746 μs | 215.7816 μs | 742.500 μs | 78 | | StandardDictionary_Remove | 10000 | 869.499 μs | 62.9090 μs | 184.5012 μs | 835.700 μs | 79 | | EsentPersistentDictionary_Remove | 10000 | 76,566.850 μs | 1,515.7748 μs | 1,745.5685 μs | 76,654.200 μs | 80 | | **FastPersistentDictionary_Add** | **100000** | **229,172.487 μs** | **4,581.7610 μs** | **5,794.4611 μs** | **228,866.700 μs** | 81 | | StandardDictionary_Add | 100000 | 32,888.018 μs | 1,342.1441 μs | 3,696.6548 μs | 31,588.850 μs | 82 | | EsentPersistentDictionary_Add | 100000 | 1,249,948.586 μs | 19,460.0175 μs | 17,250.7987 μs | 1,249,580.750 μs | 83 | | FastPersistentDictionary_Get | 100000 | 47,154.111 μs | 884.0972 μs | 1,501.2643 μs | 47,125.200 μs | 84 | | StandardDictionary_Get | 100000 | 13,312.728 μs | 287.6890 μs | 830.0487 μs | 13,184.250 μs | 85 | | EsentPersistentDictionary_Get | 100000 | 328,134.381 μs | 5,873.8948 μs | 4,904.9682 μs | 328,413.050 μs | 86 | | FastPersistentDictionary_Remove | 100000 | 8,773.308 μs | 175.0641 μs | 227.6328 μs | 8,712.500 μs | 87 | | StandardDictionary_Remove | 100000 | 13,193.624 μs | 335.9221 μs | 979.9007 μs | 13,229.400 μs | 88 | | EsentPersistentDictionary_Remove | 100000 | 803,621.169 μs | 15,802.8693 μs | 15,520.5307 μs | 800,826.950 μs | 89 | | **FastPersistentDictionary_Add** | **1000000** | **2,295,026.753 μs** | **35,852.7067 μs** | **33,536.6441 μs** | **2,292,892.900 μs** | 90 | | StandardDictionary_Add | 1000000 | 519,974.638 μs | 14,354.2653 μs | 41,872.0685 μs | 509,290.700 μs | 91 | | EsentPersistentDictionary_Add | 1000000 | 13,038,880.720 μs | 82,744.1640 μs | 77,398.9424 μs | 13,047,758.000 μs | 92 | | FastPersistentDictionary_Get | 1000000 | 493,184.675 μs | 9,822.8597 μs | 11,312.0199 μs | 492,070.800 μs | 93 | | StandardDictionary_Get | 1000000 | 151,635.342 μs | 3,024.3169 μs | 3,361.5202 μs | 152,587.600 μs | 94 | | EsentPersistentDictionary_Get | 1000000 | 3,279,782.257 μs | 55,003.8343 μs | 51,450.6208 μs | 3,301,896.650 μs | 95 | | FastPersistentDictionary_Remove | 1000000 | 112,024.419 μs | 2,202.4345 μs | 3,087.5058 μs | 112,387.200 μs | 96 | | StandardDictionary_Remove | 1000000 | 152,094.443 μs | 2,997.1979 μs | 4,486.0651 μs | 152,306.050 μs | 97 | | EsentPersistentDictionary_Remove | 1000000 | 8,093,507.270 μs | 43,590.9941 μs | 40,775.0430 μs | 8,090,683.550 μs | 98 | 99 | 100 | # Quick Start Demo 101 | 102 | Setting up a FastPersistentDictionary is simple and straightforward and matches the implementation of a standard Dictionary in C# as much as possible with various additional extension methods. 103 | 104 | ``` 105 | int N = 100000; 106 | private FastPersistentDictionary fastPersistentDictionary= new FastPersistentDictionary(path: fastPersistentDictionary); 107 | //Strings for example but can work with any type. 108 | for (int i = 0; i < N; i++) 109 | { 110 | string key = $"Key{i}"; 111 | string value = $"Value{i}"; 112 | FastPersistentDictionary.Add(key, value); 113 | } 114 | 115 | 116 | for (int i = 0; i < N; i++) 117 | { 118 | string key = $"Key{i}"; 119 | string value; 120 | bool found = FastPersistentDictionary.TryGetValue(key, out value); 121 | } 122 | ``` 123 | 124 | 125 | # Table of Contents 126 | - [Project Title](#project-title) 127 | - [Quick Start Demo](#quick-start-demo) 128 | - [Table of Contents](#table-of-contents) 129 | - [Usage](#usage) 130 | - [Contribute](#contribute) 131 | - [License](#license) 132 | 133 | 134 | # Usage 135 | [(Back to top)](#table-of-contents) 136 | 137 | # FastPersistentDictionary 138 | 139 | The `FastPersistentDictionary` is a highly efficient and flexible persistent dictionary for C# that provides a range of dictionary functionalities while ensuring data persistence on disk. Developed by James Grice, this library is designed to be utilized just like any other dictionary, but with the added benefit of persistency and reduced disk space usage. 140 | 141 | ## Installation 142 | 143 | To install the `FastPersistentDictionary` library, you can download it from [GitHub](https://github.com/jgric2/Fast-Persistent-Dictionary) and include it in your project. 144 | 145 | ## Quick Start Guide 146 | 147 | ### 1. Creating a FastPersistentDictionary 148 | 149 | Here's a simple example to create a `FastPersistentDictionary`: 150 | 151 | ```csharp 152 | using FastPersistentDictionary; 153 | 154 | var dictionary = new FastPersistentDictionary("path/to/your/dictionary/file"); 155 | ``` 156 | 157 | ### 2. Adding Entries 158 | 159 | To add entries to the dictionary: 160 | 161 | ```csharp 162 | dictionary.Add("key1", 100); 163 | dictionary.Add("key2", 200); 164 | ``` 165 | 166 | ### 3. Accessing Entries 167 | 168 | The `FastPersistentDictionary` can be accessed like a regular dictionary: 169 | 170 | ```csharp 171 | int value = dictionary["key1"]; 172 | Console.WriteLine(value); // Output: 100 173 | ``` 174 | 175 | ### 4. Updating Entries 176 | 177 | Updating existing entries is straightforward: 178 | 179 | ```csharp 180 | dictionary["key1"] = 150; 181 | ``` 182 | 183 | ### 5. Checking Existence of Keys and Values 184 | 185 | You can check if a key or a value exists in the dictionary: 186 | 187 | ```csharp 188 | bool hasKey = dictionary.ContainsKey("key1"); // true 189 | bool hasValue = dictionary.ContainsValue(200); // true 190 | ``` 191 | 192 | ### 6. Removing Entries 193 | 194 | Removing entries from the dictionary: 195 | 196 | ```csharp 197 | dictionary.Remove("key1"); 198 | ``` 199 | 200 | ### 7. Iterating Over Entries 201 | 202 | You can easily iterate over the entries in the dictionary: 203 | 204 | ```csharp 205 | foreach (var kvp in dictionary) 206 | { 207 | Console.WriteLine($"{kvp.Key} : {kvp.Value}"); 208 | } 209 | ``` 210 | 211 | ### 8. Compacting the Database 212 | 213 | To minimize the file size on the disk manually, you can compact the database: 214 | 215 | ```csharp 216 | dictionary.CompactDatabaseFile(); 217 | ``` 218 | 219 | ### 9. Saving and Loading the Dictionary 220 | 221 | You can save the current state of the dictionary to a file: 222 | 223 | ```csharp 224 | dictionary.SaveDictionary("path/to/save/file"); 225 | ``` 226 | 227 | And you can load an existing dictionary from a file: 228 | 229 | ```csharp 230 | var loadedDictionary = new FastPersistentDictionary("path/to/save/file"); 231 | loadedDictionary.LoadDictionary("path/to/save/file"); 232 | ``` 233 | 234 | ### 10. Disposing the Dictionary 235 | 236 | Make sure to dispose of the dictionary properly when it is no longer needed: 237 | 238 | ```csharp 239 | dictionary.Dispose(); 240 | ``` 241 | 242 | ## Advanced Features 243 | 244 | ### 1. Bulk Operations 245 | 246 | You can perform bulk-read operations: 247 | 248 | ```csharp 249 | var keys = new[] { "key1", "key2" }; 250 | var values = dictionary.GetBulk(keys); 251 | ``` 252 | 253 | ### 2. Mathematical Operations 254 | 255 | The `FastPersistentDictionary` supports mathematical operations: 256 | 257 | ```csharp 258 | int min = dictionary.Min(); 259 | int sum = dictionary.Sum(); 260 | int max = dictionary.Max(); 261 | double average = dictionary.Average(kvp => kvp.Value); 262 | ``` 263 | 264 | ### 3. Query and Filter 265 | 266 | Advanced querying, filtering, and extracting subsets of the dictionary: 267 | 268 | ```csharp 269 | var subset = dictionary.GetSubset((key, value) => value > 100); 270 | 271 | bool any = dictionary.Any(kvp => kvp.Value > 150); 272 | bool all = dictionary.All(kvp => kvp.Value > 50); 273 | ``` 274 | 275 | ### 4. Importing Other Dictionaries 276 | 277 | Import entries from another dictionary: 278 | 279 | ```csharp 280 | var anotherDictionary = new Dictionary 281 | { 282 | { "key3", 300 }, 283 | { "key4", 400 } 284 | }; 285 | dictionary.Merge(anotherDictionary); 286 | ``` 287 | 288 | ## Summary 289 | 290 | The `FastPersistentDictionary` offers a wide range of functionalities for persistent data storage with minimal overhead on disk space. With the ease of use of any other dictionary and options for advanced operations, it is a powerful tool for large datasets and projects in C#. 291 | 292 | 293 | 294 | # Contribute 295 | [(Back to top)](#table-of-contents) 296 | 297 | Your support is invaluable and truly welcomed! Here's how you can contribute: 298 | 299 | - **Write Tests and Benchmarks**: Enhance code reliability and performance evaluation. 300 | - **Enhance Documentation**: Assist others in comprehending and effectively using FastPersistentDictionary. 301 | - **Submit Feature Requests and Bug Reports**: Suggest new ideas and report issues to help refine FastPersistentDictionary. 302 | - **Optimize Performance**: Offer optimizations and improvements to existing features.. 303 | 304 | 305 | # License 306 | [(Back to top)](#table-of-contents) 307 | 308 | [MIT license](./LICENSE) 309 | 310 | 311 | --------------------------------------------------------------------------------