├── .gitignore ├── DataMangler.gallio ├── Tests ├── Properties │ └── AssemblyInfo.cs ├── Tests.csproj ├── SerializationTests.cs ├── IndexTests.cs └── BasicTests.cs ├── Properties └── AssemblyInfo.cs ├── PropertySerializer.cs ├── README.md ├── DataMangler.csproj ├── DataStructures.cs ├── DataMangler.sln ├── StreamCollection.cs ├── Interfaces.cs ├── Helpers.cs ├── Util.cs ├── TangleKey.cs ├── Index.cs ├── Serialization.cs ├── StreamRef.cs ├── LICENSE └── Tangle.cs /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | *.user 3 | *.suo 4 | Tests/bin 5 | Tests/obj 6 | bin 7 | obj 8 | _ReSharper* -------------------------------------------------------------------------------- /DataMangler.gallio: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Tests\bin\Debug\Tests.dll 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | ..\..\..\AppData\Roaming\Gallio\Icarus\Reports 17 | test-report-{0}-{1} 18 | -------------------------------------------------------------------------------- /Tests/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("Tests")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("Tests")] 13 | [assembly: AssemblyCopyright("")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("4f9f6b28-43c3-4957-a5c9-2eb4b810a586")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("DataMangler")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("DataMangler")] 13 | [assembly: AssemblyCopyright("")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("a1c5f60a-0ec1-47cd-b710-24640858de43")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /PropertySerializer.cs: -------------------------------------------------------------------------------- 1 | /* 2 | The contents of this file are subject to the Mozilla Public License 3 | Version 1.1 (the "License"); you may not use this file except in 4 | compliance with the License. You may obtain a copy of the License at 5 | http://www.mozilla.org/MPL/ 6 | 7 | Software distributed under the License is distributed on an "AS IS" 8 | basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the 9 | License for the specific language governing rights and limitations 10 | under the License. 11 | 12 | The Original Code is DataMangler Key-Value Store. 13 | 14 | The Initial Developer of the Original Code is Mozilla Corporation. 15 | 16 | Original Author: Kevin Gadd (kevin.gadd@gmail.com) 17 | */ 18 | 19 | using System; 20 | using System.Collections.Generic; 21 | using System.Linq; 22 | using System.Text; 23 | using Squared.Task.Data; 24 | using Squared.Util.Bind; 25 | 26 | namespace Squared.Data.Mangler { 27 | public class TanglePropertySerializer : PropertySerializerBase { 28 | public readonly Tangle Tangle; 29 | 30 | public TanglePropertySerializer ( 31 | Tangle tangle 32 | ) : this (tangle, GetDefaultMemberName) { 33 | } 34 | 35 | public TanglePropertySerializer ( 36 | Tangle tangle, Func getMemberName 37 | ) : base(getMemberName) { 38 | Tangle = tangle; 39 | } 40 | 41 | protected override IEnumerator SaveBinding (string name, BoundMember member) { 42 | yield return Tangle.Set(name, member.Value); 43 | } 44 | 45 | protected override IEnumerator LoadBinding (string name, BoundMember member) { 46 | var fValue = Tangle.Get(name); 47 | yield return fValue; 48 | 49 | if (!fValue.Failed) 50 | member.Value = (T)fValue.Result; 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Data Mangler 2 | ============ 3 | 4 | What is it? 5 | ----------- 6 | 7 | It's a persistent embedded key-value store for .NET. 8 | 9 | Wait, really? Why? 10 | ------------------ 11 | 12 | I needed a fast storage solution for a data-crunching application I was writing. I tried SQLite, but it was way too slow. All of the key-value stores out there were either under incompatible open-source licenses or required you to set up a server and send queries over the network. 13 | 14 | How is it different? 15 | -------------------- 16 | It uses memory-mapped files instead of handling disk I/O and caching itself. This ends up being a pretty good choice because while it's possible to do caching better than the kernel, it takes a lot more time and knowledge than most programmers have at their disposal. This also means you don't have to tune caching parameters to avoid running out of memory; the kernel figures out when to throw out cached pages. 17 | 18 | It's designed to be embedded in a .NET application. The overhead of marshalling requests over the network to your storage server is gone, and so is the overhead of converting types from the .NET type system to whatever type system your storage engine uses. 19 | 20 | It's type-safe. Each Tangle you create can only store values of a specific type (known at creation time) which means that you get the full benefits of the .NET type system when interacting with the datastore. In those edge cases where you really do need variants, you can still get them - just store objects. 21 | 22 | Keys are arbitrary sequences of bytes instead of strings. So if you want to use an integer or a float as a key, you can. 23 | 24 | Read operations against a Tangle are executed in parallel and you can interact with multiple Tangles at once safely because they use separate storage streams. Write operations are sequential and cannot occur at the same time as read operations. This makes it a bad fit for servers with thousands of users, but the lack of locking and synchronization overhead means it is blazing fast for single-user cases, and it still makes great use of multiple cores. 25 | 26 | Your dataset doesn't have to fit in memory. Despite using memory-mapped files, it only maps reasonably-sized chunks of the database into memory at any given time, so you can use it as a part of a larger application without completely exhausting your address space even if you're storing gigabytes of data. -------------------------------------------------------------------------------- /Tests/Tests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Debug 5 | AnyCPU 6 | 8.0.30703 7 | 2.0 8 | {D8C068B2-B856-4ACB-8F2C-5F1BF2AEE03F} 9 | Library 10 | Properties 11 | Squared.Data.Mangler.Tests 12 | Tests 13 | v4.0 14 | 512 15 | 16 | 17 | true 18 | full 19 | false 20 | bin\Debug\ 21 | DEBUG;TRACE 22 | prompt 23 | 4 24 | true 25 | 26 | 27 | pdbonly 28 | true 29 | bin\Release\ 30 | TRACE 31 | prompt 32 | 4 33 | true 34 | 35 | 36 | 37 | ..\..\Fracture\Ext\nunit.framework.dll 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | {173ABDCA-7278-46FF-A300-D3BF8CCEF181} 52 | Squared.Task 53 | 54 | 55 | {D7F549CF-E0A6-491C-A78C-ECAB590BB2A7} 56 | Squared.Util 57 | 58 | 59 | {DC905B32-07FA-4161-9627-C8AF6E6241EE} 60 | DataMangler 61 | 62 | 63 | 64 | 71 | -------------------------------------------------------------------------------- /DataMangler.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Debug 5 | AnyCPU 6 | 8.0.30703 7 | 2.0 8 | {DC905B32-07FA-4161-9627-C8AF6E6241EE} 9 | Library 10 | Properties 11 | Squared.Data.Mangler 12 | DataMangler 13 | v4.0 14 | 512 15 | 16 | 17 | 18 | true 19 | full 20 | false 21 | bin\Debug\ 22 | DEBUG;TRACE 23 | prompt 24 | 4 25 | true 26 | 27 | 28 | pdbonly 29 | true 30 | bin\Release\ 31 | TRACE 32 | prompt 33 | 4 34 | true 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | {173ABDCA-7278-46FF-A300-D3BF8CCEF181} 60 | Squared.Task 61 | 62 | 63 | {D7F549CF-E0A6-491C-A78C-ECAB590BB2A7} 64 | Squared.Util 65 | 66 | 67 | 68 | 75 | -------------------------------------------------------------------------------- /DataStructures.cs: -------------------------------------------------------------------------------- 1 | /* 2 | The contents of this file are subject to the Mozilla Public License 3 | Version 1.1 (the "License"); you may not use this file except in 4 | compliance with the License. You may obtain a copy of the License at 5 | http://www.mozilla.org/MPL/ 6 | 7 | Software distributed under the License is distributed on an "AS IS" 8 | basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the 9 | License for the specific language governing rights and limitations 10 | under the License. 11 | 12 | The Original Code is DataMangler Key-Value Store. 13 | 14 | The Initial Developer of the Original Code is Mozilla Corporation. 15 | 16 | Original Author: Kevin Gadd (kevin.gadd@gmail.com) 17 | */ 18 | 19 | using System; 20 | using System.Runtime.InteropServices; 21 | using System.IO; 22 | 23 | namespace Squared.Data.Mangler.Internal { 24 | [StructLayout(LayoutKind.Sequential, Pack = 1, Size = 128)] 25 | internal unsafe struct BTreeHeader { 26 | public static readonly uint Size; 27 | 28 | public long RootIndex; 29 | public long WastedDataBytes; 30 | public long ItemCount; 31 | public long MutationSentinel; 32 | public long FreelistIndexOffset; 33 | 34 | static BTreeHeader () { 35 | Size = (uint)Marshal.SizeOf(typeof(BTreeHeader)); 36 | } 37 | } 38 | 39 | [StructLayout(LayoutKind.Sequential, Pack = 1)] 40 | internal unsafe struct BTreeNode { 41 | public static readonly uint Size; 42 | public static readonly uint TotalSize; 43 | public static readonly uint OffsetOfValues; 44 | public static readonly uint OffsetOfLeaves; 45 | 46 | public const int T = 36; 47 | public const int MaxValues = (2 * T) - 1; 48 | public const int MaxLeaves = MaxValues + 1; 49 | 50 | static BTreeNode () { 51 | if (T % 2 != 0) 52 | throw new InvalidDataException(); 53 | 54 | Size = (uint)Marshal.SizeOf(typeof(BTreeNode)); 55 | TotalSize = Size + (MaxValues * BTreeValue.Size) + (MaxLeaves * BTreeLeaf.Size); 56 | 57 | // Round nodes up to a size of 2KB 58 | if (TotalSize > 2048) 59 | throw new InvalidDataException(); 60 | TotalSize = 2048; 61 | 62 | OffsetOfValues = Size; 63 | OffsetOfLeaves = OffsetOfValues + (MaxValues * BTreeValue.Size); 64 | } 65 | 66 | public byte IsValid; 67 | public byte HasLeaves; 68 | public ushort NumValues; 69 | } 70 | 71 | [StructLayout(LayoutKind.Sequential, Pack = 1, Size = 4)] 72 | internal unsafe struct BTreeLeaf { 73 | public static readonly uint Size; 74 | 75 | static BTreeLeaf () { 76 | Size = (uint)Marshal.SizeOf(typeof(BTreeLeaf)); 77 | } 78 | 79 | public uint NodeIndex; 80 | 81 | public BTreeLeaf (long nodeIndex) { 82 | NodeIndex = (uint)nodeIndex; 83 | } 84 | } 85 | 86 | [StructLayout(LayoutKind.Sequential, Pack = 1)] 87 | internal unsafe struct BTreeValue { 88 | public const int KeyPrefixSize = 4; 89 | public static readonly uint Size; 90 | 91 | static BTreeValue () { 92 | Size = (uint)Marshal.SizeOf(typeof(BTreeValue)); 93 | } 94 | 95 | public uint DataOffset; 96 | public uint DataLength; 97 | public uint ExtraDataBytes; 98 | public uint KeyOffset; 99 | public ushort KeyLength; 100 | public ushort KeyType; 101 | public fixed byte KeyPrefix[KeyPrefixSize]; 102 | } 103 | 104 | [StructLayout(LayoutKind.Sequential, Pack = 1, Size = 256)] 105 | internal unsafe struct FreelistIndex { 106 | public const int FirstPower = 4; 107 | public const int BucketCount = 16; 108 | public const int BucketSize = 2; 109 | 110 | public static readonly uint Size; 111 | 112 | static FreelistIndex () { 113 | Size = (uint)Marshal.SizeOf(typeof(FreelistIndex)); 114 | } 115 | 116 | public fixed uint HeadOffsets[BucketCount]; 117 | } 118 | 119 | [StructLayout(LayoutKind.Sequential, Pack = 1)] 120 | internal unsafe struct FreelistNode { 121 | public static readonly uint Size; 122 | 123 | static FreelistNode () { 124 | Size = (uint)Marshal.SizeOf(typeof(FreelistNode)); 125 | } 126 | 127 | public uint BlockSize; 128 | public uint NextBlockOffset; 129 | } 130 | } -------------------------------------------------------------------------------- /DataMangler.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 11.00 3 | # Visual Studio 2010 4 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tests", "Tests\Tests.csproj", "{D8C068B2-B856-4ACB-8F2C-5F1BF2AEE03F}" 5 | EndProject 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataMangler", "DataMangler.csproj", "{DC905B32-07FA-4161-9627-C8AF6E6241EE}" 7 | EndProject 8 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Squared.Task", "..\Fracture\Squared\TaskLib\Squared.Task.csproj", "{173ABDCA-7278-46FF-A300-D3BF8CCEF181}" 9 | EndProject 10 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Squared.Util", "..\Fracture\Squared\Util\Squared.Util.csproj", "{D7F549CF-E0A6-491C-A78C-ECAB590BB2A7}" 11 | EndProject 12 | Global 13 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 14 | Debug|Any CPU = Debug|Any CPU 15 | Debug|x86 = Debug|x86 16 | Mono|Any CPU = Mono|Any CPU 17 | Mono|x86 = Mono|x86 18 | Release|Any CPU = Release|Any CPU 19 | Release|x86 = Release|x86 20 | EndGlobalSection 21 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 22 | {D8C068B2-B856-4ACB-8F2C-5F1BF2AEE03F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 23 | {D8C068B2-B856-4ACB-8F2C-5F1BF2AEE03F}.Debug|Any CPU.Build.0 = Debug|Any CPU 24 | {D8C068B2-B856-4ACB-8F2C-5F1BF2AEE03F}.Debug|x86.ActiveCfg = Debug|Any CPU 25 | {D8C068B2-B856-4ACB-8F2C-5F1BF2AEE03F}.Mono|Any CPU.ActiveCfg = Release|Any CPU 26 | {D8C068B2-B856-4ACB-8F2C-5F1BF2AEE03F}.Mono|Any CPU.Build.0 = Release|Any CPU 27 | {D8C068B2-B856-4ACB-8F2C-5F1BF2AEE03F}.Mono|x86.ActiveCfg = Release|Any CPU 28 | {D8C068B2-B856-4ACB-8F2C-5F1BF2AEE03F}.Release|Any CPU.ActiveCfg = Release|Any CPU 29 | {D8C068B2-B856-4ACB-8F2C-5F1BF2AEE03F}.Release|Any CPU.Build.0 = Release|Any CPU 30 | {D8C068B2-B856-4ACB-8F2C-5F1BF2AEE03F}.Release|x86.ActiveCfg = Release|Any CPU 31 | {DC905B32-07FA-4161-9627-C8AF6E6241EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 32 | {DC905B32-07FA-4161-9627-C8AF6E6241EE}.Debug|Any CPU.Build.0 = Debug|Any CPU 33 | {DC905B32-07FA-4161-9627-C8AF6E6241EE}.Debug|x86.ActiveCfg = Debug|Any CPU 34 | {DC905B32-07FA-4161-9627-C8AF6E6241EE}.Mono|Any CPU.ActiveCfg = Release|Any CPU 35 | {DC905B32-07FA-4161-9627-C8AF6E6241EE}.Mono|Any CPU.Build.0 = Release|Any CPU 36 | {DC905B32-07FA-4161-9627-C8AF6E6241EE}.Mono|x86.ActiveCfg = Release|Any CPU 37 | {DC905B32-07FA-4161-9627-C8AF6E6241EE}.Release|Any CPU.ActiveCfg = Release|Any CPU 38 | {DC905B32-07FA-4161-9627-C8AF6E6241EE}.Release|Any CPU.Build.0 = Release|Any CPU 39 | {DC905B32-07FA-4161-9627-C8AF6E6241EE}.Release|x86.ActiveCfg = Release|Any CPU 40 | {173ABDCA-7278-46FF-A300-D3BF8CCEF181}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 41 | {173ABDCA-7278-46FF-A300-D3BF8CCEF181}.Debug|Any CPU.Build.0 = Debug|Any CPU 42 | {173ABDCA-7278-46FF-A300-D3BF8CCEF181}.Debug|x86.ActiveCfg = Debug|x86 43 | {173ABDCA-7278-46FF-A300-D3BF8CCEF181}.Debug|x86.Build.0 = Debug|x86 44 | {173ABDCA-7278-46FF-A300-D3BF8CCEF181}.Mono|Any CPU.ActiveCfg = Mono|Any CPU 45 | {173ABDCA-7278-46FF-A300-D3BF8CCEF181}.Mono|Any CPU.Build.0 = Mono|Any CPU 46 | {173ABDCA-7278-46FF-A300-D3BF8CCEF181}.Mono|x86.ActiveCfg = Mono|x86 47 | {173ABDCA-7278-46FF-A300-D3BF8CCEF181}.Mono|x86.Build.0 = Mono|x86 48 | {173ABDCA-7278-46FF-A300-D3BF8CCEF181}.Release|Any CPU.ActiveCfg = Release|Any CPU 49 | {173ABDCA-7278-46FF-A300-D3BF8CCEF181}.Release|Any CPU.Build.0 = Release|Any CPU 50 | {173ABDCA-7278-46FF-A300-D3BF8CCEF181}.Release|x86.ActiveCfg = Release|x86 51 | {173ABDCA-7278-46FF-A300-D3BF8CCEF181}.Release|x86.Build.0 = Release|x86 52 | {D7F549CF-E0A6-491C-A78C-ECAB590BB2A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 53 | {D7F549CF-E0A6-491C-A78C-ECAB590BB2A7}.Debug|Any CPU.Build.0 = Debug|Any CPU 54 | {D7F549CF-E0A6-491C-A78C-ECAB590BB2A7}.Debug|x86.ActiveCfg = Debug|x86 55 | {D7F549CF-E0A6-491C-A78C-ECAB590BB2A7}.Debug|x86.Build.0 = Debug|x86 56 | {D7F549CF-E0A6-491C-A78C-ECAB590BB2A7}.Mono|Any CPU.ActiveCfg = Mono|Any CPU 57 | {D7F549CF-E0A6-491C-A78C-ECAB590BB2A7}.Mono|Any CPU.Build.0 = Mono|Any CPU 58 | {D7F549CF-E0A6-491C-A78C-ECAB590BB2A7}.Mono|x86.ActiveCfg = Mono|x86 59 | {D7F549CF-E0A6-491C-A78C-ECAB590BB2A7}.Mono|x86.Build.0 = Mono|x86 60 | {D7F549CF-E0A6-491C-A78C-ECAB590BB2A7}.Release|Any CPU.ActiveCfg = Release|Any CPU 61 | {D7F549CF-E0A6-491C-A78C-ECAB590BB2A7}.Release|Any CPU.Build.0 = Release|Any CPU 62 | {D7F549CF-E0A6-491C-A78C-ECAB590BB2A7}.Release|x86.ActiveCfg = Release|x86 63 | {D7F549CF-E0A6-491C-A78C-ECAB590BB2A7}.Release|x86.Build.0 = Release|x86 64 | EndGlobalSection 65 | GlobalSection(SolutionProperties) = preSolution 66 | HideSolutionNode = FALSE 67 | EndGlobalSection 68 | EndGlobal 69 | -------------------------------------------------------------------------------- /StreamCollection.cs: -------------------------------------------------------------------------------- 1 | /* 2 | The contents of this file are subject to the Mozilla Public License 3 | Version 1.1 (the "License"); you may not use this file except in 4 | compliance with the License. You may obtain a copy of the License at 5 | http://www.mozilla.org/MPL/ 6 | 7 | Software distributed under the License is distributed on an "AS IS" 8 | basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the 9 | License for the specific language governing rights and limitations 10 | under the License. 11 | 12 | The Original Code is DataMangler Key-Value Store. 13 | 14 | The Initial Developer of the Original Code is Mozilla Corporation. 15 | 16 | Original Author: Kevin Gadd (kevin.gadd@gmail.com) 17 | */ 18 | 19 | using System; 20 | using System.Collections.Generic; 21 | using System.IO; 22 | using System.Linq; 23 | using System.Text; 24 | 25 | namespace Squared.Data.Mangler { 26 | public abstract class StreamSource : IDisposable { 27 | abstract internal Internal.StreamRef Open (string streamName); 28 | 29 | abstract public void Dispose (); 30 | } 31 | 32 | public abstract class CachingStreamSourceBase : StreamSource { 33 | protected readonly Dictionary Streams = new Dictionary(); 34 | 35 | override internal Internal.StreamRef Open (string streamName) { 36 | FileStream result; 37 | 38 | if (!Streams.TryGetValue(streamName, out result)) { 39 | result = OpenStream(streamName); 40 | Streams[streamName] = result; 41 | } 42 | 43 | return new Internal.StreamRef(result, false); 44 | } 45 | 46 | protected abstract FileStream OpenStream (string streamName); 47 | 48 | override public void Dispose () { 49 | foreach (var stream in Streams.Values) 50 | stream.Dispose(); 51 | Streams.Clear(); 52 | } 53 | } 54 | 55 | public class SubStreamSource : StreamSource { 56 | private readonly StreamSource Inner; 57 | public readonly string Prefix; 58 | public readonly bool OwnsInnerSource; 59 | 60 | public SubStreamSource (StreamSource inner, string prefix, bool ownsInnerSource=true) { 61 | Inner = inner; 62 | Prefix = prefix; 63 | OwnsInnerSource = ownsInnerSource; 64 | } 65 | 66 | internal override Internal.StreamRef Open (string streamName) { 67 | return Inner.Open(Prefix + streamName); 68 | } 69 | 70 | public override void Dispose () { 71 | if (OwnsInnerSource) 72 | Inner.Dispose(); 73 | } 74 | } 75 | 76 | public class AlternateStreamSource : CachingStreamSourceBase { 77 | public readonly string Filename; 78 | 79 | public AlternateStreamSource (string filename) { 80 | Filename = filename; 81 | } 82 | 83 | protected override FileStream OpenStream (string streamName) { 84 | return Internal.Native.OpenAlternateStream(Filename, streamName); 85 | } 86 | } 87 | 88 | public class FolderStreamSource : CachingStreamSourceBase { 89 | private string _Folder; 90 | 91 | public FolderStreamSource (string folder) { 92 | _Folder = folder; 93 | } 94 | 95 | protected static string GetPath (string folder, string streamName) { 96 | return Path.Combine(folder, streamName); 97 | } 98 | 99 | protected override FileStream OpenStream (string streamName) { 100 | var path = GetPath(_Folder, streamName); 101 | return File.Open(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Delete | FileShare.ReadWrite); 102 | } 103 | 104 | public string Folder { 105 | get { 106 | return _Folder; 107 | } 108 | set { 109 | if (value == _Folder) 110 | return; 111 | 112 | var oldFolder = _Folder; 113 | 114 | Dispose(); 115 | 116 | Directory.CreateDirectory(value); 117 | 118 | foreach (var filename in Directory.GetFiles(oldFolder)) { 119 | var newFilename = Path.Combine(value, Path.GetFileName(filename)); 120 | if (File.Exists(newFilename)) 121 | File.Delete(newFilename); 122 | 123 | File.Move(filename, newFilename); 124 | } 125 | 126 | _Folder = value; 127 | 128 | Directory.Delete(oldFolder, true); 129 | } 130 | } 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /Tests/SerializationTests.cs: -------------------------------------------------------------------------------- 1 | /* 2 | The contents of this file are subject to the Mozilla Public License 3 | Version 1.1 (the "License"); you may not use this file except in 4 | compliance with the License. You may obtain a copy of the License at 5 | http://www.mozilla.org/MPL/ 6 | 7 | Software distributed under the License is distributed on an "AS IS" 8 | basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the 9 | License for the specific language governing rights and limitations 10 | under the License. 11 | 12 | The Original Code is DataMangler Key-Value Store. 13 | 14 | The Initial Developer of the Original Code is Mozilla Corporation. 15 | 16 | Original Author: Kevin Gadd (kevin.gadd@gmail.com) 17 | */ 18 | 19 | using System; 20 | using System.Collections.Generic; 21 | using System.Linq; 22 | using System.Text; 23 | using NUnit.Framework; 24 | using System.IO; 25 | using Squared.Task; 26 | 27 | namespace Squared.Data.Mangler.Tests { 28 | public struct SpecialType { 29 | public readonly TangleKey Key; 30 | public readonly UInt32 Value; 31 | 32 | public SpecialType (TangleKey key, UInt32 value) { 33 | Key = key; 34 | Value = value; 35 | } 36 | 37 | [TangleDeserializer] 38 | static void Deserialize (ref DeserializationContext context, out SpecialType output) { 39 | var br = new BinaryReader(context.Stream); 40 | output = new SpecialType(context.Key, br.ReadUInt32()); 41 | } 42 | 43 | [TangleSerializer] 44 | static void Serialize (ref SerializationContext context, ref SpecialType input) { 45 | if (!context.Key.Equals(input.Key)) 46 | throw new InvalidDataException(); 47 | 48 | var bw = new BinaryWriter(context.Stream); 49 | bw.Write(input.Value); 50 | bw.Flush(); 51 | } 52 | } 53 | 54 | [TestFixture] 55 | public class SerializationTests : BasicTestFixture { 56 | public Tangle Tangle; 57 | 58 | [SetUp] 59 | public override void SetUp () { 60 | base.SetUp(); 61 | 62 | Tangle = new Tangle( 63 | Scheduler, Storage, 64 | ownsStorage: true 65 | ); 66 | } 67 | 68 | [TearDown] 69 | public override void TearDown () { 70 | Tangle.Dispose(); 71 | base.TearDown(); 72 | } 73 | 74 | [Test] 75 | public void UsesStaticSerializerAndDeserializerMethodsAutomatically () { 76 | var key = new TangleKey("hello"); 77 | Scheduler.WaitFor(Tangle.Set(key, new SpecialType(key, 4))); 78 | var result = Scheduler.WaitFor(Tangle.Get("hello")); 79 | Assert.AreEqual(4, result.Value); 80 | } 81 | 82 | [Test] 83 | public void SerializerAndDeserializerHaveAccessToKey () { 84 | var key = new TangleKey("hello"); 85 | 86 | // This will fail because the specified keys don't match, and that lets us know 87 | // that the serializer had access to the key. Kind of a hack. 88 | try { 89 | Scheduler.WaitFor(Tangle.Set("world", new SpecialType(key, 4))); 90 | } catch (FutureException fe) { 91 | Assert.IsInstanceOf(fe.InnerException); 92 | Assert.IsInstanceOf(fe.InnerException.InnerException); 93 | } 94 | 95 | // As a side effect, this also tests the Tangle's ability to recover from 96 | // a failed serialization. If an exception from the Serializer were to bubble 97 | // up, the BTree would be left in an invalid state and this set would fail. 98 | Scheduler.WaitFor(Tangle.Set(key, new SpecialType(key, 4))); 99 | 100 | var result = Scheduler.WaitFor(Tangle.Get("hello")); 101 | Assert.IsTrue(key.Equals(result.Key)); 102 | } 103 | } 104 | 105 | [TestFixture] 106 | public class PropertySerializerTests : BasicTestFixture { 107 | public class ClassWithProperties { 108 | public int A; 109 | public int B { 110 | get; 111 | set; 112 | } 113 | public string C; 114 | } 115 | 116 | public Tangle Tangle; 117 | public TanglePropertySerializer Serializer; 118 | 119 | [SetUp] 120 | public override void SetUp () { 121 | base.SetUp(); 122 | Tangle = new Tangle(Scheduler, Storage); 123 | Serializer = new TanglePropertySerializer(Tangle); 124 | } 125 | 126 | [TearDown] 127 | public override void TearDown () { 128 | Tangle.Dispose(); 129 | base.TearDown(); 130 | } 131 | 132 | [Test] 133 | public void TestSerializesProperties () { 134 | var instance = new ClassWithProperties { 135 | A = 1, 136 | B = 2, 137 | C = "foo" 138 | }; 139 | 140 | Serializer.Bind(() => instance.A); 141 | Serializer.Bind(() => instance.B); 142 | Serializer.Bind(() => instance.C); 143 | 144 | Scheduler.WaitFor(Serializer.Save()); 145 | 146 | Assert.AreEqual(1, (int)Scheduler.WaitFor(Tangle.Get("A"))); 147 | Assert.AreEqual(2, (int)Scheduler.WaitFor(Tangle.Get("B"))); 148 | Assert.AreEqual("foo", (string)Scheduler.WaitFor(Tangle.Get("C"))); 149 | } 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /Interfaces.cs: -------------------------------------------------------------------------------- 1 | /* 2 | The contents of this file are subject to the Mozilla Public License 3 | Version 1.1 (the "License"); you may not use this file except in 4 | compliance with the License. You may obtain a copy of the License at 5 | http://www.mozilla.org/MPL/ 6 | 7 | Software distributed under the License is distributed on an "AS IS" 8 | basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the 9 | License for the specific language governing rights and limitations 10 | under the License. 11 | 12 | The Original Code is DataMangler Key-Value Store. 13 | 14 | The Initial Developer of the Original Code is Mozilla Corporation. 15 | 16 | Original Author: Kevin Gadd (kevin.gadd@gmail.com) 17 | */ 18 | 19 | using System; 20 | using System.Collections.Generic; 21 | using System.Linq; 22 | using System.Text; 23 | using Squared.Task; 24 | 25 | namespace Squared.Data.Mangler { 26 | public interface ITangle : IDisposable { 27 | /// 28 | /// Creates a barrier and inserts it into the tangle's work queue. 29 | /// The barrier is signaled when reached and prevents work items following it from being executed as long as it remains closed. 30 | /// 31 | /// If true, the barrier is created opened. 32 | /// The created barrier. 33 | IBarrier CreateBarrier (bool createOpened); 34 | 35 | /// 36 | /// Retrieves a value from the tangle, looking it up via the specified key. 37 | /// 38 | /// The key of the value to retrieve. 39 | /// The retrieved value. 40 | IFuture Get (TangleKey key); 41 | 42 | /// 43 | /// Retrieves all the values stored within the tangle, in no particular order. 44 | /// 45 | /// The values stored within the tangle, as an array of type T[]. 46 | IFuture GetAllValues (); 47 | 48 | /// 49 | /// Retrieves the keys of all the values stored within the tangle, in no particular order. 50 | /// 51 | /// The keys of the tangle's values. 52 | Future GetAllKeys (); 53 | 54 | /// 55 | /// The number of values stored within the tangle. 56 | /// 57 | long Count { 58 | get; 59 | } 60 | } 61 | 62 | public interface IBarrier : ISchedulable, IDisposable { 63 | /// 64 | /// Opens the barrier, allowing work items later in the queue to be executed. 65 | /// 66 | void Open (); 67 | 68 | /// 69 | /// Returns a future that becomes completed once the barrier has been reached, regardless of whether the barrier is open or closed. 70 | /// 71 | IFuture Future { 72 | get; 73 | } 74 | } 75 | 76 | public interface IBatch { 77 | Future Execute (); 78 | } 79 | 80 | /// 81 | /// Called to update a value within a tangle. 82 | /// 83 | /// The current value of the item. 84 | /// The new value of the item. 85 | public delegate T UpdateCallback (T oldValue); 86 | 87 | /// 88 | /// Called to update a value within a tangle. 89 | /// 90 | /// The current value of the item. 91 | /// The new value of the item. By default, this is the value provided when calling AddOrUpdate. 92 | /// True to update the item's value, false to abort. 93 | public delegate bool DecisionUpdateCallback (ref T oldValue, ref T newValue); 94 | 95 | /// 96 | /// Called to generate the right side key for a join. 97 | /// 98 | /// The type of the left side key(s). 99 | /// The type of the left side value(s). 100 | /// The type of the right side key(s). 101 | /// The left side key. 102 | /// The left side value. 103 | /// The right side key. 104 | public delegate TRightKey JoinKeySelector 105 | (TLeftKey leftKey, ref TLeft leftValue); 106 | 107 | /// 108 | /// Called to generate the result of a join. 109 | /// 110 | /// The type of the left side key(s). 111 | /// The type of the left side value(s). 112 | /// The type of the right side key(s). 113 | /// The type of the right side value(s). 114 | /// The type of the result value(s). 115 | /// The left side key. 116 | /// The left side value. 117 | /// The right side key. 118 | /// The right side value. 119 | /// The join result. 120 | public delegate TOut JoinValueSelector 121 | (TLeftKey leftKey, ref TLeft leftValue, TRightKey rightKey, ref TRight rightValue); 122 | 123 | public delegate TIndexKey IndexFunc (ref TValue value); 124 | 125 | // Making the argument not ref eliminates ambiguity between IndexFunc/IndexMultipleFunc, and 126 | // it's helpful anyway since generator functions can't have ref parameters 127 | public delegate IEnumerable IndexMultipleFunc (TValue value); 128 | 129 | public delegate TangleKey TangleKeyConverter (TValue value); 130 | } 131 | 132 | namespace Squared.Data.Mangler.Internal { 133 | public interface IWorkItem : IDisposable { 134 | void Execute (Tangle tangle); 135 | } 136 | 137 | public interface IWorkItemWithFuture : IWorkItem { 138 | Future Future { 139 | get; 140 | } 141 | } 142 | 143 | internal interface IReplaceCallback { 144 | bool ShouldReplace (Tangle tangle, ref BTreeValue btreeValue, ushort keyType, ref T newValue); 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /Helpers.cs: -------------------------------------------------------------------------------- 1 | /* 2 | The contents of this file are subject to the Mozilla Public License 3 | Version 1.1 (the "License"); you may not use this file except in 4 | compliance with the License. You may obtain a copy of the License at 5 | http://www.mozilla.org/MPL/ 6 | 7 | Software distributed under the License is distributed on an "AS IS" 8 | basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the 9 | License for the specific language governing rights and limitations 10 | under the License. 11 | 12 | The Original Code is DataMangler Key-Value Store. 13 | 14 | The Initial Developer of the Original Code is Mozilla Corporation. 15 | 16 | Original Author: Kevin Gadd (kevin.gadd@gmail.com) 17 | */ 18 | 19 | using System; 20 | using System.Collections.Generic; 21 | using System.Linq; 22 | using System.Text; 23 | using Squared.Task; 24 | 25 | namespace Squared.Data.Mangler { 26 | internal struct BatchItem { 27 | public readonly bool AllowReplacement; 28 | public readonly UpdateCallback Callback; 29 | public readonly DecisionUpdateCallback DecisionCallback; 30 | public TangleKey Key; 31 | public T Value; 32 | 33 | public BatchItem (TangleKey key, ref T value, bool allowReplacement) { 34 | Key = key; 35 | Value = value; 36 | AllowReplacement = allowReplacement; 37 | Callback = null; 38 | DecisionCallback = null; 39 | } 40 | 41 | public BatchItem (TangleKey key, ref T value, UpdateCallback callback) { 42 | Key = key; 43 | Value = value; 44 | AllowReplacement = false; 45 | Callback = callback; 46 | DecisionCallback = null; 47 | } 48 | 49 | public BatchItem (TangleKey key, ref T value, DecisionUpdateCallback callback) { 50 | Key = key; 51 | Value = value; 52 | AllowReplacement = false; 53 | Callback = null; 54 | DecisionCallback = callback; 55 | } 56 | } 57 | 58 | /// 59 | /// A Batch allows you to apply multiple modifications to a Tangle as a single work item. 60 | /// To use it, create an instance of the appropriate batch type, add modifications to it using its methods, and then Execute it on the Tangle. 61 | /// Using a batch to modify a Tangle is faster than calling the Tangle's methods individually. 62 | /// 63 | /// The type of the tangle's items. 64 | public class Batch : IBatch { 65 | public readonly int Capacity; 66 | public readonly Tangle Tangle; 67 | 68 | internal readonly BatchItem[] Buffer; 69 | private int _Count; 70 | 71 | /// 72 | /// Creates a batch. 73 | /// 74 | /// The tangle against which to issue the batch's commands. 75 | /// The maximum number of modifications that the batch can contain. 76 | public Batch (Tangle tangle, int capacity) { 77 | Tangle = tangle; 78 | Capacity = capacity; 79 | Buffer = new BatchItem[capacity]; 80 | } 81 | 82 | /// 83 | /// The number of modifications currently contained by the batch. 84 | /// 85 | public int Count { 86 | get { 87 | return _Count; 88 | } 89 | } 90 | 91 | public void Add (TangleKey key, T value) { 92 | Add(key, ref value); 93 | } 94 | 95 | public void Add (TangleKey key, ref T value) { 96 | if (_Count >= Capacity) 97 | throw new IndexOutOfRangeException(); 98 | 99 | Buffer[_Count++] = new BatchItem(key, ref value, false); 100 | } 101 | 102 | public void Set (TangleKey key, T value) { 103 | Set(key, ref value); 104 | } 105 | 106 | public void Set (TangleKey key, ref T value) { 107 | if (_Count >= Capacity) 108 | throw new IndexOutOfRangeException(); 109 | 110 | Buffer[_Count++] = new BatchItem(key, ref value, true); 111 | } 112 | 113 | public void AddOrUpdate (TangleKey key, T value, UpdateCallback updateCallback) { 114 | AddOrUpdate(key, ref value, updateCallback); 115 | } 116 | 117 | public void AddOrUpdate (TangleKey key, ref T value, UpdateCallback updateCallback) { 118 | if (_Count >= Capacity) 119 | throw new IndexOutOfRangeException(); 120 | 121 | Buffer[_Count++] = new BatchItem(key, ref value, updateCallback); 122 | } 123 | 124 | public void AddOrUpdate (TangleKey key, T value, DecisionUpdateCallback updateCallback) { 125 | AddOrUpdate(key, ref value, updateCallback); 126 | } 127 | 128 | public void AddOrUpdate (TangleKey key, ref T value, DecisionUpdateCallback updateCallback) { 129 | if (_Count >= Capacity) 130 | throw new IndexOutOfRangeException(); 131 | 132 | Buffer[_Count++] = new BatchItem(key, ref value, updateCallback); 133 | } 134 | 135 | /// 136 | /// Adds the batch to the Tangle's work queue. 137 | /// 138 | /// A future that becomes completed once all the work items in the batch have completed. 139 | public Future Execute () { 140 | return Tangle.QueueWorkItem(new Tangle.BatchThunk(this)); 141 | } 142 | } 143 | 144 | /// 145 | /// Represents a collection of barriers in one or more tangles. 146 | /// 147 | public class BarrierCollection : IBarrier { 148 | private readonly List Barriers = new List(); 149 | private readonly IFuture Composite; 150 | 151 | /// 152 | /// Inserts barriers into one or more tangles and creates a barrier collection out of them. 153 | /// 154 | /// If true, the barrier collection will not become signaled until all the barriers within it are signaled. Otherwise, only one barrier within the collection must become signaled before the collection is signaled. 155 | /// The list of tangles to create barriers in. 156 | public BarrierCollection (bool waitForAll, IEnumerable tangles) { 157 | foreach (var tangle in tangles) 158 | Barriers.Add(tangle.CreateBarrier(false)); 159 | 160 | var futures = from barrier in Barriers select barrier.Future; 161 | if (waitForAll) 162 | Composite = Squared.Task.Future.WaitForAll(futures); 163 | else 164 | Composite = Squared.Task.Future.WaitForFirst(futures); 165 | } 166 | 167 | /// 168 | /// Inserts barriers into one or more tangles and creates a barrier collection out of them. 169 | /// 170 | /// If true, the barrier collection will not become signaled until all the barriers within it are signaled. Otherwise, only one barrier within the collection must become signaled before the collection is signaled. 171 | /// The list of tangles to create barriers in. 172 | public BarrierCollection (bool waitForAll, params ITangle[] tangles) 173 | : this(waitForAll, (IEnumerable)tangles) { 174 | } 175 | 176 | public void Dispose () { 177 | foreach (var barrier in Barriers) 178 | barrier.Dispose(); 179 | 180 | Barriers.Clear(); 181 | Composite.Dispose(); 182 | } 183 | 184 | /// 185 | /// Opens all the barriers within the collection. 186 | /// 187 | public void Open () { 188 | foreach (var barrier in Barriers) 189 | barrier.Open(); 190 | } 191 | 192 | public IFuture Future { 193 | get { 194 | return Composite; 195 | } 196 | } 197 | 198 | void ISchedulable.Schedule (TaskScheduler scheduler, IFuture future) { 199 | future.Bind(Composite); 200 | } 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /Tests/IndexTests.cs: -------------------------------------------------------------------------------- 1 | /* 2 | The contents of this file are subject to the Mozilla Public License 3 | Version 1.1 (the "License"); you may not use this file except in 4 | compliance with the License. You may obtain a copy of the License at 5 | http://www.mozilla.org/MPL/ 6 | 7 | Software distributed under the License is distributed on an "AS IS" 8 | basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the 9 | License for the specific language governing rights and limitations 10 | under the License. 11 | 12 | The Original Code is DataMangler Key-Value Store. 13 | 14 | The Initial Developer of the Original Code is Mozilla Corporation. 15 | 16 | Original Author: Kevin Gadd (kevin.gadd@gmail.com) 17 | */ 18 | 19 | using System; 20 | using System.Collections.Generic; 21 | using System.Linq; 22 | using System.Text; 23 | using NUnit.Framework; 24 | using Squared.Task; 25 | 26 | namespace Squared.Data.Mangler.Tests { 27 | [TestFixture] 28 | public class IndexTests : BasicTestFixture { 29 | public Tangle Tangle; 30 | 31 | [SetUp] 32 | public unsafe override void SetUp () { 33 | base.SetUp(); 34 | 35 | var serializer = new Squared.Data.Mangler.Serialization.StringSerializer( 36 | Encoding.UTF8 37 | ); 38 | 39 | Tangle = new Tangle( 40 | Scheduler, Storage, 41 | serializer: serializer.Serialize, 42 | deserializer: serializer.Deserialize, 43 | ownsStorage: true 44 | ); 45 | } 46 | 47 | [TearDown] 48 | public override void TearDown () { 49 | Tangle.Dispose(); 50 | base.TearDown(); 51 | } 52 | 53 | [Test] 54 | public void IndexUpdatedWhenAddingNewValues () { 55 | var ByValue = Scheduler.WaitFor(Tangle.CreateIndex("ByValue", (ref string v) => v)); 56 | 57 | var key = new TangleKey("hello"); 58 | var value = "world"; 59 | 60 | Scheduler.WaitFor(Tangle.Set(key, value)); 61 | 62 | Assert.AreEqual(key, Scheduler.WaitFor(ByValue.FindOne(value))); 63 | Assert.AreEqual(value, Scheduler.WaitFor(ByValue.GetOne(value))); 64 | } 65 | 66 | [Test] 67 | public void IndexHandlesMultipleKeysForTheSameValue () { 68 | var ByValue = Scheduler.WaitFor(Tangle.CreateIndex("ByValue", (ref string v) => v)); 69 | 70 | var key1 = new TangleKey("hello"); 71 | var key2 = new TangleKey("greetings"); 72 | var value = "world"; 73 | 74 | Scheduler.WaitFor(Tangle.Set(key1, value)); 75 | Scheduler.WaitFor(Tangle.Set(key2, value)); 76 | 77 | Assert.AreEqual( 78 | new TangleKey[] { key1, key2 }, 79 | Scheduler.WaitFor(ByValue.Find(value)) 80 | ); 81 | Assert.AreEqual( 82 | new string[] { value, value }, 83 | Scheduler.WaitFor(ByValue.Get(value)) 84 | ); 85 | } 86 | 87 | [Test] 88 | public void IndexUpdatedWhenValueChanged () { 89 | var ByValue = Scheduler.WaitFor(Tangle.CreateIndex("ByValue", (ref string v) => v)); 90 | 91 | var key = new TangleKey("hello"); 92 | var value1 = "world"; 93 | var value2 = "place"; 94 | 95 | Scheduler.WaitFor(Tangle.Set(key, value1)); 96 | 97 | Assert.AreEqual(key, Scheduler.WaitFor(ByValue.FindOne(value1))); 98 | 99 | Scheduler.WaitFor(Tangle.Set(key, value2)); 100 | 101 | try { 102 | Scheduler.WaitFor(ByValue.FindOne(value1)); 103 | Assert.Fail("Expected to throw"); 104 | } catch (FutureException fe) { 105 | Assert.IsInstanceOf(fe.InnerException); 106 | } 107 | 108 | Assert.AreEqual(key, Scheduler.WaitFor(ByValue.FindOne(value2))); 109 | } 110 | 111 | [Test] 112 | public void CanAddIndexToTangleWithExistingValues () { 113 | var key1 = new TangleKey("hello"); 114 | var value1 = "world"; 115 | var key2 = new TangleKey("greetings"); 116 | var value2 = "place"; 117 | 118 | Scheduler.WaitFor(Tangle.Set(key1, value1)); 119 | Scheduler.WaitFor(Tangle.Set(key2, value2)); 120 | 121 | var ByValue = Scheduler.WaitFor(Tangle.CreateIndex("ByValue", (ref string v) => v)); 122 | 123 | Assert.AreEqual(key1, Scheduler.WaitFor(ByValue.FindOne(value1))); 124 | Assert.AreEqual(key2, Scheduler.WaitFor(ByValue.FindOne(value2))); 125 | } 126 | 127 | [Test] 128 | public void FetchKeysForMultipleValues () { 129 | var ByValue = Scheduler.WaitFor(Tangle.CreateIndex("ByValue", (ref string v) => v)); 130 | 131 | var key1 = new TangleKey("hello"); 132 | var key2 = new TangleKey("greetings"); 133 | var key3 = new TangleKey("hi"); 134 | var value1 = "world"; 135 | var value2 = "cat"; 136 | 137 | Scheduler.WaitFor(Tangle.Set(key1, value1)); 138 | Scheduler.WaitFor(Tangle.Set(key2, value1)); 139 | Scheduler.WaitFor(Tangle.Set(key3, value2)); 140 | 141 | Assert.AreEqual( 142 | new TangleKey[] { key1, key2, key3 }, 143 | Scheduler.WaitFor(ByValue.Find(new[] { value1, value2 })) 144 | ); 145 | Assert.AreEqual( 146 | new string[] { value1, value1, value2 }, 147 | Scheduler.WaitFor(ByValue.Get(new [] { value1, value2 })) 148 | ); 149 | } 150 | 151 | [Test] 152 | public void CanUseEnumeratorAsIndexFunction () { 153 | // Bleh, unless we explicitly specify the type argument to CreateIndex, 154 | // it assumes a type of instead of picking the IEnumerable overload. 155 | var ByWords = Scheduler.WaitFor(Tangle.CreateIndex( 156 | "ByWords", 157 | (string v) => v.Split(' ') 158 | )); 159 | 160 | var key1 = new TangleKey("a"); 161 | var key2 = new TangleKey("b"); 162 | var value1 = "Hello World"; 163 | var value2 = "Greetings World"; 164 | 165 | Scheduler.WaitFor(Tangle.Set(key1, value1)); 166 | Scheduler.WaitFor(Tangle.Set(key2, value2)); 167 | 168 | Assert.AreEqual(new [] { key1, key2 }, Scheduler.WaitFor(ByWords.Find("World"))); 169 | Assert.AreEqual(new [] { key2 }, Scheduler.WaitFor(ByWords.Find("Greetings"))); 170 | } 171 | 172 | [Test] 173 | public void FetchKeysForMultipleValuesWithEnumeratorIndex () { 174 | var ByWords = Scheduler.WaitFor(Tangle.CreateIndex( 175 | "ByWords", 176 | (string v) => v.Split(' ') 177 | )); 178 | 179 | var key1 = new TangleKey("a"); 180 | var key2 = new TangleKey("b"); 181 | var value1 = "Hello World"; 182 | var value2 = "Greetings World"; 183 | 184 | Scheduler.WaitFor(Tangle.Set(key1, value1)); 185 | Scheduler.WaitFor(Tangle.Set(key2, value2)); 186 | 187 | Assert.AreEqual( 188 | new[] { key1, key2 }, 189 | Scheduler.WaitFor(ByWords.Find(new [] { "Hello", "Greetings" })) 190 | ); 191 | } 192 | 193 | [Test] 194 | public void TestGetAllKeys () { 195 | var ByWords = Scheduler.WaitFor(Tangle.CreateIndex( 196 | "ByWords", 197 | (string v) => v.Split(' ') 198 | )); 199 | 200 | var key1 = new TangleKey("a"); 201 | var key2 = new TangleKey("b"); 202 | var value1 = "Hello World"; 203 | var value2 = "Greetings World"; 204 | 205 | Scheduler.WaitFor(Tangle.Set(key1, value1)); 206 | Scheduler.WaitFor(Tangle.Set(key2, value2)); 207 | 208 | Assert.AreEqual( 209 | new[] { "Greetings", "Hello", "World" }, 210 | Scheduler.WaitFor(ByWords.GetAllKeys()) 211 | .Select((k) => k.Value as string) 212 | .OrderBy((k) => k) 213 | .ToArray() 214 | ); 215 | } 216 | } 217 | } 218 | -------------------------------------------------------------------------------- /Util.cs: -------------------------------------------------------------------------------- 1 | /* 2 | The contents of this file are subject to the Mozilla Public License 3 | Version 1.1 (the "License"); you may not use this file except in 4 | compliance with the License. You may obtain a copy of the License at 5 | http://www.mozilla.org/MPL/ 6 | 7 | Software distributed under the License is distributed on an "AS IS" 8 | basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the 9 | License for the specific language governing rights and limitations 10 | under the License. 11 | 12 | The Original Code is DataMangler Key-Value Store. 13 | 14 | The Initial Developer of the Original Code is Mozilla Corporation. 15 | 16 | Original Author: Kevin Gadd (kevin.gadd@gmail.com) 17 | */ 18 | 19 | using System; 20 | using System.Collections.Generic; 21 | using System.IO.MemoryMappedFiles; 22 | using System.Runtime.InteropServices; 23 | using System.IO; 24 | using System.Linq.Expressions; 25 | using Squared.Task; 26 | 27 | namespace Squared.Data.Mangler.Internal { 28 | delegate SafeBuffer GetSafeBufferFunc (UnmanagedMemoryAccessor accessor); 29 | delegate Int64 GetPointerOffsetFunc (MemoryMappedViewAccessor accessor); 30 | 31 | public static class InternalExtensions { 32 | private static readonly GetSafeBufferFunc _GetSafeBuffer; 33 | private static readonly GetPointerOffsetFunc _GetPointerOffset; 34 | 35 | static InternalExtensions () { 36 | _GetSafeBuffer = CreateGetSafeBuffer(); 37 | _GetPointerOffset = CreateGetPointerOffset(); 38 | } 39 | 40 | // To manipulate structures directly in mapped memory, we have 41 | // to be able to get a pointer to the mapping. While this is possible, 42 | // the classes for using mapped files do not expose a way to do this 43 | // directly. So, we pull out the SafeBuffer object associated with the 44 | // mapping and then use a public method to get a pointer. 45 | // Kind of nasty, but what else can you do? 46 | private static GetSafeBufferFunc CreateGetSafeBuffer () { 47 | var t = typeof(UnmanagedMemoryAccessor); 48 | var field = t.GetField( 49 | "_buffer", 50 | System.Reflection.BindingFlags.NonPublic | 51 | System.Reflection.BindingFlags.Instance 52 | ); 53 | if (field == null) 54 | throw new ArgumentNullException(); 55 | 56 | var argument = Expression.Parameter(t, "accessor"); 57 | var expr = Expression.Field(argument, field); 58 | 59 | return Expression.Lambda( 60 | expr, "GetSafeBuffer", new[] { argument } 61 | ).Compile(); 62 | } 63 | 64 | // When we get a pointer from a SafeBuffer associated with a mapped 65 | // view, the pointer is going to be wrong unless the offset into 66 | // the file that we mapped was aligned with a page boundary. 67 | // So, once we get the pointer, we have to find out the alignment 68 | // necessary to line it up with a page, and add that to the pointer 69 | // so that we are looking at the start of the mapping, instead of 70 | // the start of the page containing the mapping. 71 | // This is a bit messier than the SafeBuffer hack, because one of the 72 | // relevant types - MemoryMappedView - is internal. 73 | private static GetPointerOffsetFunc CreateGetPointerOffset () { 74 | var tAccessor = typeof(MemoryMappedViewAccessor); 75 | var tView = tAccessor.Assembly.GetType( 76 | "System.IO.MemoryMappedFiles.MemoryMappedView", true 77 | ); 78 | 79 | var fieldView = tAccessor.GetField( 80 | "m_view", 81 | System.Reflection.BindingFlags.NonPublic | 82 | System.Reflection.BindingFlags.Instance 83 | ); 84 | if (fieldView == null) 85 | throw new ArgumentNullException(); 86 | 87 | var fieldOffset = tView.GetField( 88 | "m_pointerOffset", 89 | System.Reflection.BindingFlags.NonPublic | 90 | System.Reflection.BindingFlags.Instance 91 | ); 92 | if (fieldOffset == null) 93 | throw new ArgumentNullException(); 94 | 95 | var argument = Expression.Parameter(tAccessor, "accessor"); 96 | var expr = Expression.Field( 97 | Expression.Field(argument, fieldView), fieldOffset 98 | ); 99 | 100 | return Expression.Lambda( 101 | expr, "GetPointerOffset", new[] { argument } 102 | ).Compile(); 103 | } 104 | 105 | internal static SafeBuffer GetSafeBuffer (this UnmanagedMemoryAccessor accessor) { 106 | var buffer = _GetSafeBuffer(accessor); 107 | if (buffer == null) 108 | throw new InvalidDataException(); 109 | return buffer; 110 | } 111 | 112 | internal static Int64 GetPointerOffset (this MemoryMappedViewAccessor accessor) { 113 | return _GetPointerOffset(accessor); 114 | } 115 | 116 | public static ArraySegment GetSegment (this MemoryStream stream) { 117 | if (stream.Length >= int.MaxValue) 118 | throw new InvalidDataException(); 119 | 120 | return new ArraySegment(stream.GetBuffer(), 0, (int)stream.Length); 121 | } 122 | 123 | public static IEnumerable AsEnumerable (this ArraySegment segment) { 124 | var a = segment.Array; 125 | for (int i = 0, c = segment.Count, o = segment.Offset; i < c; i++) 126 | yield return a[i + o]; 127 | } 128 | } 129 | 130 | public unsafe delegate void GenericPtrToStructureFunc (byte* source, out T destination, uint size) 131 | where T : struct; 132 | public unsafe delegate void GenericStructureToPtrFunc (ref T source, byte* destination, uint size) 133 | where T : struct; 134 | 135 | public unsafe static class Unsafe { 136 | public static void ReadBytes (byte* ptr, long offset, byte[] buffer, long bufferOffset, uint count) { 137 | fixed (byte* pDest = &buffer[bufferOffset]) 138 | Native.memmove(pDest, ptr + offset, new UIntPtr((uint)count)); 139 | } 140 | 141 | public static void WriteBytes (byte* ptr, long offset, ArraySegment bytes) { 142 | WriteBytes(ptr, offset, bytes.Array, bytes.Offset, bytes.Count); 143 | } 144 | 145 | public static void WriteBytes (byte* ptr, long offset, byte[] source, int sourceOffset, int count) { 146 | fixed (byte* pSource = &source[sourceOffset]) 147 | Native.memmove(ptr + offset, pSource, new UIntPtr((uint)count)); 148 | } 149 | 150 | public static void ZeroBytes (byte* ptr, long offset, uint count) { 151 | Native.memset(ptr + offset, 0, new UIntPtr((uint)count)); 152 | } 153 | } 154 | 155 | public unsafe static class Unsafe 156 | where T : struct { 157 | 158 | public static readonly GenericPtrToStructureFunc PtrToStructure; 159 | public static readonly GenericStructureToPtrFunc StructureToPtr; 160 | 161 | static Unsafe () { 162 | var tSafeBuffer = typeof(SafeBuffer); 163 | 164 | var method = tSafeBuffer.GetMethod( 165 | "GenericPtrToStructure", 166 | System.Reflection.BindingFlags.Static | 167 | System.Reflection.BindingFlags.NonPublic 168 | ).MakeGenericMethod(typeof(T)); 169 | PtrToStructure = (GenericPtrToStructureFunc)Delegate.CreateDelegate( 170 | typeof(GenericPtrToStructureFunc), method, true 171 | ); 172 | 173 | method = tSafeBuffer.GetMethod( 174 | "GenericStructureToPtr", 175 | System.Reflection.BindingFlags.Static | 176 | System.Reflection.BindingFlags.NonPublic 177 | ).MakeGenericMethod(typeof(T)); 178 | StructureToPtr = (GenericStructureToPtrFunc)Delegate.CreateDelegate( 179 | typeof(GenericStructureToPtrFunc), method, true 180 | ); 181 | } 182 | } 183 | 184 | public static class IntegerUtil { 185 | public static int Log2 (uint value) { 186 | int result = 0; 187 | 188 | while ((value >>= 1) != 0) 189 | result += 1; 190 | 191 | return result; 192 | } 193 | 194 | public static bool IsPowerOfTwo (uint value) { 195 | unchecked { 196 | return (value != 0) && 197 | ((value & (value - 1)) != 0); 198 | } 199 | } 200 | 201 | public static uint NextPowerOfTwo (uint value) { 202 | unchecked { 203 | value--; 204 | value |= value >> 1; 205 | value |= value >> 2; 206 | value |= value >> 4; 207 | value |= value >> 8; 208 | value |= value >> 16; 209 | value++; 210 | } 211 | 212 | return value; 213 | } 214 | } 215 | } -------------------------------------------------------------------------------- /TangleKey.cs: -------------------------------------------------------------------------------- 1 | /* 2 | The contents of this file are subject to the Mozilla Public License 3 | Version 1.1 (the "License"); you may not use this file except in 4 | compliance with the License. You may obtain a copy of the License at 5 | http://www.mozilla.org/MPL/ 6 | 7 | Software distributed under the License is distributed on an "AS IS" 8 | basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the 9 | License for the specific language governing rights and limitations 10 | under the License. 11 | 12 | The Original Code is DataMangler Key-Value Store. 13 | 14 | The Initial Developer of the Original Code is Mozilla Corporation. 15 | 16 | Original Author: Kevin Gadd (kevin.gadd@gmail.com) 17 | */ 18 | 19 | using System; 20 | using System.Collections.Generic; 21 | using System.Linq.Expressions; 22 | using System.Text; 23 | using System.IO; 24 | using Squared.Data.Mangler.Internal; 25 | 26 | namespace Squared.Data.Mangler { 27 | public struct TangleKey : IComparable, IEquatable { 28 | private static readonly Dictionary Converters = new Dictionary(); 29 | private static readonly Dictionary TypeIdToType = new Dictionary(); 30 | private static readonly Dictionary TypeToTypeId = new Dictionary(); 31 | 32 | static TangleKey () { 33 | Converters[typeof(TangleKey)] = (TangleKeyConverter)((key) => key); 34 | 35 | RegisterType(); 36 | RegisterType(); 37 | RegisterType(); 38 | RegisterType(); 39 | RegisterType(); 40 | RegisterType(); 41 | RegisterType(); 42 | RegisterType(); 43 | RegisterType(); 44 | } 45 | 46 | private static void RegisterType (bool autoConverter = true) { 47 | if (TypeToTypeId.Count >= (ushort.MaxValue - 2)) 48 | throw new InvalidOperationException("Too many registered types"); 49 | 50 | var type = typeof(T); 51 | ushort id = (byte)(TypeToTypeId.Count + 1); 52 | TypeToTypeId[type] = id; 53 | TypeIdToType[id] = type; 54 | 55 | if (autoConverter) { 56 | var constructor = typeof(TangleKey).GetConstructor(new Type[] { type }); 57 | var parameter = Expression.Parameter(type, "value"); 58 | var expression = Expression.New(constructor, new[] { parameter }); 59 | var converterType = typeof(TangleKeyConverter); 60 | var converterExpression = Expression.Lambda(converterType, expression, parameter); 61 | var converter = converterExpression.Compile(); 62 | Converters[type] = converter; 63 | } 64 | } 65 | 66 | public static void RegisterKeyType (TangleKeyConverter converter) { 67 | RegisterType(autoConverter: false); 68 | Converters[typeof(T)] = converter; 69 | } 70 | 71 | public static TangleKeyConverter GetConverter () { 72 | Delegate converter; 73 | if (!Converters.TryGetValue(typeof(T), out converter)) 74 | throw new InvalidOperationException(String.Format("Type '{0}' is not convertible to TangleKey", typeof(T).Name)); 75 | 76 | return (TangleKeyConverter)converter; 77 | } 78 | 79 | public readonly ushort OriginalTypeId; 80 | public readonly ArraySegment Data; 81 | 82 | public TangleKey (byte key) 83 | : this(ImmutableBufferPool.GetBytes(key), typeof(byte)) { 84 | } 85 | 86 | public TangleKey (ushort key) 87 | : this(ImmutableBufferPool.GetBytes(key), typeof(ushort)) { 88 | } 89 | 90 | public TangleKey (short key) 91 | : this(ImmutableBufferPool.GetBytes(key), typeof(short)) { 92 | } 93 | 94 | public TangleKey (uint key) 95 | : this(ImmutableBufferPool.GetBytes(key), typeof(uint)) { 96 | } 97 | 98 | public TangleKey (int key) 99 | : this(ImmutableBufferPool.GetBytes(key), typeof(int)) { 100 | } 101 | 102 | public TangleKey (ulong key) 103 | : this(ImmutableBufferPool.GetBytes(key), typeof(ulong)) { 104 | } 105 | 106 | public TangleKey (long key) 107 | : this(ImmutableBufferPool.GetBytes(key), typeof(long)) { 108 | } 109 | 110 | public TangleKey (string key) 111 | : this(ImmutableBufferPool.GetBytes(key, Encoding.UTF8), typeof(string)) { 112 | } 113 | 114 | public TangleKey (byte[] array) 115 | : this(array, 0, array.Length, TypeToTypeId[typeof(byte[])]) { 116 | } 117 | 118 | public TangleKey (byte[] array, int offset, int count) 119 | : this(array, offset, count, TypeToTypeId[typeof(string)]) { 120 | } 121 | 122 | public TangleKey (byte[] array, ushort originalType) 123 | : this(array, 0, array.Length, originalType) { 124 | } 125 | 126 | public TangleKey (byte[] array, int offset, int count, Type originalType) 127 | : this(new ArraySegment(array, offset, count), originalType) { 128 | } 129 | 130 | public TangleKey (byte[] array, int offset, int count, ushort originalType) 131 | : this(new ArraySegment(array, offset, count), originalType) { 132 | } 133 | 134 | public TangleKey (ArraySegment data, Type originalType) 135 | : this(data, TypeToTypeId[originalType]) { 136 | } 137 | 138 | public TangleKey (ArraySegment data, ushort originalType) { 139 | if (data.Count >= ushort.MaxValue) 140 | throw new InvalidDataException("Key too long"); 141 | if (originalType == 0) 142 | throw new InvalidDataException("Invalid key type"); 143 | 144 | Data = data; 145 | OriginalTypeId = originalType; 146 | } 147 | 148 | public Type OriginalType { 149 | get { 150 | return TypeIdToType[OriginalTypeId]; 151 | } 152 | } 153 | 154 | public object Value { 155 | get { 156 | var type = OriginalType; 157 | 158 | if (type == typeof(string)) { 159 | return Encoding.ASCII.GetString(Data.Array, Data.Offset, Data.Count); 160 | } else if (type == typeof(int)) { 161 | return BitConverter.ToInt32(Data.Array, Data.Offset); 162 | } else if (type == typeof(uint)) { 163 | return BitConverter.ToUInt32(Data.Array, Data.Offset); 164 | } else if (type == typeof(long)) { 165 | return BitConverter.ToInt64(Data.Array, Data.Offset); 166 | } else if (type == typeof(ulong)) { 167 | return BitConverter.ToUInt64(Data.Array, Data.Offset); 168 | } else /* if (type == typeof(byte[])) */ { 169 | return Data; 170 | } 171 | } 172 | } 173 | 174 | public static implicit operator TangleKey (string key) { 175 | return new TangleKey(key); 176 | } 177 | 178 | public static implicit operator TangleKey (uint key) { 179 | return new TangleKey(key); 180 | } 181 | 182 | public static implicit operator TangleKey (int key) { 183 | return new TangleKey(key); 184 | } 185 | 186 | public static implicit operator TangleKey (ulong key) { 187 | return new TangleKey(key); 188 | } 189 | 190 | public static implicit operator TangleKey (long key) { 191 | return new TangleKey(key); 192 | } 193 | 194 | public override string ToString () { 195 | var value = Value; 196 | 197 | if (value is ArraySegment) { 198 | var sb = new StringBuilder(); 199 | for (int i = 0; i < Data.Count; i++) 200 | sb.AppendFormat("{0:X2}", Data.Array[i + Data.Offset]); 201 | return sb.ToString(); 202 | } else { 203 | return value.ToString(); 204 | } 205 | } 206 | 207 | private static unsafe int CompareData (ArraySegment lhs, ArraySegment rhs) { 208 | int result; 209 | uint compareLength = (uint)Math.Min(lhs.Count, rhs.Count); 210 | 211 | fixed (byte * pLhs = &lhs.Array[lhs.Offset]) 212 | fixed (byte * pRhs = &rhs.Array[rhs.Offset]) 213 | result = Native.memcmp(pLhs, pRhs, new UIntPtr(compareLength)); 214 | 215 | if (result == 0) { 216 | if (lhs.Count > rhs.Count) 217 | result = 1; 218 | else if (rhs.Count > lhs.Count) 219 | result = -1; 220 | } 221 | 222 | return result; 223 | } 224 | 225 | public int CompareTo (TangleKey rhs) { 226 | int result = OriginalTypeId.CompareTo(rhs.OriginalTypeId); 227 | if (result == 0) 228 | result = CompareData(Data, rhs.Data); 229 | 230 | return result; 231 | } 232 | 233 | public bool Equals (TangleKey other) { 234 | return CompareTo(other) == 0; 235 | } 236 | 237 | public override bool Equals (object other) { 238 | if (other is TangleKey) 239 | return this.Equals((TangleKey)other); 240 | else 241 | return base.Equals(other); 242 | } 243 | 244 | // FNV hash algorithm 245 | public override int GetHashCode () { 246 | unchecked { 247 | const int p = 16777619; 248 | int hash = -2128831035; 249 | 250 | for (int i = 0, c = Data.Count; i < c; i++) 251 | hash = (hash ^ Data.Array[i + Data.Offset]) * p; 252 | 253 | hash += hash << 13; 254 | hash ^= hash >> 7; 255 | hash += hash << 3; 256 | hash ^= hash >> 17; 257 | hash += hash << 5; 258 | 259 | return hash; 260 | } 261 | } 262 | } 263 | } 264 | -------------------------------------------------------------------------------- /Index.cs: -------------------------------------------------------------------------------- 1 | /* 2 | The contents of this file are subject to the Mozilla Public License 3 | Version 1.1 (the "License"); you may not use this file except in 4 | compliance with the License. You may obtain a copy of the License at 5 | http://www.mozilla.org/MPL/ 6 | 7 | Software distributed under the License is distributed on an "AS IS" 8 | basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the 9 | License for the specific language governing rights and limitations 10 | under the License. 11 | 12 | The Original Code is DataMangler Key-Value Store. 13 | 14 | The Initial Developer of the Original Code is Mozilla Corporation. 15 | 16 | Original Author: Kevin Gadd (kevin.gadd@gmail.com) 17 | */ 18 | 19 | using System; 20 | using System.Collections.Generic; 21 | using System.IO; 22 | using System.Linq; 23 | using System.Text; 24 | using Squared.Data.Mangler.Internal; 25 | using Squared.Task; 26 | 27 | namespace Squared.Data.Mangler { 28 | public abstract class IndexBase : IDisposable { 29 | internal abstract void OnValueRemoved (TangleKey key, ref TValue oldValue); 30 | internal abstract void OnValueAdded (TangleKey key, ref TValue newValue); 31 | 32 | internal abstract void Clear (); 33 | 34 | public abstract void Dispose (); 35 | } 36 | 37 | internal struct IndexFunctionAdapter : IEnumerable, IEnumerator { 38 | public readonly IndexFunc Function; 39 | public TValue Input; 40 | private bool Advanced; 41 | 42 | public IndexFunctionAdapter (IndexFunc function, ref TValue input) { 43 | Function = function; 44 | Input = input; 45 | Advanced = false; 46 | } 47 | 48 | IEnumerator IEnumerable.GetEnumerator () { 49 | return this; 50 | } 51 | 52 | System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator () { 53 | return this; 54 | } 55 | 56 | TIndexKey IEnumerator.Current { 57 | get { 58 | return Function(ref Input); 59 | } 60 | } 61 | 62 | void IDisposable.Dispose () { 63 | } 64 | 65 | object System.Collections.IEnumerator.Current { 66 | get { 67 | return Function(ref Input); 68 | } 69 | } 70 | 71 | bool System.Collections.IEnumerator.MoveNext () { 72 | if (Advanced) 73 | return false; 74 | 75 | Advanced = true; 76 | return true; 77 | } 78 | 79 | void System.Collections.IEnumerator.Reset () { 80 | Advanced = false; 81 | } 82 | } 83 | 84 | public partial class Index : IndexBase { 85 | public readonly Tangle Tangle; 86 | public readonly string Name; 87 | 88 | public readonly IndexFunc IndexFunction; 89 | public readonly IndexMultipleFunc IndexMultipleFunction; 90 | 91 | private readonly BTree BTree; 92 | private readonly TangleKeyConverter KeyConverter; 93 | 94 | protected Index (Tangle tangle, string name, Delegate function) { 95 | IndexFunction = function as IndexFunc; 96 | IndexMultipleFunction = function as IndexMultipleFunc; 97 | 98 | if ((IndexFunction == null) && (IndexMultipleFunction == null)) 99 | throw new InvalidOperationException("An index must have either an IndexFunc or IndexMultipleFunc"); 100 | 101 | Tangle = tangle; 102 | Name = name; 103 | KeyConverter = TangleKey.GetConverter(); 104 | 105 | IndexBase temp; 106 | if (tangle.Indices.TryGetValue(name, out temp)) 107 | throw new InvalidOperationException("An index with that name already exists"); 108 | 109 | BTree = new BTree(tangle.Storage, Name + "_"); 110 | 111 | tangle.Indices.Add(name, this); 112 | 113 | if (tangle.Count != BTree.MutationSentinel) 114 | Populate(); 115 | } 116 | 117 | protected void Populate () { 118 | long nodeCount = Tangle.NodeCount; 119 | 120 | for (long i = 0; i < nodeCount; i++) { 121 | foreach (var kvp in Tangle.InternalEnumerateNode(i)) { 122 | var value = kvp.Value; 123 | UpdateIndexForEntry(kvp.Key, ref value, true); 124 | } 125 | } 126 | } 127 | 128 | public static Future> Create (Tangle tangle, string name, IndexFunc function) { 129 | return tangle.QueueWorkItem(new CreateThunk(name, function)); 130 | } 131 | 132 | public static Future> Create (Tangle tangle, string name, IndexMultipleFunc function) { 133 | return tangle.QueueWorkItem(new CreateThunk(name, function)); 134 | } 135 | 136 | public Future FindOne (TIndexKey value) { 137 | var key = KeyConverter(value); 138 | return Tangle.QueueWorkItem(new FindOneThunk(this, key)); 139 | } 140 | 141 | public Future GetOne (TIndexKey value) { 142 | var key = KeyConverter(value); 143 | return Tangle.QueueWorkItem(new GetOneThunk(this, key)); 144 | } 145 | 146 | public Future Find (TIndexKey value) { 147 | var key = KeyConverter(value); 148 | return Tangle.QueueWorkItem(new FindThunk(this, key)); 149 | } 150 | 151 | public Future Find (IEnumerable values) { 152 | return Tangle.QueueWorkItem(new FindMultipleThunk( 153 | this, from value in values select KeyConverter(value) 154 | )); 155 | } 156 | 157 | public Future Get (TIndexKey value) { 158 | var key = KeyConverter(value); 159 | return Tangle.QueueWorkItem(new GetThunk(this, key)); 160 | } 161 | 162 | public Future Get (IEnumerable values) { 163 | return Tangle.QueueWorkItem(new GetMultipleThunk( 164 | this, from value in values select KeyConverter(value) 165 | )); 166 | } 167 | 168 | /// 169 | /// Reads every key from the index, in no particular order. 170 | /// 171 | /// A future that will contain the retrieved keys. 172 | public Future GetAllKeys () { 173 | return Tangle.QueueWorkItem(new GetAllKeysThunk(this)); 174 | } 175 | 176 | internal unsafe void UpdateIndexForEntry (TangleKey key, ref TValue value, bool add) { 177 | long nodeIndex; 178 | uint valueIndex; 179 | 180 | IEnumerable sequence; 181 | 182 | if (IndexFunction != null) 183 | sequence = new IndexFunctionAdapter( 184 | IndexFunction, ref value 185 | ); 186 | else 187 | sequence = IndexMultipleFunction(value); 188 | 189 | foreach (var synthesizedValue in sequence) { 190 | TangleKey synthesizedKey = KeyConverter(synthesizedValue); 191 | 192 | bool foundExisting = BTree.FindKey(synthesizedKey, true, out nodeIndex, out valueIndex); 193 | 194 | StreamRange range; 195 | if (foundExisting) { 196 | range = BTree.AccessNode(nodeIndex, true); 197 | } else if (add) { 198 | range = BTree.PrepareForInsert(nodeIndex, valueIndex); 199 | } else { 200 | throw new InvalidOperationException(); 201 | } 202 | 203 | long minimumSize; 204 | using (range) { 205 | if (!foundExisting) 206 | BTree.WriteNewKey(range, valueIndex, synthesizedKey); 207 | 208 | minimumSize = BTree.GetValueDataTotalBytes( 209 | range, valueIndex, 210 | foundExisting ? synthesizedKey.OriginalTypeId : (ushort)0 211 | ); 212 | } 213 | 214 | if (foundExisting) 215 | BTree.UnlockNode(range); 216 | else 217 | BTree.FinalizeInsert(range); 218 | 219 | if (add) { 220 | // Ensure we will have enough room to insert this key, if necessary 221 | if (minimumSize == 0) 222 | minimumSize = 4; 223 | 224 | minimumSize += 6 + key.Data.Count; 225 | } 226 | 227 | BTreeValue * pValue; 228 | ushort lockedKeyType; 229 | 230 | fixed (byte * pKeyBytes = &key.Data.Array[key.Data.Offset]) 231 | using (var indexRange = BTree.LockValue(nodeIndex, valueIndex, minimumSize, out pValue, out lockedKeyType)) 232 | using (var dataRange = BTree.DataStream.AccessRange(pValue->DataOffset, (uint)minimumSize)) { 233 | if ((pValue->DataLength < 4)) { 234 | pValue->ExtraDataBytes -= (4 - pValue->DataLength); 235 | pValue->DataLength = 4; 236 | *(int *)dataRange.Pointer = 0; 237 | } 238 | 239 | int numKeys = *(int *)dataRange.Pointer; 240 | uint offset = 4; 241 | 242 | uint? keyPosition = null, totalKeySize = null; 243 | for (int i = 0; i < numKeys; i++) { 244 | int keyLength = *(int *)(dataRange.Pointer + offset); 245 | offset += 4; 246 | ushort comparisonKeyType = *(ushort *)(dataRange.Pointer + offset); 247 | offset += 2; 248 | 249 | if ((comparisonKeyType == key.OriginalTypeId) && (Native.memcmp( 250 | dataRange.Pointer + offset, pKeyBytes, 251 | new UIntPtr((uint)Math.Min(key.Data.Count, keyLength)) 252 | ) == 0)) { 253 | keyPosition = offset - 6; 254 | totalKeySize = (uint)(6 + keyLength); 255 | break; 256 | } 257 | 258 | offset += (uint)keyLength; 259 | } 260 | 261 | if (add) { 262 | if (!keyPosition.HasValue) { 263 | // Add the key at the end of the list 264 | var insertPosition = pValue->DataLength; 265 | if ((pValue->DataLength + pValue->ExtraDataBytes) < (insertPosition + 6 + key.Data.Count)) 266 | throw new InvalidDataException(); 267 | 268 | *(int *)(dataRange.Pointer + insertPosition) = key.Data.Count; 269 | *(ushort *)(dataRange.Pointer + insertPosition + 4) = key.OriginalTypeId; 270 | Native.memmove( 271 | dataRange.Pointer + insertPosition + 6, 272 | pKeyBytes, new UIntPtr((uint)key.Data.Count) 273 | ); 274 | 275 | pValue->DataLength += (uint)(6 + key.Data.Count); 276 | pValue->ExtraDataBytes -= (uint)(6 + key.Data.Count); 277 | 278 | *(int *)dataRange.Pointer += 1; 279 | } 280 | } else if (keyPosition.HasValue) { 281 | // Remove the key by moving the items after it back in the list 282 | 283 | var moveSize = dataRange.Size - (keyPosition.Value + totalKeySize); 284 | if (moveSize > 0) 285 | Native.memmove( 286 | dataRange.Pointer + keyPosition.Value, 287 | dataRange.Pointer + keyPosition.Value + totalKeySize.Value, 288 | new UIntPtr((uint)moveSize) 289 | ); 290 | 291 | pValue->DataLength -= (uint)(6 + key.Data.Count); 292 | pValue->ExtraDataBytes += (uint)(6 + key.Data.Count); 293 | 294 | *(int *)dataRange.Pointer -= 1; 295 | } 296 | 297 | BTree.UnlockValue(pValue, synthesizedKey.OriginalTypeId); 298 | BTree.UnlockNode(indexRange); 299 | } 300 | } 301 | 302 | if (add) 303 | BTree.MutationSentinel += 1; 304 | else 305 | BTree.MutationSentinel -= 1; 306 | } 307 | 308 | internal override void OnValueRemoved (TangleKey key, ref TValue oldValue) { 309 | UpdateIndexForEntry(key, ref oldValue, false); 310 | } 311 | 312 | internal override void OnValueAdded (TangleKey key, ref TValue newValue) { 313 | UpdateIndexForEntry(key, ref newValue, true); 314 | } 315 | 316 | internal override void Clear () { 317 | BTree.Clear(); 318 | } 319 | 320 | private unsafe static void DeserializeKeys (ref DeserializationContext context, out HashSet output) { 321 | if (context.SourceLength < 4) 322 | throw new InvalidDataException(); 323 | 324 | int count = *(int *)(context.Source); 325 | output = new HashSet(); 326 | uint offset = 4; 327 | 328 | for (int i = 0; i < count; i++) { 329 | if (context.SourceLength < (offset + 6)) 330 | throw new InvalidDataException(); 331 | 332 | int keyLength = *(int *)(context.Source + offset); 333 | ushort keyType = *(ushort *)(context.Source + offset + 4); 334 | 335 | offset += 6; 336 | if (context.SourceLength < (offset + keyLength)) 337 | throw new InvalidDataException(); 338 | 339 | var keyBytes = ImmutableArrayPool.Allocate(keyLength); 340 | Unsafe.ReadBytes(context.Source, offset, keyBytes.Array, keyBytes.Offset, (uint)keyBytes.Count); 341 | 342 | output.Add(new TangleKey(keyBytes, keyType)); 343 | offset += (uint)keyLength; 344 | } 345 | } 346 | 347 | public override void Dispose () { 348 | BTree.Dispose(); 349 | } 350 | } 351 | } 352 | -------------------------------------------------------------------------------- /Serialization.cs: -------------------------------------------------------------------------------- 1 | /* 2 | The contents of this file are subject to the Mozilla Public License 3 | Version 1.1 (the "License"); you may not use this file except in 4 | compliance with the License. You may obtain a copy of the License at 5 | http://www.mozilla.org/MPL/ 6 | 7 | Software distributed under the License is distributed on an "AS IS" 8 | basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the 9 | License for the specific language governing rights and limitations 10 | under the License. 11 | 12 | The Original Code is DataMangler Key-Value Store. 13 | 14 | The Initial Developer of the Original Code is Mozilla Corporation. 15 | 16 | Original Author: Kevin Gadd (kevin.gadd@gmail.com) 17 | */ 18 | 19 | using System; 20 | using System.Collections.Generic; 21 | using System.Linq; 22 | using System.Runtime.InteropServices; 23 | using System.Runtime.Serialization.Formatters.Binary; 24 | using System.Text; 25 | using System.IO; 26 | using System.Threading; 27 | using System.Xml.Serialization; 28 | using Squared.Data.Mangler.Internal; 29 | using Squared.Data.Mangler.Serialization; 30 | 31 | namespace Squared.Data.Mangler { 32 | [AttributeUsage(AttributeTargets.Method, Inherited = false, AllowMultiple = false)] 33 | public class TangleSerializerAttribute : Attribute { 34 | } 35 | 36 | [AttributeUsage(AttributeTargets.Method, Inherited = false, AllowMultiple = false)] 37 | public class TangleDeserializerAttribute : Attribute { 38 | } 39 | 40 | internal unsafe delegate bool GetKeyOfEntryFunc (BTreeValue * pEntry, ushort keyType, out TangleKey key); 41 | 42 | public unsafe struct SerializationContext { 43 | private bool BytesProvided, StreamInUse; 44 | private ArraySegment _Bytes; 45 | private readonly MemoryStream _Stream; 46 | private readonly BTreeValue * ValuePointer; 47 | private readonly ushort KeyType; 48 | private readonly GetKeyOfEntryFunc GetKeyOfEntry; 49 | private TangleKey _Key; 50 | private bool _KeyCached; 51 | 52 | internal SerializationContext (GetKeyOfEntryFunc getKeyOfEntry, BTreeValue * pEntry, ushort keyType, MemoryStream stream) { 53 | GetKeyOfEntry = getKeyOfEntry; 54 | ValuePointer = pEntry; 55 | KeyType = keyType; 56 | _Key = default(TangleKey); 57 | _KeyCached = false; 58 | _Stream = stream; 59 | BytesProvided = false; 60 | StreamInUse = false; 61 | _Bytes = default(ArraySegment); 62 | } 63 | 64 | public void SetResult (byte[] bytes) { 65 | SetResult(new ArraySegment(bytes)); 66 | } 67 | 68 | public void SetResult (ArraySegment bytes) { 69 | if (StreamInUse) 70 | throw new InvalidOperationException("You are already using this context's Stream"); 71 | if (BytesProvided) 72 | throw new InvalidOperationException("You already provided an ArraySegment"); 73 | 74 | BytesProvided = true; 75 | _Bytes = bytes; 76 | } 77 | 78 | public TangleKey Key { 79 | get { 80 | if (!_KeyCached) 81 | _KeyCached = GetKeyOfEntry(ValuePointer, KeyType, out _Key); 82 | 83 | if (!_KeyCached) 84 | throw new InvalidDataException(); 85 | 86 | return _Key; 87 | } 88 | } 89 | 90 | public Stream Stream { 91 | get { 92 | if (BytesProvided) 93 | throw new InvalidOperationException("You already provided an ArraySegment"); 94 | 95 | StreamInUse = true; 96 | return _Stream; 97 | } 98 | } 99 | 100 | internal ArraySegment Bytes { 101 | get { 102 | if (BytesProvided) 103 | return _Bytes; 104 | else 105 | return new ArraySegment(_Stream.GetBuffer(), 0, (int)_Stream.Length); 106 | } 107 | } 108 | 109 | public void SerializeValue (U input) { 110 | SerializeValue(ref input); 111 | } 112 | 113 | public void SerializeValue (ref U input) { 114 | SerializeValue(Defaults.Serializer, ref input); 115 | } 116 | 117 | public void SerializeValue (Serializer serializer, U input) { 118 | SerializeValue(serializer, ref input); 119 | } 120 | 121 | public void SerializeValue (Serializer serializer, ref U input) { 122 | var subContext = new SerializationContext(GetKeyOfEntry, ValuePointer, KeyType, _Stream); 123 | serializer(ref subContext, ref input); 124 | if (subContext.BytesProvided) 125 | _Stream.Write(subContext._Bytes.Array, subContext._Bytes.Offset, subContext._Bytes.Count); 126 | } 127 | } 128 | 129 | public unsafe struct DeserializationContext { 130 | private UnmanagedMemoryStream _Stream; 131 | private readonly BTreeValue * ValuePointer; 132 | private readonly GetKeyOfEntryFunc GetKeyOfEntry; 133 | private readonly ushort KeyType; 134 | private TangleKey _Key; 135 | private bool _KeyCached; 136 | 137 | public readonly byte* Source; 138 | public readonly uint SourceLength; 139 | 140 | internal DeserializationContext (GetKeyOfEntryFunc getKeyOfEntry, BTreeValue * pEntry, ushort keyType, byte * source, uint sourceLength) { 141 | GetKeyOfEntry = getKeyOfEntry; 142 | _Key = default(TangleKey); 143 | _KeyCached = false; 144 | KeyType = keyType; 145 | ValuePointer = pEntry; 146 | Source = source; 147 | SourceLength = sourceLength; 148 | _Stream = null; 149 | } 150 | 151 | public TangleKey Key { 152 | get { 153 | if (!_KeyCached) 154 | _KeyCached = GetKeyOfEntry(ValuePointer, KeyType, out _Key); 155 | 156 | if (!_KeyCached) 157 | throw new InvalidDataException(); 158 | 159 | return _Key; 160 | } 161 | } 162 | 163 | public UnmanagedMemoryStream Stream { 164 | get { 165 | if (_Stream == null) 166 | _Stream = new UnmanagedMemoryStream(Source, SourceLength, SourceLength, FileAccess.Read); 167 | 168 | return _Stream; 169 | } 170 | } 171 | 172 | public void DeserializeValue (uint offset, out U output) { 173 | DeserializeValue(offset, SourceLength - offset, out output); 174 | } 175 | 176 | public void DeserializeValue (uint offset, uint length, out U output) { 177 | DeserializeValue(Defaults.Deserializer, offset, length, out output); 178 | } 179 | 180 | public void DeserializeValue (Deserializer deserializer, uint offset, out U output) { 181 | DeserializeValue(deserializer, offset, SourceLength - offset, out output); 182 | } 183 | 184 | public void DeserializeValue (Deserializer deserializer, uint offset, uint length, out U output) { 185 | var subContext = new DeserializationContext(GetKeyOfEntry, ValuePointer, KeyType, Source + offset, length); 186 | try { 187 | deserializer(ref subContext, out output); 188 | } finally { 189 | subContext.Dispose(); 190 | } 191 | } 192 | 193 | internal void Dispose () { 194 | if (_Stream != null) 195 | _Stream.Dispose(); 196 | } 197 | } 198 | 199 | public static class ImmutableBufferPool { 200 | public static ArraySegment GetBytes (byte value) { 201 | var result = ImmutableArrayPool.Allocate(1); 202 | result.Array[result.Offset] = value; 203 | 204 | return result; 205 | } 206 | 207 | public unsafe static ArraySegment GetBytes (short value) { 208 | var result = ImmutableArrayPool.Allocate(2); 209 | 210 | fixed (byte* pBuffer = result.Array) 211 | *(short *)(pBuffer + result.Offset) = value; 212 | 213 | return result; 214 | } 215 | 216 | public unsafe static ArraySegment GetBytes (ushort value) { 217 | var result = ImmutableArrayPool.Allocate(2); 218 | 219 | fixed (byte* pBuffer = result.Array) 220 | *(ushort *)(pBuffer + result.Offset) = value; 221 | 222 | return result; 223 | } 224 | 225 | public unsafe static ArraySegment GetBytes (int value) { 226 | var result = ImmutableArrayPool.Allocate(4); 227 | 228 | fixed (byte * pBuffer = result.Array) 229 | *(int *)(pBuffer + result.Offset) = value; 230 | 231 | return result; 232 | } 233 | 234 | public unsafe static ArraySegment GetBytes (uint value) { 235 | var result = ImmutableArrayPool.Allocate(4); 236 | 237 | fixed (byte * pBuffer = result.Array) 238 | *(uint *)(pBuffer + result.Offset) = value; 239 | 240 | return result; 241 | } 242 | 243 | public unsafe static ArraySegment GetBytes (long value) { 244 | var result = ImmutableArrayPool.Allocate(8); 245 | 246 | fixed (byte * pBuffer = result.Array) 247 | *(long *)(pBuffer + result.Offset) = value; 248 | 249 | return result; 250 | } 251 | 252 | public unsafe static ArraySegment GetBytes (ulong value) { 253 | var result = ImmutableArrayPool.Allocate(8); 254 | 255 | fixed (byte * pBuffer = result.Array) 256 | *(ulong *)(pBuffer + result.Offset) = value; 257 | 258 | return result; 259 | } 260 | 261 | public static ArraySegment GetBytes (string value, Encoding encoding) { 262 | var length = encoding.GetByteCount(value); 263 | var result = ImmutableArrayPool.Allocate(length); 264 | 265 | encoding.GetBytes(value, 0, value.Length, result.Array, result.Offset); 266 | 267 | return result; 268 | } 269 | } 270 | 271 | public static class ImmutableArrayPool { 272 | private class State { 273 | public readonly T[][] Buffers; 274 | public int ElementsUsed, BuffersUsed; 275 | 276 | public State (int capacity) { 277 | Buffers = new T[BufferCount][]; 278 | for (int i = 0; i < Buffers.Length; i++) 279 | Buffers[i] = new T[capacity]; 280 | 281 | ElementsUsed = 0; 282 | BuffersUsed = 0; 283 | } 284 | } 285 | 286 | // The large object heap threshold is roughly 85KB so we set our block size to 64KB 287 | // this ensures that our blocks start in gen0 and can get collected early, instead 288 | // of spending their entire life on the large object heap 289 | public const int MaxSizeBytes = 64 * 1024; 290 | public const int BufferCount = 8; 291 | 292 | public static readonly int Capacity; 293 | 294 | private readonly static ThreadLocal ThreadLocal = new ThreadLocal(); 295 | 296 | static ImmutableArrayPool () { 297 | var itemSize = Marshal.SizeOf(typeof(T)); 298 | Capacity = MaxSizeBytes / itemSize; 299 | } 300 | 301 | public static ArraySegment Allocate (int count) { 302 | if (count > Capacity) 303 | return new ArraySegment(new T[count], 0, count); 304 | 305 | var data = ThreadLocal.Value; 306 | 307 | bool usedUpElements = false; 308 | bool allocateNew = (data == null) || 309 | ((usedUpElements = (data.ElementsUsed >= Capacity - count)) && 310 | (data.BuffersUsed >= BufferCount - 1)); 311 | 312 | if (allocateNew) { 313 | data = ThreadLocal.Value = new State(Capacity); 314 | usedUpElements = false; 315 | } 316 | 317 | if (usedUpElements) { 318 | data.ElementsUsed = 0; 319 | data.BuffersUsed += 1; 320 | } 321 | 322 | var offset = data.ElementsUsed; 323 | data.ElementsUsed += count; 324 | 325 | return new ArraySegment(data.Buffers[data.BuffersUsed], offset, count); 326 | } 327 | } 328 | } 329 | 330 | namespace Squared.Data.Mangler.Serialization { 331 | /// 332 | /// Handles converting a single value from the Tangle into raw binary for storage. 333 | /// 334 | public delegate void Serializer (ref SerializationContext context, ref T input); 335 | /// 336 | /// Handles converting a single stored value from raw binary back into its native format, when the Tangle is loading it from storage. 337 | /// 338 | public unsafe delegate void Deserializer (ref DeserializationContext context, out T output); 339 | 340 | public class StringSerializer { 341 | public readonly Encoding Encoding; 342 | public readonly Deserializer Deserialize; 343 | public readonly Serializer Serialize; 344 | 345 | public StringSerializer (Encoding encoding = null) { 346 | Encoding = encoding ?? Encoding.UTF8; 347 | Serialize = _Serialize; 348 | Deserialize = _Deserialize; 349 | } 350 | 351 | private void _Serialize (ref SerializationContext context, ref string input) { 352 | var bytes = Encoding.GetBytes(input); 353 | context.Stream.Write(bytes, 0, bytes.Length); 354 | } 355 | 356 | private unsafe void _Deserialize (ref DeserializationContext context, out string output) { 357 | output = new String((sbyte *)context.Source, 0, (int)context.SourceLength, Encoding); 358 | } 359 | } 360 | 361 | public static class BlittableSerializer 362 | where T : struct { 363 | 364 | public static readonly uint Size; 365 | public static readonly Serializer Serialize; 366 | public static readonly Deserializer Deserialize; 367 | 368 | static BlittableSerializer () { 369 | Size = (uint)Marshal.SizeOf(typeof(T)); 370 | Serialize = _Serialize; 371 | Deserialize = _Deserialize; 372 | } 373 | 374 | static unsafe void _Serialize (ref SerializationContext context, ref T input) { 375 | var buffer = ImmutableArrayPool.Allocate((int)Size); 376 | fixed (byte * pBuffer = buffer.Array) 377 | Unsafe.StructureToPtr(ref input, pBuffer + buffer.Offset, Size); 378 | 379 | context.Stream.Write(buffer.Array, buffer.Offset, (int)Size); 380 | } 381 | 382 | static unsafe void _Deserialize (ref DeserializationContext context, out T output) { 383 | Unsafe.PtrToStructure(context.Source, out output, context.SourceLength); 384 | } 385 | } 386 | 387 | public static class Defaults { 388 | public static Serializer Serializer = SerializeWithBinaryFormatter; 389 | public static Deserializer Deserializer = DeserializeWithBinaryFormatter; 390 | 391 | unsafe static Defaults () { 392 | var t = typeof(T); 393 | var tsa = typeof(TangleSerializerAttribute); 394 | var tda = typeof(TangleDeserializerAttribute); 395 | 396 | foreach (var method in t.GetMethods( 397 | System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.Public | 398 | System.Reflection.BindingFlags.NonPublic 399 | )) { 400 | var sa = method.GetCustomAttributes(tsa, true); 401 | if (sa.Length == 1) 402 | Serializer = (Serializer)(Delegate.CreateDelegate(typeof(Serializer), method, true)) ?? Serializer; 403 | 404 | sa = method.GetCustomAttributes(tda, true); 405 | if (sa.Length == 1) 406 | Deserializer = (Deserializer)(Delegate.CreateDelegate(typeof(Deserializer), method, true)) ?? Deserializer; 407 | } 408 | } 409 | 410 | public static void SerializeWithBinaryFormatter (ref SerializationContext context, ref T input) { 411 | var formatter = new BinaryFormatter(); 412 | formatter.Serialize(context.Stream, input); 413 | } 414 | 415 | public static void DeserializeWithBinaryFormatter (ref DeserializationContext context, out T output) { 416 | var formatter = new BinaryFormatter(); 417 | output = (T)formatter.Deserialize(context.Stream); 418 | } 419 | 420 | public static void SerializeToXml (ref SerializationContext context, ref T input) { 421 | var serializeType = typeof(T); 422 | if (input != null) 423 | serializeType = input.GetType(); 424 | 425 | var ser = new XmlSerializer(serializeType); 426 | ser.Serialize(context.Stream, input); 427 | } 428 | 429 | public static void DeserializeFromXml (ref DeserializationContext context, out T output) { 430 | if (typeof(T) == typeof(object)) 431 | throw new InvalidOperationException("Cannot deserialize XML to type Object. Provide a specific type."); 432 | 433 | var ser = new XmlSerializer(typeof(T)); 434 | output = (T)ser.Deserialize(context.Stream); 435 | } 436 | } 437 | } 438 | -------------------------------------------------------------------------------- /StreamRef.cs: -------------------------------------------------------------------------------- 1 | /* 2 | The contents of this file are subject to the Mozilla Public License 3 | Version 1.1 (the "License"); you may not use this file except in 4 | compliance with the License. You may obtain a copy of the License at 5 | http://www.mozilla.org/MPL/ 6 | 7 | Software distributed under the License is distributed on an "AS IS" 8 | basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the 9 | License for the specific language governing rights and limitations 10 | under the License. 11 | 12 | The Original Code is DataMangler Key-Value Store. 13 | 14 | The Initial Developer of the Original Code is Mozilla Corporation. 15 | 16 | Original Author: Kevin Gadd (kevin.gadd@gmail.com) 17 | */ 18 | 19 | using System; 20 | using System.Diagnostics; 21 | using System.IO.MemoryMappedFiles; 22 | using System.Threading; 23 | using System.Runtime.InteropServices; 24 | using System.IO; 25 | using Microsoft.Win32.SafeHandles; 26 | using System.Collections.Generic; 27 | using System.Security; 28 | using System.Collections.Concurrent; 29 | using Squared.Util; 30 | 31 | namespace Squared.Data.Mangler.Internal { 32 | [StructLayout(LayoutKind.Sequential, Pack = 1, Size = 256)] 33 | internal unsafe struct StreamHeader { 34 | public uint FormatVersion; 35 | public long DataLength; 36 | } 37 | 38 | internal unsafe struct StreamHeaderRef : IDisposable { 39 | public readonly StreamHeader* Ptr; 40 | 41 | private readonly MemoryMappedViewAccessor Accessor; 42 | private readonly SafeBuffer Buffer; 43 | 44 | public StreamHeaderRef (MemoryMappedViewAccessor accessor) { 45 | Accessor = accessor; 46 | Buffer = accessor.GetSafeBuffer(); 47 | 48 | byte* temp = null; 49 | Buffer.AcquirePointer(ref temp); 50 | 51 | Ptr = (StreamHeader*)temp; 52 | } 53 | 54 | public void Dispose () { 55 | Buffer.ReleasePointer(); 56 | } 57 | } 58 | 59 | internal unsafe struct StreamRange : IDisposable { 60 | public readonly StreamRef Stream; 61 | 62 | public readonly byte* Pointer; 63 | 64 | public readonly long Offset, Size; 65 | 66 | private bool IsDisposed; 67 | private readonly SafeBuffer Buffer; 68 | private readonly MemoryMappedViewAccessor View; 69 | private readonly ViewCache.CacheEntry CacheEntry; 70 | 71 | public StreamRange (StreamRef stream, MemoryMappedViewAccessor view, long offset, uint size, long actualOffset, long actualSize) { 72 | Stream = stream; 73 | View = view; 74 | CacheEntry = default(ViewCache.CacheEntry); 75 | Offset = offset; 76 | Size = size; 77 | Buffer = view.GetSafeBuffer(); 78 | Pointer = null; 79 | Buffer.AcquirePointer(ref Pointer); 80 | IsDisposed = false; 81 | unchecked { 82 | Pointer += view.GetPointerOffset(); 83 | Pointer += (offset - actualOffset); 84 | } 85 | } 86 | 87 | public StreamRange (StreamRef stream, ViewCache.CacheEntry cacheEntry, long offset, uint size) { 88 | Stream = stream; 89 | CacheEntry = cacheEntry; 90 | View = null; 91 | Offset = offset; 92 | Size = size; 93 | Buffer = cacheEntry.Buffer; 94 | IsDisposed = false; 95 | unchecked { 96 | Pointer = cacheEntry.Pointer + cacheEntry.PointerOffset; 97 | Pointer += (offset - cacheEntry.Offset); 98 | } 99 | } 100 | 101 | public void Dispose () { 102 | if (IsDisposed) 103 | return; 104 | 105 | IsDisposed = true; 106 | if (View != null) { 107 | Buffer.ReleasePointer(); 108 | 109 | // https://connect.microsoft.com/VisualStudio/feedback/details/552859/memorymappedviewaccessor-flush-throws-ioexception 110 | // View.Dispose(); 111 | View.SafeMemoryMappedViewHandle.Dispose(); 112 | } else if (CacheEntry != null) { 113 | CacheEntry.RemoveRef(); 114 | } else { 115 | // Uninitialized 116 | } 117 | } 118 | 119 | public bool IsValid { 120 | get { 121 | if (IsDisposed) 122 | return false; 123 | 124 | return ((View != null) && (Buffer != null)) || (CacheEntry != null); 125 | } 126 | } 127 | } 128 | 129 | [Flags] 130 | public enum NativeFileAccess : uint { 131 | GenericRead = 0x80000000, 132 | GenericWrite = 0x40000000 133 | } 134 | 135 | [Flags] 136 | public enum NativeFileFlags : uint { 137 | WriteThrough = 0x80000000, 138 | Overlapped = 0x40000000, 139 | NoBuffering = 0x20000000, 140 | RandomAccess = 0x10000000, 141 | SequentialScan = 0x8000000, 142 | DeleteOnClose = 0x4000000, 143 | BackupSemantics = 0x2000000, 144 | PosixSemantics = 0x1000000, 145 | OpenReparsePoint = 0x200000, 146 | OpenNoRecall = 0x100000 147 | } 148 | 149 | public static class Native { 150 | [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] 151 | public static extern SafeFileHandle CreateFile ( 152 | string filename, 153 | NativeFileAccess access, 154 | FileShare share, 155 | IntPtr security, 156 | FileMode mode, 157 | NativeFileFlags flags, 158 | IntPtr template 159 | ); 160 | 161 | [DllImport("msvcrt.dll", CallingConvention = CallingConvention.Cdecl)] 162 | [SuppressUnmanagedCodeSecurity] 163 | public static extern unsafe int memcmp (byte * lhs, byte * rhs, UIntPtr count); 164 | 165 | [DllImport("msvcrt.dll", CallingConvention = CallingConvention.Cdecl)] 166 | [SuppressUnmanagedCodeSecurity] 167 | public static extern unsafe int memmove (byte* dest, byte* src, UIntPtr count); 168 | 169 | [DllImport("msvcrt.dll", CallingConvention = CallingConvention.Cdecl)] 170 | [SuppressUnmanagedCodeSecurity] 171 | public static extern unsafe int memset (byte* dest, int value, UIntPtr count); 172 | 173 | public static FileStream OpenAlternateStream (string filename, string streamName) { 174 | const string prefix = @"\\?\"; 175 | var path = String.Format("{0}{1}:{2}", prefix, filename, streamName); 176 | var handle = Native.CreateFile( 177 | path, 178 | NativeFileAccess.GenericRead | NativeFileAccess.GenericWrite, 179 | FileShare.Delete | FileShare.ReadWrite, IntPtr.Zero, FileMode.OpenOrCreate, 180 | NativeFileFlags.RandomAccess, IntPtr.Zero 181 | ); 182 | if (handle.IsInvalid || handle.IsClosed) 183 | throw new IOException("Could not open stream " + path); 184 | return new FileStream(handle, FileAccess.ReadWrite); 185 | } 186 | } 187 | 188 | internal class ViewCache : IDisposable { 189 | // Views are scaled up to this size when possible to reduce time spent mapping/unmapping pages. 190 | // If this is set too high you risk exhausting available address space. 191 | // If it's set too low the cache will not be particularly effective. 192 | public const uint ChunkSize = (1024 * 1024) * 16; 193 | public const int MaximumStorageFailures = 2; 194 | 195 | public unsafe class CacheEntry : IDisposable { 196 | public readonly long CreatedWhen; 197 | public readonly SafeBuffer Buffer; 198 | public readonly byte* Pointer; 199 | public readonly long PointerOffset; 200 | public readonly long Offset; 201 | public readonly uint Size; 202 | public readonly MemoryMappedViewAccessor View; 203 | 204 | internal bool IsDisposed; 205 | private int _RefCount; 206 | 207 | public CacheEntry (MemoryMappedViewAccessor view, long offset, uint size) { 208 | CreatedWhen = Time.Ticks; 209 | View = view; 210 | Offset = offset; 211 | Size = size; 212 | IsDisposed = false; 213 | _RefCount = 1; 214 | Buffer = view.GetSafeBuffer(); 215 | Pointer = null; 216 | Buffer.AcquirePointer(ref Pointer); 217 | PointerOffset = view.GetPointerOffset(); 218 | } 219 | 220 | public bool AddRef () { 221 | Interlocked.Increment(ref _RefCount); 222 | 223 | if (IsDisposed) 224 | return false; 225 | else 226 | return true; 227 | } 228 | 229 | public void RemoveRef () { 230 | if (IsDisposed) { 231 | // This can happen if the stream is grown while a reference is held to one of its views. 232 | // In this case, we don't want using() blocks and finally handlers to throw exceptions. 233 | 234 | if (Interlocked.Decrement(ref _RefCount) < 0) 235 | throw new ObjectDisposedException("CacheEntry"); 236 | 237 | return; 238 | } 239 | 240 | if (Interlocked.Decrement(ref _RefCount) == 0) 241 | Release(); 242 | } 243 | 244 | void IDisposable.Dispose () { 245 | RemoveRef(); 246 | } 247 | 248 | internal void Release () { 249 | IsDisposed = true; 250 | Buffer.ReleasePointer(); 251 | 252 | // https://connect.microsoft.com/VisualStudio/feedback/details/552859/memorymappedviewaccessor-flush-throws-ioexception 253 | // View.Dispose(); 254 | View.SafeMemoryMappedViewHandle.Dispose(); 255 | } 256 | } 257 | 258 | protected static readonly HashSet Caches = new HashSet(); 259 | 260 | public readonly MemoryMappedFile File; 261 | public readonly long FileLength; 262 | public readonly int Capacity; 263 | private readonly WeakReference WeakSelf; 264 | private readonly CacheEntry[] Cache; 265 | private bool _IsDisposed; 266 | 267 | public static void EmergencyFlush () { 268 | WeakReference[] caches; 269 | lock (Caches) { 270 | caches = new WeakReference[Caches.Count]; 271 | Caches.CopyTo(caches); 272 | } 273 | 274 | var process = Process.GetCurrentProcess(); 275 | process.Refresh(); 276 | var memoryBefore = process.VirtualMemorySize64; 277 | 278 | foreach (var wr in caches) { 279 | var cache = wr.Target as ViewCache; 280 | if (cache == null) 281 | continue; 282 | 283 | cache.Flush(); 284 | } 285 | 286 | GC.Collect(); 287 | 288 | process.Refresh(); 289 | var memoryAfter = process.VirtualMemorySize64; 290 | 291 | Debug.WriteLine("Failure to map region triggered an emergency cache flush. Freed {0:00000.0} KB of address space.", (memoryBefore - memoryAfter) / 1024.0); 292 | 293 | Thread.Sleep(50); 294 | } 295 | 296 | public ViewCache (MemoryMappedFile file, long fileLength, int capacity = 4) { 297 | File = file; 298 | FileLength = fileLength; 299 | Capacity = capacity; 300 | Cache = new CacheEntry[capacity]; 301 | _IsDisposed = false; 302 | 303 | WeakSelf = new WeakReference(this); 304 | 305 | lock (Caches) 306 | Caches.Add(WeakSelf); 307 | } 308 | 309 | public MemoryMappedViewAccessor CreateViewUncached (long offset, uint size, MemoryMappedFileAccess access, out long actualOffset, out uint actualSize) { 310 | unchecked { 311 | actualOffset = (offset / ChunkSize * ChunkSize); 312 | if (actualOffset < 0) 313 | actualOffset = 0; 314 | 315 | var actualEnd = ((offset + size) + (ChunkSize - 1)) / ChunkSize * ChunkSize; 316 | if (actualEnd < actualOffset) 317 | actualEnd = actualOffset; 318 | if (actualEnd >= FileLength) 319 | actualEnd = FileLength; 320 | 321 | actualSize = (uint)(actualEnd - actualOffset); 322 | 323 | int failCount = 0; 324 | while (true) { 325 | try { 326 | return File.CreateViewAccessor(actualOffset, actualSize, access); 327 | } catch (IOException ex) { 328 | if (ex.Message.Contains("Not enough storage")) { 329 | failCount += 1; 330 | 331 | if (failCount <= MaximumStorageFailures) { 332 | EmergencyFlush(); 333 | } else { 334 | throw; 335 | } 336 | } else { 337 | throw; 338 | } 339 | } 340 | } 341 | } 342 | } 343 | 344 | public CacheEntry CreateView (long offset, uint size, MemoryMappedFileAccess access) { 345 | if (_IsDisposed) 346 | throw new ObjectDisposedException("Cache"); 347 | 348 | if (access == MemoryMappedFileAccess.Write) 349 | access = MemoryMappedFileAccess.ReadWrite; 350 | 351 | int? freeSlot = null; 352 | int? oldestUsedSlot = null; 353 | long oldestUsedTimestamp = long.MaxValue; 354 | 355 | for (int i = 0; i < Capacity; i++) { 356 | var item = Cache[i]; 357 | 358 | if (item == null || item.IsDisposed) { 359 | freeSlot = i; 360 | continue; 361 | } 362 | 363 | if (item.CreatedWhen < oldestUsedTimestamp) 364 | oldestUsedSlot = i; 365 | 366 | if (offset < item.Offset) 367 | continue; 368 | if (offset + size > item.Offset + item.Size) 369 | continue; 370 | 371 | if (item.AddRef()) 372 | return item; 373 | else { 374 | freeSlot = i; 375 | break; 376 | } 377 | } 378 | 379 | if (!freeSlot.HasValue && !oldestUsedSlot.HasValue) 380 | throw new InvalidDataException(); 381 | 382 | long actualOffset; 383 | uint actualSize; 384 | var view = CreateViewUncached(offset, size, MemoryMappedFileAccess.ReadWrite, out actualOffset, out actualSize); 385 | 386 | var newEntry = new CacheEntry(view, actualOffset, actualSize); 387 | newEntry.AddRef(); 388 | 389 | long slotIndex = freeSlot.GetValueOrDefault(oldestUsedSlot.GetValueOrDefault(0)); 390 | var oldEntry = Interlocked.Exchange(ref Cache[slotIndex], newEntry); 391 | 392 | if (oldEntry != null) 393 | oldEntry.RemoveRef(); 394 | 395 | return newEntry; 396 | } 397 | 398 | public void Flush () { 399 | if (_IsDisposed) 400 | return; 401 | 402 | for (int i = 0; i < Capacity; i++) { 403 | var ce = Interlocked.Exchange(ref Cache[i], null); 404 | 405 | if (ce != null) 406 | ce.RemoveRef(); 407 | } 408 | } 409 | 410 | public void Dispose () { 411 | lock (Caches) 412 | Caches.Remove(WeakSelf); 413 | 414 | Flush(); 415 | _IsDisposed = true; 416 | } 417 | 418 | public bool IsDisposed { 419 | get { 420 | return _IsDisposed; 421 | } 422 | } 423 | } 424 | 425 | internal class StreamRef : IDisposable { 426 | public static readonly uint HeaderSize = (uint)Marshal.SizeOf(typeof(StreamHeader)); 427 | public const uint InitialSize = 256 * 1024; 428 | public const uint DoublingThreshold = 1024 * 1024 * 64; 429 | public const uint PostDoublingGrowthRate = 1024 * 1024 * 8; 430 | 431 | public event EventHandler LengthChanging, LengthChanged; 432 | 433 | protected ViewCache Cache; 434 | protected MemoryMappedFile Handle; 435 | protected MemoryMappedViewAccessor HeaderView; 436 | 437 | public readonly FileStream NativeStream; 438 | public readonly bool OwnsStream; 439 | 440 | protected long StreamCapacity; 441 | 442 | public StreamRef (FileStream nativeStream, bool ownsStream = true) { 443 | NativeStream = nativeStream; 444 | OwnsStream = ownsStream; 445 | 446 | CreateHandles(InitialSize); 447 | } 448 | 449 | protected void CreateHandles (long capacity) { 450 | if (NativeStream.Length > capacity) 451 | capacity = NativeStream.Length; 452 | 453 | Handle = MemoryMappedFile.CreateFromFile( 454 | NativeStream, null, capacity, 455 | MemoryMappedFileAccess.ReadWrite, 456 | null, HandleInheritability.None, true 457 | ); 458 | HeaderView = Handle.CreateViewAccessor(0, HeaderSize); 459 | StreamCapacity = capacity; 460 | Cache = new ViewCache(Handle, StreamCapacity); 461 | } 462 | 463 | internal unsafe StreamHeaderRef AccessHeader () { 464 | StreamHeaderRef result; 465 | 466 | result = new StreamHeaderRef(HeaderView); 467 | 468 | return result; 469 | } 470 | 471 | protected void EnsureCapacity (long capacity) { 472 | if (capacity <= StreamCapacity) 473 | return; 474 | 475 | // We grow the stream by a fixed amount every time we run out 476 | // of space. Doubling or some other algorithm might be better, 477 | // but this is simple and predictable. 478 | long newCapacity = StreamCapacity; 479 | if (StreamCapacity >= DoublingThreshold) { 480 | while (newCapacity < capacity) 481 | newCapacity += PostDoublingGrowthRate; 482 | } else { 483 | while (newCapacity < capacity) 484 | newCapacity *= 2; 485 | } 486 | 487 | if (LengthChanging != null) 488 | LengthChanging(this, EventArgs.Empty); 489 | 490 | DisposeViews(); 491 | 492 | CreateHandles(newCapacity); 493 | 494 | if (LengthChanged != null) 495 | LengthChanged(this, EventArgs.Empty); 496 | } 497 | 498 | /// 499 | /// Allocates byte(s) of unused space at the end of the stream. 500 | /// 501 | /// The number of bytes to allocate. 502 | /// The offset into the stream where the allocated bytes are located. 503 | public unsafe long? AllocateSpace (uint size) { 504 | if (size == 0) 505 | return null; 506 | 507 | long oldSize, newSize; 508 | // This is thread-safe, but because we bump the DataLength without 509 | // making any effort to ensure the data in the region is valid, 510 | // other threads may attempt to read it and find random garbage 511 | // there. 512 | // On the bright side, MSDN claims that unused regions in a mapped 513 | // file are always zeroes, and this seems to be true so far. Given 514 | // this, most of the time you just need a 'this data is valid' bit 515 | // tucked away to protect yourself from reading uninitialized data. 516 | using (var header = AccessHeader()) { 517 | newSize = Interlocked.Add(ref header.Ptr->DataLength, size); 518 | oldSize = newSize - size; 519 | } 520 | 521 | EnsureCapacity(newSize + HeaderSize); 522 | return oldSize; 523 | } 524 | 525 | public unsafe long Shrink (int size) { 526 | using (var header = AccessHeader()) 527 | return Interlocked.Add(ref header.Ptr->DataLength, -size); 528 | } 529 | 530 | public unsafe void Clear () { 531 | using (var header = AccessHeader()) 532 | header.Ptr->DataLength = 0; 533 | } 534 | 535 | public unsafe uint FormatVersion { 536 | get { 537 | using (var header = AccessHeader()) 538 | return header.Ptr->FormatVersion; 539 | } 540 | set { 541 | using (var header = AccessHeader()) 542 | header.Ptr->FormatVersion = value; 543 | } 544 | } 545 | 546 | public unsafe long Length { 547 | get { 548 | using (var header = AccessHeader()) 549 | return header.Ptr->DataLength; 550 | } 551 | } 552 | 553 | public long Capacity { 554 | get { 555 | return Math.Max(StreamCapacity, NativeStream.Length); 556 | } 557 | } 558 | 559 | /// 560 | /// Accesses a range of bytes within the stream. 561 | /// 562 | /// The offset within the stream, relative to the end of the stream header. 563 | /// The size of the range to access, in bytes. 564 | public StreamRange AccessRange (long offset, uint size, MemoryMappedFileAccess access = MemoryMappedFileAccess.ReadWrite) { 565 | unchecked { 566 | long actualBegin = offset + HeaderSize; 567 | uint actualSize = size; 568 | 569 | EnsureCapacity(HeaderSize + offset + actualSize); 570 | 571 | var viewRef = Cache.CreateView(actualBegin, actualSize, access); 572 | 573 | return new StreamRange( 574 | this, viewRef, actualBegin, actualSize 575 | ); 576 | } 577 | } 578 | 579 | /// 580 | /// Accesses a range of bytes within the stream, bypassing the stream cache. 581 | /// 582 | /// The offset within the stream, relative to the end of the stream header. 583 | /// The size of the range to access, in bytes. 584 | public StreamRange AccessRangeUncached (long offset, uint size, MemoryMappedFileAccess access = MemoryMappedFileAccess.ReadWrite) { 585 | unchecked { 586 | long relativeOffset = offset + HeaderSize; 587 | 588 | EnsureCapacity(relativeOffset + size); 589 | 590 | long actualOffset; 591 | uint actualSize; 592 | var view = Cache.CreateViewUncached(relativeOffset, size, access, out actualOffset, out actualSize); 593 | 594 | return new StreamRange( 595 | this, view, relativeOffset, size, actualOffset, actualSize 596 | ); 597 | } 598 | } 599 | 600 | public void FlushCache () { 601 | if (Cache != null) 602 | Cache.Flush(); 603 | } 604 | 605 | private void DisposeViews () { 606 | if (Cache != null) { 607 | Cache.Dispose(); 608 | Cache = null; 609 | } 610 | if (HeaderView != null) { 611 | // https://connect.microsoft.com/VisualStudio/feedback/details/552859/memorymappedviewaccessor-flush-throws-ioexception 612 | // HeaderView.Dispose(); 613 | HeaderView.SafeMemoryMappedViewHandle.Dispose(); 614 | HeaderView = null; 615 | } 616 | if (Handle != null) { 617 | Handle.Dispose(); 618 | Handle = null; 619 | } 620 | } 621 | 622 | protected unsafe long GetTotalLength () { 623 | using (var header = AccessHeader()) 624 | return header.Ptr->DataLength + HeaderSize; 625 | } 626 | 627 | public void Dispose () { 628 | long totalLength = GetTotalLength(); 629 | 630 | if (LengthChanging != null) 631 | LengthChanging(this, EventArgs.Empty); 632 | 633 | DisposeViews(); 634 | 635 | NativeStream.SetLength(totalLength); 636 | if (OwnsStream) 637 | NativeStream.Dispose(); 638 | } 639 | } 640 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MOZILLA PUBLIC LICENSE 2 | Version 1.1 3 | 4 | --------------- 5 | 6 | 1. Definitions. 7 | 8 | 1.0.1. "Commercial Use" means distribution or otherwise making the 9 | Covered Code available to a third party. 10 | 11 | 1.1. "Contributor" means each entity that creates or contributes to 12 | the creation of Modifications. 13 | 14 | 1.2. "Contributor Version" means the combination of the Original 15 | Code, prior Modifications used by a Contributor, and the Modifications 16 | made by that particular Contributor. 17 | 18 | 1.3. "Covered Code" means the Original Code or Modifications or the 19 | combination of the Original Code and Modifications, in each case 20 | including portions thereof. 21 | 22 | 1.4. "Electronic Distribution Mechanism" means a mechanism generally 23 | accepted in the software development community for the electronic 24 | transfer of data. 25 | 26 | 1.5. "Executable" means Covered Code in any form other than Source 27 | Code. 28 | 29 | 1.6. "Initial Developer" means the individual or entity identified 30 | as the Initial Developer in the Source Code notice required by Exhibit 31 | A. 32 | 33 | 1.7. "Larger Work" means a work which combines Covered Code or 34 | portions thereof with code not governed by the terms of this License. 35 | 36 | 1.8. "License" means this document. 37 | 38 | 1.8.1. "Licensable" means having the right to grant, to the maximum 39 | extent possible, whether at the time of the initial grant or 40 | subsequently acquired, any and all of the rights conveyed herein. 41 | 42 | 1.9. "Modifications" means any addition to or deletion from the 43 | substance or structure of either the Original Code or any previous 44 | Modifications. When Covered Code is released as a series of files, a 45 | Modification is: 46 | A. Any addition to or deletion from the contents of a file 47 | containing Original Code or previous Modifications. 48 | 49 | B. Any new file that contains any part of the Original Code or 50 | previous Modifications. 51 | 52 | 1.10. "Original Code" means Source Code of computer software code 53 | which is described in the Source Code notice required by Exhibit A as 54 | Original Code, and which, at the time of its release under this 55 | License is not already Covered Code governed by this License. 56 | 57 | 1.10.1. "Patent Claims" means any patent claim(s), now owned or 58 | hereafter acquired, including without limitation, method, process, 59 | and apparatus claims, in any patent Licensable by grantor. 60 | 61 | 1.11. "Source Code" means the preferred form of the Covered Code for 62 | making modifications to it, including all modules it contains, plus 63 | any associated interface definition files, scripts used to control 64 | compilation and installation of an Executable, or source code 65 | differential comparisons against either the Original Code or another 66 | well known, available Covered Code of the Contributor's choice. The 67 | Source Code can be in a compressed or archival form, provided the 68 | appropriate decompression or de-archiving software is widely available 69 | for no charge. 70 | 71 | 1.12. "You" (or "Your") means an individual or a legal entity 72 | exercising rights under, and complying with all of the terms of, this 73 | License or a future version of this License issued under Section 6.1. 74 | For legal entities, "You" includes any entity which controls, is 75 | controlled by, or is under common control with You. For purposes of 76 | this definition, "control" means (a) the power, direct or indirect, 77 | to cause the direction or management of such entity, whether by 78 | contract or otherwise, or (b) ownership of more than fifty percent 79 | (50%) of the outstanding shares or beneficial ownership of such 80 | entity. 81 | 82 | 2. Source Code License. 83 | 84 | 2.1. The Initial Developer Grant. 85 | The Initial Developer hereby grants You a world-wide, royalty-free, 86 | non-exclusive license, subject to third party intellectual property 87 | claims: 88 | (a) under intellectual property rights (other than patent or 89 | trademark) Licensable by Initial Developer to use, reproduce, 90 | modify, display, perform, sublicense and distribute the Original 91 | Code (or portions thereof) with or without Modifications, and/or 92 | as part of a Larger Work; and 93 | 94 | (b) under Patents Claims infringed by the making, using or 95 | selling of Original Code, to make, have made, use, practice, 96 | sell, and offer for sale, and/or otherwise dispose of the 97 | Original Code (or portions thereof). 98 | 99 | (c) the licenses granted in this Section 2.1(a) and (b) are 100 | effective on the date Initial Developer first distributes 101 | Original Code under the terms of this License. 102 | 103 | (d) Notwithstanding Section 2.1(b) above, no patent license is 104 | granted: 1) for code that You delete from the Original Code; 2) 105 | separate from the Original Code; or 3) for infringements caused 106 | by: i) the modification of the Original Code or ii) the 107 | combination of the Original Code with other software or devices. 108 | 109 | 2.2. Contributor Grant. 110 | Subject to third party intellectual property claims, each Contributor 111 | hereby grants You a world-wide, royalty-free, non-exclusive license 112 | 113 | (a) under intellectual property rights (other than patent or 114 | trademark) Licensable by Contributor, to use, reproduce, modify, 115 | display, perform, sublicense and distribute the Modifications 116 | created by such Contributor (or portions thereof) either on an 117 | unmodified basis, with other Modifications, as Covered Code 118 | and/or as part of a Larger Work; and 119 | 120 | (b) under Patent Claims infringed by the making, using, or 121 | selling of Modifications made by that Contributor either alone 122 | and/or in combination with its Contributor Version (or portions 123 | of such combination), to make, use, sell, offer for sale, have 124 | made, and/or otherwise dispose of: 1) Modifications made by that 125 | Contributor (or portions thereof); and 2) the combination of 126 | Modifications made by that Contributor with its Contributor 127 | Version (or portions of such combination). 128 | 129 | (c) the licenses granted in Sections 2.2(a) and 2.2(b) are 130 | effective on the date Contributor first makes Commercial Use of 131 | the Covered Code. 132 | 133 | (d) Notwithstanding Section 2.2(b) above, no patent license is 134 | granted: 1) for any code that Contributor has deleted from the 135 | Contributor Version; 2) separate from the Contributor Version; 136 | 3) for infringements caused by: i) third party modifications of 137 | Contributor Version or ii) the combination of Modifications made 138 | by that Contributor with other software (except as part of the 139 | Contributor Version) or other devices; or 4) under Patent Claims 140 | infringed by Covered Code in the absence of Modifications made by 141 | that Contributor. 142 | 143 | 3. Distribution Obligations. 144 | 145 | 3.1. Application of License. 146 | The Modifications which You create or to which You contribute are 147 | governed by the terms of this License, including without limitation 148 | Section 2.2. The Source Code version of Covered Code may be 149 | distributed only under the terms of this License or a future version 150 | of this License released under Section 6.1, and You must include a 151 | copy of this License with every copy of the Source Code You 152 | distribute. You may not offer or impose any terms on any Source Code 153 | version that alters or restricts the applicable version of this 154 | License or the recipients' rights hereunder. However, You may include 155 | an additional document offering the additional rights described in 156 | Section 3.5. 157 | 158 | 3.2. Availability of Source Code. 159 | Any Modification which You create or to which You contribute must be 160 | made available in Source Code form under the terms of this License 161 | either on the same media as an Executable version or via an accepted 162 | Electronic Distribution Mechanism to anyone to whom you made an 163 | Executable version available; and if made available via Electronic 164 | Distribution Mechanism, must remain available for at least twelve (12) 165 | months after the date it initially became available, or at least six 166 | (6) months after a subsequent version of that particular Modification 167 | has been made available to such recipients. You are responsible for 168 | ensuring that the Source Code version remains available even if the 169 | Electronic Distribution Mechanism is maintained by a third party. 170 | 171 | 3.3. Description of Modifications. 172 | You must cause all Covered Code to which You contribute to contain a 173 | file documenting the changes You made to create that Covered Code and 174 | the date of any change. You must include a prominent statement that 175 | the Modification is derived, directly or indirectly, from Original 176 | Code provided by the Initial Developer and including the name of the 177 | Initial Developer in (a) the Source Code, and (b) in any notice in an 178 | Executable version or related documentation in which You describe the 179 | origin or ownership of the Covered Code. 180 | 181 | 3.4. Intellectual Property Matters 182 | (a) Third Party Claims. 183 | If Contributor has knowledge that a license under a third party's 184 | intellectual property rights is required to exercise the rights 185 | granted by such Contributor under Sections 2.1 or 2.2, 186 | Contributor must include a text file with the Source Code 187 | distribution titled "LEGAL" which describes the claim and the 188 | party making the claim in sufficient detail that a recipient will 189 | know whom to contact. If Contributor obtains such knowledge after 190 | the Modification is made available as described in Section 3.2, 191 | Contributor shall promptly modify the LEGAL file in all copies 192 | Contributor makes available thereafter and shall take other steps 193 | (such as notifying appropriate mailing lists or newsgroups) 194 | reasonably calculated to inform those who received the Covered 195 | Code that new knowledge has been obtained. 196 | 197 | (b) Contributor APIs. 198 | If Contributor's Modifications include an application programming 199 | interface and Contributor has knowledge of patent licenses which 200 | are reasonably necessary to implement that API, Contributor must 201 | also include this information in the LEGAL file. 202 | 203 | (c) Representations. 204 | Contributor represents that, except as disclosed pursuant to 205 | Section 3.4(a) above, Contributor believes that Contributor's 206 | Modifications are Contributor's original creation(s) and/or 207 | Contributor has sufficient rights to grant the rights conveyed by 208 | this License. 209 | 210 | 3.5. Required Notices. 211 | You must duplicate the notice in Exhibit A in each file of the Source 212 | Code. If it is not possible to put such notice in a particular Source 213 | Code file due to its structure, then You must include such notice in a 214 | location (such as a relevant directory) where a user would be likely 215 | to look for such a notice. If You created one or more Modification(s) 216 | You may add your name as a Contributor to the notice described in 217 | Exhibit A. You must also duplicate this License in any documentation 218 | for the Source Code where You describe recipients' rights or ownership 219 | rights relating to Covered Code. You may choose to offer, and to 220 | charge a fee for, warranty, support, indemnity or liability 221 | obligations to one or more recipients of Covered Code. However, You 222 | may do so only on Your own behalf, and not on behalf of the Initial 223 | Developer or any Contributor. You must make it absolutely clear than 224 | any such warranty, support, indemnity or liability obligation is 225 | offered by You alone, and You hereby agree to indemnify the Initial 226 | Developer and every Contributor for any liability incurred by the 227 | Initial Developer or such Contributor as a result of warranty, 228 | support, indemnity or liability terms You offer. 229 | 230 | 3.6. Distribution of Executable Versions. 231 | You may distribute Covered Code in Executable form only if the 232 | requirements of Section 3.1-3.5 have been met for that Covered Code, 233 | and if You include a notice stating that the Source Code version of 234 | the Covered Code is available under the terms of this License, 235 | including a description of how and where You have fulfilled the 236 | obligations of Section 3.2. The notice must be conspicuously included 237 | in any notice in an Executable version, related documentation or 238 | collateral in which You describe recipients' rights relating to the 239 | Covered Code. You may distribute the Executable version of Covered 240 | Code or ownership rights under a license of Your choice, which may 241 | contain terms different from this License, provided that You are in 242 | compliance with the terms of this License and that the license for the 243 | Executable version does not attempt to limit or alter the recipient's 244 | rights in the Source Code version from the rights set forth in this 245 | License. If You distribute the Executable version under a different 246 | license You must make it absolutely clear that any terms which differ 247 | from this License are offered by You alone, not by the Initial 248 | Developer or any Contributor. You hereby agree to indemnify the 249 | Initial Developer and every Contributor for any liability incurred by 250 | the Initial Developer or such Contributor as a result of any such 251 | terms You offer. 252 | 253 | 3.7. Larger Works. 254 | You may create a Larger Work by combining Covered Code with other code 255 | not governed by the terms of this License and distribute the Larger 256 | Work as a single product. In such a case, You must make sure the 257 | requirements of this License are fulfilled for the Covered Code. 258 | 259 | 4. Inability to Comply Due to Statute or Regulation. 260 | 261 | If it is impossible for You to comply with any of the terms of this 262 | License with respect to some or all of the Covered Code due to 263 | statute, judicial order, or regulation then You must: (a) comply with 264 | the terms of this License to the maximum extent possible; and (b) 265 | describe the limitations and the code they affect. Such description 266 | must be included in the LEGAL file described in Section 3.4 and must 267 | be included with all distributions of the Source Code. Except to the 268 | extent prohibited by statute or regulation, such description must be 269 | sufficiently detailed for a recipient of ordinary skill to be able to 270 | understand it. 271 | 272 | 5. Application of this License. 273 | 274 | This License applies to code to which the Initial Developer has 275 | attached the notice in Exhibit A and to related Covered Code. 276 | 277 | 6. Versions of the License. 278 | 279 | 6.1. New Versions. 280 | Netscape Communications Corporation ("Netscape") may publish revised 281 | and/or new versions of the License from time to time. Each version 282 | will be given a distinguishing version number. 283 | 284 | 6.2. Effect of New Versions. 285 | Once Covered Code has been published under a particular version of the 286 | License, You may always continue to use it under the terms of that 287 | version. You may also choose to use such Covered Code under the terms 288 | of any subsequent version of the License published by Netscape. No one 289 | other than Netscape has the right to modify the terms applicable to 290 | Covered Code created under this License. 291 | 292 | 6.3. Derivative Works. 293 | If You create or use a modified version of this License (which you may 294 | only do in order to apply it to code which is not already Covered Code 295 | governed by this License), You must (a) rename Your license so that 296 | the phrases "Mozilla", "MOZILLAPL", "MOZPL", "Netscape", 297 | "MPL", "NPL" or any confusingly similar phrase do not appear in your 298 | license (except to note that your license differs from this License) 299 | and (b) otherwise make it clear that Your version of the license 300 | contains terms which differ from the Mozilla Public License and 301 | Netscape Public License. (Filling in the name of the Initial 302 | Developer, Original Code or Contributor in the notice described in 303 | Exhibit A shall not of themselves be deemed to be modifications of 304 | this License.) 305 | 306 | 7. DISCLAIMER OF WARRANTY. 307 | 308 | COVERED CODE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS, 309 | WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, 310 | WITHOUT LIMITATION, WARRANTIES THAT THE COVERED CODE IS FREE OF 311 | DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. 312 | THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED CODE 313 | IS WITH YOU. SHOULD ANY COVERED CODE PROVE DEFECTIVE IN ANY RESPECT, 314 | YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE 315 | COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER 316 | OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF 317 | ANY COVERED CODE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. 318 | 319 | 8. TERMINATION. 320 | 321 | 8.1. This License and the rights granted hereunder will terminate 322 | automatically if You fail to comply with terms herein and fail to cure 323 | such breach within 30 days of becoming aware of the breach. All 324 | sublicenses to the Covered Code which are properly granted shall 325 | survive any termination of this License. Provisions which, by their 326 | nature, must remain in effect beyond the termination of this License 327 | shall survive. 328 | 329 | 8.2. If You initiate litigation by asserting a patent infringement 330 | claim (excluding declatory judgment actions) against Initial Developer 331 | or a Contributor (the Initial Developer or Contributor against whom 332 | You file such action is referred to as "Participant") alleging that: 333 | 334 | (a) such Participant's Contributor Version directly or indirectly 335 | infringes any patent, then any and all rights granted by such 336 | Participant to You under Sections 2.1 and/or 2.2 of this License 337 | shall, upon 60 days notice from Participant terminate prospectively, 338 | unless if within 60 days after receipt of notice You either: (i) 339 | agree in writing to pay Participant a mutually agreeable reasonable 340 | royalty for Your past and future use of Modifications made by such 341 | Participant, or (ii) withdraw Your litigation claim with respect to 342 | the Contributor Version against such Participant. If within 60 days 343 | of notice, a reasonable royalty and payment arrangement are not 344 | mutually agreed upon in writing by the parties or the litigation claim 345 | is not withdrawn, the rights granted by Participant to You under 346 | Sections 2.1 and/or 2.2 automatically terminate at the expiration of 347 | the 60 day notice period specified above. 348 | 349 | (b) any software, hardware, or device, other than such Participant's 350 | Contributor Version, directly or indirectly infringes any patent, then 351 | any rights granted to You by such Participant under Sections 2.1(b) 352 | and 2.2(b) are revoked effective as of the date You first made, used, 353 | sold, distributed, or had made, Modifications made by that 354 | Participant. 355 | 356 | 8.3. If You assert a patent infringement claim against Participant 357 | alleging that such Participant's Contributor Version directly or 358 | indirectly infringes any patent where such claim is resolved (such as 359 | by license or settlement) prior to the initiation of patent 360 | infringement litigation, then the reasonable value of the licenses 361 | granted by such Participant under Sections 2.1 or 2.2 shall be taken 362 | into account in determining the amount or value of any payment or 363 | license. 364 | 365 | 8.4. In the event of termination under Sections 8.1 or 8.2 above, 366 | all end user license agreements (excluding distributors and resellers) 367 | which have been validly granted by You or any distributor hereunder 368 | prior to termination shall survive termination. 369 | 370 | 9. LIMITATION OF LIABILITY. 371 | 372 | UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT 373 | (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL 374 | DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED CODE, 375 | OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR 376 | ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY 377 | CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, 378 | WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER 379 | COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN 380 | INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF 381 | LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY 382 | RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW 383 | PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE 384 | EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO 385 | THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. 386 | 387 | 10. U.S. GOVERNMENT END USERS. 388 | 389 | The Covered Code is a "commercial item," as that term is defined in 390 | 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer 391 | software" and "commercial computer software documentation," as such 392 | terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 393 | C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), 394 | all U.S. Government End Users acquire Covered Code with only those 395 | rights set forth herein. 396 | 397 | 11. MISCELLANEOUS. 398 | 399 | This License represents the complete agreement concerning subject 400 | matter hereof. If any provision of this License is held to be 401 | unenforceable, such provision shall be reformed only to the extent 402 | necessary to make it enforceable. This License shall be governed by 403 | California law provisions (except to the extent applicable law, if 404 | any, provides otherwise), excluding its conflict-of-law provisions. 405 | With respect to disputes in which at least one party is a citizen of, 406 | or an entity chartered or registered to do business in the United 407 | States of America, any litigation relating to this License shall be 408 | subject to the jurisdiction of the Federal Courts of the Northern 409 | District of California, with venue lying in Santa Clara County, 410 | California, with the losing party responsible for costs, including 411 | without limitation, court costs and reasonable attorneys' fees and 412 | expenses. The application of the United Nations Convention on 413 | Contracts for the International Sale of Goods is expressly excluded. 414 | Any law or regulation which provides that the language of a contract 415 | shall be construed against the drafter shall not apply to this 416 | License. 417 | 418 | 12. RESPONSIBILITY FOR CLAIMS. 419 | 420 | As between Initial Developer and the Contributors, each party is 421 | responsible for claims and damages arising, directly or indirectly, 422 | out of its utilization of rights under this License and You agree to 423 | work with Initial Developer and Contributors to distribute such 424 | responsibility on an equitable basis. Nothing herein is intended or 425 | shall be deemed to constitute any admission of liability. 426 | 427 | 13. MULTIPLE-LICENSED CODE. 428 | 429 | Initial Developer may designate portions of the Covered Code as 430 | "Multiple-Licensed". "Multiple-Licensed" means that the Initial 431 | Developer permits you to utilize portions of the Covered Code under 432 | Your choice of the NPL or the alternative licenses, if any, specified 433 | by the Initial Developer in the file described in Exhibit A. 434 | 435 | EXHIBIT A -Mozilla Public License. 436 | 437 | ``The contents of this file are subject to the Mozilla Public License 438 | Version 1.1 (the "License"); you may not use this file except in 439 | compliance with the License. You may obtain a copy of the License at 440 | http://www.mozilla.org/MPL/ 441 | 442 | Software distributed under the License is distributed on an "AS IS" 443 | basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the 444 | License for the specific language governing rights and limitations 445 | under the License. 446 | 447 | The Original Code is ______________________________________. 448 | 449 | The Initial Developer of the Original Code is ________________________. 450 | Portions created by ______________________ are Copyright (C) ______ 451 | _______________________. All Rights Reserved. 452 | 453 | Contributor(s): ______________________________________. 454 | 455 | Alternatively, the contents of this file may be used under the terms 456 | of the _____ license (the "[___] License"), in which case the 457 | provisions of [______] License are applicable instead of those 458 | above. If you wish to allow use of your version of this file only 459 | under the terms of the [____] License and not to allow others to use 460 | your version of this file under the MPL, indicate your decision by 461 | deleting the provisions above and replace them with the notice and 462 | other provisions required by the [___] License. If you do not delete 463 | the provisions above, a recipient may use your version of this file 464 | under either the MPL or the [___] License." 465 | 466 | [NOTE: The text of this Exhibit A may differ slightly from the text of 467 | the notices in the Source Code files of the Original Code. You should 468 | use the text of this Exhibit A rather than the text found in the 469 | Original Code Source Code for Your Modifications.] 470 | 471 | -------------------------------------------------------------------------------- /Tangle.cs: -------------------------------------------------------------------------------- 1 | /* 2 | The contents of this file are subject to the Mozilla Public License 3 | Version 1.1 (the "License"); you may not use this file except in 4 | compliance with the License. You may obtain a copy of the License at 5 | http://www.mozilla.org/MPL/ 6 | 7 | Software distributed under the License is distributed on an "AS IS" 8 | basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the 9 | License for the specific language governing rights and limitations 10 | under the License. 11 | 12 | The Original Code is DataMangler Key-Value Store. 13 | 14 | The Initial Developer of the Original Code is Mozilla Corporation. 15 | 16 | Original Author: Kevin Gadd (kevin.gadd@gmail.com) 17 | */ 18 | 19 | using System; 20 | using System.Collections.Generic; 21 | using System.IO; 22 | using Squared.Data.Mangler.Serialization; 23 | using Squared.Task; 24 | using System.Threading; 25 | using Squared.Data.Mangler.Internal; 26 | using System.Collections.Concurrent; 27 | using TaskScheduler = Squared.Task.TaskScheduler; 28 | 29 | namespace Squared.Data.Mangler { 30 | public class KeyNotFoundException : Exception { 31 | public readonly TangleKey Key; 32 | 33 | public KeyNotFoundException (TangleKey key) { 34 | Key = key; 35 | } 36 | 37 | public override string Message { 38 | get { 39 | return String.Format("The key '{0}' was not found.", Key); 40 | } 41 | } 42 | } 43 | 44 | public class TangleModifiedException : Exception { 45 | public TangleModifiedException () 46 | : base("The tangle's contents were modified after this FindResult was created so it is no longer valid.") { 47 | } 48 | } 49 | 50 | public class SerializerThrewException : Exception { 51 | public readonly TangleKey Key; 52 | 53 | public SerializerThrewException (TangleKey key, Exception innerException) 54 | : base("", innerException) { 55 | Key = key; 56 | } 57 | 58 | public override string Message { 59 | get { 60 | return String.Format("The data for key '{0}' was not written because the serializer threw an exception.", Key); 61 | } 62 | } 63 | } 64 | 65 | /// 66 | /// Represents a persistent dictionary keyed by arbitrary byte strings. The values are not stored in any given order on disk, and the values are not required to be resident in memory. 67 | /// At any given time a portion of the Tangle's values may be resident in memory. If a value is not resident in memory, it will be fetched asynchronously from disk. 68 | /// The Tangle's keys are implicitly ordered, which allows for efficient lookups of individual values by key. 69 | /// Converting values to/from their disk format is handled by the provided TangleSerializer and TangleDeserializer. 70 | /// The Tangle's disk storage engine partitions its storage up into pages based on the provided page size. For optimal performance, this should be an integer multiple of the size of a memory page (typically 4KB). 71 | /// 72 | /// The type of the value stored within the tangle. 73 | public unsafe partial class Tangle : ITangle { 74 | public struct LockedData : IDisposable { 75 | public readonly byte * Pointer; 76 | public readonly uint Size; 77 | 78 | private readonly ManualResetEventSlim DisposedSignal; 79 | 80 | internal LockedData (byte * pointer, uint size, ManualResetEventSlim disposedSignal) { 81 | Pointer = pointer; 82 | Size = size; 83 | DisposedSignal = disposedSignal; 84 | } 85 | 86 | public void Dispose () { 87 | DisposedSignal.Set(); 88 | } 89 | } 90 | 91 | public struct FindResult { 92 | public readonly Tangle Tangle; 93 | public readonly TangleKey Key; 94 | private readonly uint Version; 95 | private readonly long NodeIndex; 96 | private readonly uint ValueIndex; 97 | 98 | internal FindResult (Tangle owner, TangleKey key, long nodeIndex, uint valueIndex) { 99 | Tangle = owner; 100 | Key = key; 101 | Version = owner.Version; 102 | NodeIndex = nodeIndex; 103 | ValueIndex = valueIndex; 104 | } 105 | 106 | public Future GetValue () { 107 | if (Tangle.Version != Version) 108 | throw new TangleModifiedException(); 109 | 110 | return Tangle.GetValueByIndex(NodeIndex, ValueIndex); 111 | } 112 | 113 | public IFuture SetValue (T newValue) { 114 | if (Tangle.Version != Version) 115 | throw new TangleModifiedException(); 116 | 117 | return Tangle.SetValueByIndex(NodeIndex, ValueIndex, ref newValue); 118 | } 119 | 120 | public Future LockData (long? minimumSize = null) { 121 | if (Tangle.Version != Version) 122 | throw new TangleModifiedException(); 123 | 124 | return Tangle.QueueWorkItem(new LockDataThunk(NodeIndex, ValueIndex, minimumSize)); 125 | } 126 | 127 | public IFuture CopyFrom (Stream input, long? bytesToCopy = null, int? bufferSize = null) { 128 | if (Tangle.Version != Version) 129 | throw new TangleModifiedException(); 130 | 131 | return Tangle.QueueWorkItem(new CopyFromStreamThunk(NodeIndex, ValueIndex, input, bytesToCopy, bufferSize)); 132 | } 133 | 134 | public IFuture CopyTo (Stream output, int? bufferSize = null) { 135 | if (Tangle.Version != Version) 136 | throw new TangleModifiedException(); 137 | 138 | return Tangle.QueueWorkItem(new CopyToStreamThunk(NodeIndex, ValueIndex, output, bufferSize)); 139 | } 140 | } 141 | 142 | public static readonly int WorkerThreadTimeoutMs = 30000; 143 | 144 | public readonly bool OwnsStorage; 145 | public readonly StreamSource Storage; 146 | public readonly TaskScheduler Scheduler; 147 | public readonly Serializer Serializer; 148 | public readonly Deserializer Deserializer; 149 | public readonly Dictionary> Indices = new Dictionary>(); 150 | 151 | protected readonly ReaderWriterLockSlim IndexLock = new ReaderWriterLockSlim(LockRecursionPolicy.NoRecursion); 152 | 153 | internal Squared.Task.Internal.WorkerThread>> _WorkerThread; 154 | 155 | private uint _Version; 156 | private bool _IsDisposed; 157 | 158 | private readonly BTree BTree; 159 | 160 | public Tangle ( 161 | TaskScheduler scheduler, 162 | StreamSource storage, 163 | Serializer serializer = null, 164 | Deserializer deserializer = null, 165 | bool ownsStorage = true 166 | ) { 167 | Scheduler = scheduler; 168 | Storage = storage; 169 | OwnsStorage = ownsStorage; 170 | 171 | Serializer = serializer ?? Defaults.Serializer; 172 | Deserializer = deserializer ?? Defaults.Deserializer; 173 | 174 | BTree = new BTree(Storage, ""); 175 | } 176 | 177 | IBarrier ITangle.CreateBarrier (bool createOpened) { 178 | return this.CreateBarrier(createOpened); 179 | } 180 | 181 | /// 182 | /// Inserts a barrier into the tangle's work queue. 183 | /// A barrier prevents the execution of work items following it as long as it remains closed, and becomes signaled once that point in the queue is reached. 184 | /// 185 | /// If true, the barrier is created open, which allows items following it in the work queue to be executed. Otherwise, the barrier is created closed (and can be opened manually.) 186 | /// The barrier that was created. 187 | public Barrier CreateBarrier (bool createOpened = false) { 188 | return new Barrier(this, createOpened); 189 | } 190 | 191 | /// 192 | /// Creates a batch that can be used to write to tangles of this type. 193 | /// 194 | /// The maximum capacity of the batch. 195 | /// A new batch instance. 196 | public Batch CreateBatch (int capacity) { 197 | return new Batch(this, capacity); 198 | } 199 | 200 | public Future> CreateIndex (string name, IndexFunc function) { 201 | return Index.Create(this, name, function); 202 | } 203 | 204 | public Future> CreateIndex (string name, IndexMultipleFunc function) { 205 | return Index.Create(this, name, function); 206 | } 207 | 208 | /// 209 | /// Reads a value from the tangle, looking it up via its key. 210 | /// 211 | /// A future that will contain the value once it has been read. 212 | /// If the specified key is not found, the future will contain a KeyNotFoundException. 213 | public Future Get (TangleKey key) { 214 | return QueueWorkItem(new GetThunk(key)); 215 | } 216 | 217 | IFuture ITangle.Get (TangleKey key) { 218 | return Get(key); 219 | } 220 | 221 | /// 222 | /// Reads multiple values from the tangle, looking them up based on a provided sequence of keys. 223 | /// 224 | /// The keys to look up in this tangle. 225 | /// A future that will contain the retrieved values. 226 | public Future Select (IEnumerable keys) { 227 | return QueueWorkItem(new GetMultipleThunk(keys)); 228 | } 229 | 230 | /// 231 | /// Reads multiple values from the tangle, looking them up based on a provided sequence of keys. 232 | /// If a provided key is not found in this tangle, each of the tangles in the sequence of cascades is tried. 233 | /// 234 | /// The keys to look up in this tangle. 235 | /// The tangles to search if this tangle does not contain a key. 236 | /// A future that will contain the retrieved values. 237 | public Future CascadingSelect (IEnumerable> cascades, IEnumerable keys) { 238 | var seen = new HashSet>(); 239 | var barriers = new List.JoinBarrierThunk>(); 240 | 241 | foreach (var tangle in cascades) { 242 | if (Object.Equals(tangle, this) || seen.Contains(tangle)) 243 | throw new InvalidOperationException("Cannot cascade a tangle with itself"); 244 | 245 | var barrier = new Tangle.JoinBarrierThunk(); 246 | tangle.QueueWorkItem(barrier); 247 | barriers.Add(barrier); 248 | seen.Add(tangle); 249 | } 250 | 251 | return QueueWorkItem(new CascadingGetMultipleThunk(barriers, cascades, keys)); 252 | } 253 | 254 | /// 255 | /// Scans over multiple values from the tangle and invokes a function on them. 256 | /// 257 | /// The keys to look up in this tangle. 258 | /// The function to invoke on each item from the tangle. This function must be thread-safe. 259 | /// A future that will be completed once all the items have been processed. 260 | public IFuture ForEach (IEnumerable keys, Action function) { 261 | return QueueWorkItem(new ForEachThunk(keys, function)); 262 | } 263 | 264 | /// 265 | /// Performs a map-reduce operation on multiple values from the tangle. 266 | /// 267 | /// The keys to look up in this tangle. 268 | /// The function to use to map each item from the tangle. This function must be thread-safe. 269 | /// The function to use to reduce a pair of mapped results into one mapped result. This function must be thread-safe. 270 | /// The value to use for keys that cannot be found within the tangle. 271 | /// The starting value to feed into the first reduce operation. 272 | /// A future that will contain the final reduced result. 273 | public IFuture MapReduce ( 274 | IEnumerable keys, Func map, 275 | Func reduce, 276 | TMapped initialValue = default(TMapped), 277 | TMapped defaultValue = default(TMapped) 278 | ) { 279 | return QueueWorkItem(new MapReduceThunk( 280 | keys, map, reduce, 281 | initialValue, defaultValue 282 | )); 283 | } 284 | 285 | /// 286 | /// Reads multiple values from the tangle, looking them up based on a provided sequence of keys, 287 | /// and then uses those values to perform a lookup within a second tangle. 288 | /// 289 | /// The keys to look up in this tangle. 290 | /// The tangle to join against. 291 | /// A delegate that takes a key/value pair from this tangle and produces a key to use for a lookup in the other tangle. 292 | /// A delegate that takes key/value pairs from both tangles and produces a result for the join. 293 | /// A future that will contain the join results. 294 | public Future Join ( 295 | Tangle right, IEnumerable keys, 296 | JoinKeySelector keySelector, 297 | JoinValueSelector valueSelector 298 | ) { 299 | Tangle.JoinBarrierThunk rightBarrier = null; 300 | if (!Object.Equals(right, this)) { 301 | rightBarrier = new Tangle.JoinBarrierThunk(); 302 | right.QueueWorkItem(rightBarrier); 303 | } 304 | 305 | return QueueWorkItem(new JoinThunk( 306 | rightBarrier, right, keys, keySelector, valueSelector 307 | )); 308 | } 309 | 310 | /// 311 | /// Reads multiple values from the tangle, looking them up based on a provided sequence of keys, 312 | /// and then uses those values to perform a lookup within a second tangle. 313 | /// 314 | /// The keys to look up in this tangle. 315 | /// The tangle to join against. 316 | /// A delegate that takes a value from this tangle and produces a key to use for a lookup in the other tangle. 317 | /// A future that will contain the join results. 318 | public Future[]> Join ( 319 | Tangle right, IEnumerable keys, 320 | Func keySelector 321 | ) { 322 | return Join( 323 | right, keys, 324 | (TLeftKey leftKey, ref T leftValue) 325 | => keySelector(leftValue), 326 | (TLeftKey leftKey, ref T leftValue, TRightKey rightKey, ref TRight rightValue) 327 | => new KeyValuePair(leftValue, rightValue) 328 | ); 329 | } 330 | 331 | /// 332 | /// Reads every key from the tangle, in no particular order. 333 | /// 334 | /// A future that will contain the retrieved keys. 335 | public Future GetAllKeys () { 336 | return QueueWorkItem(new GetAllKeysThunk()); 337 | } 338 | 339 | /// 340 | /// Reads every value from the tangle, in no particular order. 341 | /// 342 | /// A future that will contain the retrieved values. 343 | public Future GetAllValues () { 344 | return QueueWorkItem(new GetAllValuesThunk()); 345 | } 346 | 347 | IFuture ITangle.GetAllValues () { 348 | return GetAllValues(); 349 | } 350 | 351 | protected Future GetValueByIndex (long nodeIndex, uint valueIndex) { 352 | return QueueWorkItem(new GetByIndexThunk(nodeIndex, valueIndex)); 353 | } 354 | 355 | protected IFuture SetValueByIndex (long nodeIndex, uint valueIndex, ref T value) { 356 | return QueueWorkItem(new SetByIndexThunk(nodeIndex, valueIndex, ref value)); 357 | } 358 | 359 | protected void InternalClear () { 360 | unchecked { _Version++; } 361 | 362 | BTree.Clear(); 363 | foreach (var index in Indices.Values) 364 | index.Clear(); 365 | } 366 | 367 | /// 368 | /// Erases the contents of the tangle and all attached indexes. 369 | /// 370 | /// A future that completes once the tangle's contents have been erased. 371 | public IFuture Clear () { 372 | return QueueWorkItem(new ClearThunk()); 373 | } 374 | 375 | /// 376 | /// Searches the tangle for a given key, and if it is found, returns a reference to the key that can be used to retrieve or replace its associated value. 377 | /// 378 | /// A future that will contain a reference to the key, if it was found. 379 | /// If the specified key is not found, the future will contain a KeyNotFoundException. 380 | public Future Find (TangleKey key) { 381 | return QueueWorkItem(new FindThunk(key)); 382 | } 383 | 384 | /// 385 | /// Stores a value into the tangle, assigning it a given key. If the given key already has an associated value, that value is replaced. 386 | /// 387 | /// A future that completes once the value has been stored to disk. 388 | public IFuture Set (TangleKey key, T value) { 389 | return QueueWorkItem(new SetThunk(key, ref value, true)); 390 | } 391 | 392 | /// 393 | /// Stores a value into the tangle, assigning it a given key. If the given key already has an associated value, the operation will abort. 394 | /// 395 | /// A future that completes once the value has been stored to disk. The future's value will be false if the operation was aborted. 396 | public Future Add (TangleKey key, T value) { 397 | return QueueWorkItem(new SetThunk(key, ref value, false)); 398 | } 399 | 400 | /// 401 | /// Stores a value into the tangle, assigning it a given key. If the given key already has an associated value, a callback is invoked to determine the new value for the key. 402 | /// 403 | /// A future that completes once the value has been stored to disk. 404 | public Future AddOrUpdate (TangleKey key, T value, UpdateCallback updateCallback) { 405 | return QueueWorkItem(new UpdateThunk(key, ref value, updateCallback)); 406 | } 407 | 408 | /// 409 | /// Stores a value into the tangle, assigning it a given key. If the given key already has an associated value, a callback is invoked to determine the new value for the key. 410 | /// 411 | /// A future that completes once the value has been stored to disk. 412 | public Future AddOrUpdate (TangleKey key, T value, DecisionUpdateCallback updateCallback) { 413 | return QueueWorkItem(new UpdateThunk(key, ref value, updateCallback)); 414 | } 415 | 416 | internal long NodeCount { 417 | get { 418 | return BTree.NodeCount; 419 | } 420 | } 421 | 422 | public uint Version { 423 | get { 424 | return _Version; 425 | } 426 | } 427 | 428 | public long Count { 429 | get { 430 | return BTree.ValueCount; 431 | } 432 | } 433 | 434 | public long WastedDataBytes { 435 | get { 436 | return BTree.WastedDataBytes; 437 | } 438 | } 439 | 440 | /// 441 | /// Queues a work item into the tangle's work queue. Work items in the queue are processed sequentially in order to prevent corruption of internal data structures. 442 | /// 443 | /// The type of the work item's result, if any. 444 | /// The work item. 445 | /// A future that will contain the result of the work item once it is complete. 446 | internal Future QueueWorkItem (IWorkItemWithFuture workItem) { 447 | if (_IsDisposed) 448 | throw new ObjectDisposedException("Tangle"); 449 | 450 | var future = workItem.Future; 451 | 452 | if (_WorkerThread == null) 453 | _WorkerThread = new Squared.Task.Internal.WorkerThread>>( 454 | WorkerThreadFunc, ThreadPriority.Normal, String.Format("Tangle<{0}> Worker", typeof(T).ToString()) 455 | ); 456 | 457 | _WorkerThread.WorkItems.Enqueue(workItem); 458 | 459 | _WorkerThread.Wake(); 460 | 461 | return future; 462 | } 463 | 464 | internal void WorkerThreadFunc (ConcurrentQueue> workItems, ManualResetEventSlim newWorkItemEvent) { 465 | while (true) { 466 | IWorkItem item; 467 | while (workItems.TryDequeue(out item)) { 468 | item.Execute(this); 469 | } 470 | 471 | if (!newWorkItemEvent.Wait(WorkerThreadTimeoutMs)) { 472 | BTree.FlushCache(); 473 | return; 474 | } 475 | 476 | newWorkItemEvent.Reset(); 477 | } 478 | } 479 | 480 | private void InternalSetFoundValue (long nodeIndex, uint valueIndex, ref T value) { 481 | unchecked { _Version++; } 482 | 483 | using (var range = BTree.AccessNode(nodeIndex, true)) { 484 | ushort keyType; 485 | var pEntry = BTree.LockValue(range, valueIndex, out keyType); 486 | 487 | if (Indices.Count > 0) { 488 | TangleKey key; 489 | T oldValue; 490 | 491 | BTree.ReadKey(pEntry, keyType, out key); 492 | ReadData(ref *pEntry, keyType, out oldValue); 493 | 494 | foreach (var index in Indices.Values) { 495 | index.OnValueRemoved(key, ref oldValue); 496 | index.OnValueAdded(key, ref value); 497 | } 498 | } 499 | 500 | var segment = BTree.Serialize(pEntry, Serializer, keyType, ref value); 501 | 502 | BTree.WriteData(pEntry, segment); 503 | 504 | BTree.UnlockValue(pEntry, keyType); 505 | 506 | BTree.UnlockNode(range); 507 | } 508 | } 509 | 510 | private bool InternalSet (TangleKey key, ref T value, IReplaceCallback replacementCallback) { 511 | unchecked { _Version++; } 512 | 513 | long nodeIndex; 514 | uint valueIndex; 515 | 516 | Exception serializerException = null; 517 | bool foundExisting = BTree.FindKey(key, true, out nodeIndex, out valueIndex); 518 | 519 | StreamRange range; 520 | if (foundExisting) { 521 | range = BTree.AccessNode(nodeIndex, true); 522 | } else { 523 | // Prepare BTree for insert. Note that once we have done this, we must successfully insert or 524 | // the index will be left in an invalid state! 525 | range = BTree.PrepareForInsert(nodeIndex, valueIndex); 526 | } 527 | 528 | using (range) { 529 | var pEntry = BTree.LockValue(range, valueIndex, foundExisting ? key.OriginalTypeId : (ushort)0); 530 | 531 | if (foundExisting) { 532 | bool shouldContinue = replacementCallback.ShouldReplace(this, ref *pEntry, key.OriginalTypeId, ref value); 533 | 534 | if (!shouldContinue) { 535 | BTree.UnlockValue(pEntry, key.OriginalTypeId); 536 | BTree.UnlockNode(range); 537 | return false; 538 | } 539 | } else { 540 | BTree.WriteNewKey(pEntry, key); 541 | } 542 | 543 | // It is very important that the entry be properly initialized at this point. 544 | // Serializers can request the key of the value being serialized, in which case the 545 | // SerializationContext will attempt to read information from the IndexEntry. 546 | // Note that since a KeyType of 0 is used to indicate that an entry is being modified, 547 | // we pass the actual KeyType to the serializer. 548 | 549 | ArraySegment segment = default(ArraySegment); 550 | try { 551 | segment = BTree.Serialize(pEntry, Serializer, key.OriginalTypeId, ref value); 552 | } catch (Exception ex) { 553 | serializerException = ex; 554 | } 555 | 556 | if ((Indices.Count > 0) && foundExisting) { 557 | T oldValue; 558 | ReadData(ref *pEntry, key.OriginalTypeId, out oldValue); 559 | 560 | foreach (var index in Indices.Values) 561 | index.OnValueRemoved(key, ref oldValue); 562 | } 563 | 564 | BTree.WriteData(pEntry, segment); 565 | 566 | BTree.UnlockValue(pEntry, key.OriginalTypeId); 567 | 568 | if (foundExisting) 569 | BTree.UnlockNode(range); 570 | else 571 | BTree.FinalizeInsert(range); 572 | } 573 | 574 | foreach (var index in Indices.Values) 575 | index.OnValueAdded(key, ref value); 576 | 577 | // If the user's serializer throws, we wait until now to rethrow the exception so that 578 | // we do not leave the index in an invalid state (in the case of a BTree insert). 579 | if (serializerException != null) 580 | throw new SerializerThrewException(key, serializerException); 581 | 582 | return true; 583 | } 584 | 585 | private bool InternalFind (TangleKey key, out FindResult result) { 586 | long nodeIndex; 587 | uint valueIndex; 588 | 589 | if (!BTree.FindKey(key, false, out nodeIndex, out valueIndex)) { 590 | result = default(FindResult); 591 | return false; 592 | } 593 | 594 | result = new FindResult(this, key, nodeIndex, valueIndex); 595 | return true; 596 | } 597 | 598 | internal bool InternalGet (TangleKey key, out T value) { 599 | long nodeIndex; 600 | uint valueIndex; 601 | 602 | if (!BTree.FindKey(key, false, out nodeIndex, out valueIndex)) { 603 | value = default(T); 604 | return false; 605 | } 606 | 607 | using (var range = BTree.AccessValue(nodeIndex, valueIndex)) { 608 | var pEntry = (BTreeValue *)range.Pointer; 609 | BTree.ReadData(ref *pEntry, Deserializer, out value); 610 | } 611 | 612 | return true; 613 | } 614 | 615 | private unsafe ushort GetValueCount (StreamRange range) { 616 | var pNode = (BTreeNode*)range.Pointer; 617 | 618 | return pNode->NumValues; 619 | } 620 | 621 | private unsafe KeyValuePair GetNodeKeyValuePair (StreamRange range, uint valueIndex) { 622 | var pEntry = (BTreeValue*)(range.Pointer + BTreeNode.OffsetOfValues + (valueIndex * BTreeValue.Size)); 623 | 624 | TangleKey key; 625 | T value; 626 | BTree.ReadKey(pEntry, out key); 627 | BTree.ReadData(pEntry, Deserializer, out value); 628 | 629 | return new KeyValuePair(key, value); 630 | } 631 | 632 | internal IEnumerable> InternalEnumerateNode (long nodeIndex) { 633 | using (var range = BTree.AccessNode(nodeIndex, false)) { 634 | var numValues = GetValueCount(range); 635 | 636 | for (uint i = 0; i < numValues; i++) 637 | yield return GetNodeKeyValuePair(range, i); 638 | } 639 | } 640 | 641 | private void InternalGetFoundValue (long nodeIndex, uint valueIndex, out T result) { 642 | using (var range = BTree.AccessValue(nodeIndex, valueIndex)) { 643 | var pEntry = (BTreeValue *)range.Pointer; 644 | 645 | BTree.ReadData(ref *pEntry, Deserializer, out result); 646 | } 647 | } 648 | 649 | private unsafe void ReadData (ref BTreeValue entry, ushort keyType, out T value) { 650 | fixed (BTreeValue * pEntry = &entry) 651 | BTree.ReadData(pEntry, keyType, Deserializer, out value); 652 | } 653 | 654 | public void Dispose () { 655 | if (_IsDisposed) 656 | return; 657 | _IsDisposed = true; 658 | 659 | if (_WorkerThread != null) { 660 | var workItems = _WorkerThread.WorkItems; 661 | _WorkerThread.Dispose(); 662 | _WorkerThread = null; 663 | 664 | IWorkItem wi; 665 | while (workItems.TryDequeue(out wi)) 666 | wi.Dispose(); 667 | } 668 | 669 | BTree.Dispose(); 670 | 671 | foreach (var index in Indices.Values) 672 | index.Dispose(); 673 | Indices.Clear(); 674 | 675 | if (OwnsStorage) 676 | Storage.Dispose(); 677 | } 678 | } 679 | } -------------------------------------------------------------------------------- /Tests/BasicTests.cs: -------------------------------------------------------------------------------- 1 | /* 2 | The contents of this file are subject to the Mozilla Public License 3 | Version 1.1 (the "License"); you may not use this file except in 4 | compliance with the License. You may obtain a copy of the License at 5 | http://www.mozilla.org/MPL/ 6 | 7 | Software distributed under the License is distributed on an "AS IS" 8 | basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the 9 | License for the specific language governing rights and limitations 10 | under the License. 11 | 12 | The Original Code is DataMangler Key-Value Store. 13 | 14 | The Initial Developer of the Original Code is Mozilla Corporation. 15 | 16 | Original Author: Kevin Gadd (kevin.gadd@gmail.com) 17 | */ 18 | 19 | using System; 20 | using System.Collections.Concurrent; 21 | using System.Collections.Generic; 22 | using System.Linq; 23 | using System.Text; 24 | using System.Threading; 25 | using NUnit.Framework; 26 | using Squared.Data.Mangler.Internal; 27 | using Squared.Data.Mangler.Serialization; 28 | using Squared.Task; 29 | using System.IO; 30 | using Squared.Util; 31 | 32 | namespace Squared.Data.Mangler.Tests { 33 | public class BasicTestFixture { 34 | public string TestFile; 35 | public StreamSource Storage; 36 | public TaskScheduler Scheduler; 37 | 38 | [SetUp] 39 | public virtual void SetUp () { 40 | Scheduler = new TaskScheduler(); 41 | 42 | TestFile = Path.GetTempFileName(); 43 | Storage = new AlternateStreamSource(TestFile); 44 | } 45 | 46 | [TearDown] 47 | public virtual void TearDown () { 48 | Scheduler.Dispose(); 49 | File.Delete(TestFile); 50 | } 51 | } 52 | 53 | [TestFixture] 54 | public class BasicTests : BasicTestFixture { 55 | public Tangle Tangle; 56 | 57 | [SetUp] 58 | public override void SetUp () { 59 | base.SetUp(); 60 | 61 | Tangle = new Tangle( 62 | Scheduler, Storage, 63 | serializer: BlittableSerializer.Serialize, 64 | deserializer: BlittableSerializer.Deserialize, 65 | ownsStorage: true 66 | ); 67 | } 68 | 69 | [TearDown] 70 | public override void TearDown () { 71 | // Tangle.ExportStreams(@"C:\dm_streams\"); 72 | Tangle.Dispose(); 73 | base.TearDown(); 74 | } 75 | 76 | [Test] 77 | public void CanGetValueByNameAfterSettingIt () { 78 | Scheduler.WaitFor(Tangle.Set("hello", 1)); 79 | Assert.AreEqual(1, Scheduler.WaitFor(Tangle.Get("hello"))); 80 | } 81 | 82 | [Test] 83 | public void GetThrowsIfKeyIsNotFound () { 84 | try { 85 | Scheduler.WaitFor(Tangle.Get("missing")); 86 | Assert.Fail("Should have thrown"); 87 | } catch (FutureException fe) { 88 | Assert.IsInstanceOf(fe.InnerException); 89 | } 90 | } 91 | 92 | [Test] 93 | public void NumericKeysWork () { 94 | var key = new TangleKey(1234); 95 | Scheduler.WaitFor(Tangle.Set(key, 1)); 96 | Assert.AreEqual(1, Scheduler.WaitFor(Tangle.Get(key))); 97 | } 98 | 99 | [Test] 100 | public void CanOverwriteExistingValueBySettingItAgain () { 101 | Scheduler.WaitFor(Tangle.Set("hello", 1)); 102 | Scheduler.WaitFor(Tangle.Set("hello", 3)); 103 | Assert.AreEqual(3, Scheduler.WaitFor(Tangle.Get("hello"))); 104 | } 105 | 106 | [Test] 107 | public void AddReturnsFalseInsteadOfOverwriting () { 108 | Assert.AreEqual(true, Scheduler.WaitFor(Tangle.Add("hello", 1))); 109 | Assert.AreEqual(false, Scheduler.WaitFor(Tangle.Add("hello", 3))); 110 | Assert.AreEqual(1, Scheduler.WaitFor(Tangle.Get("hello"))); 111 | } 112 | 113 | [Test] 114 | public void FindReturnsReferenceThatCanBeUsedToFetchValue () { 115 | Scheduler.WaitFor(Tangle.Set("a", 1)); 116 | Scheduler.WaitFor(Tangle.Set("b", 2)); 117 | 118 | var itemRef = Scheduler.WaitFor(Tangle.Find("a")); 119 | Assert.AreEqual("a", itemRef.Key.ToString()); 120 | Assert.AreEqual(1, Scheduler.WaitFor(itemRef.GetValue())); 121 | } 122 | 123 | [Test] 124 | public void FindReturnsReferenceThatCanBeUsedToReplaceValue () { 125 | Scheduler.WaitFor(Tangle.Set("a", 1)); 126 | Scheduler.WaitFor(Tangle.Set("b", 2)); 127 | 128 | var itemRef = Scheduler.WaitFor(Tangle.Find("a")); 129 | Scheduler.WaitFor(itemRef.SetValue(3)); 130 | } 131 | 132 | [Test] 133 | public void FindReferenceIsInvalidatedByAddingNewKey () { 134 | Scheduler.WaitFor(Tangle.Set("a", 1)); 135 | Scheduler.WaitFor(Tangle.Set("b", 2)); 136 | 137 | var itemRef = Scheduler.WaitFor(Tangle.Find("a")); 138 | Scheduler.WaitFor(Tangle.Set("c", 4)); 139 | 140 | Assert.Throws( 141 | () => Scheduler.WaitFor(itemRef.SetValue(3)) 142 | ); 143 | Assert.Throws( 144 | () => Scheduler.WaitFor(itemRef.GetValue()) 145 | ); 146 | Assert.Throws( 147 | () => Scheduler.WaitFor(itemRef.LockData()) 148 | ); 149 | } 150 | 151 | [Test] 152 | public void FindReferenceIsInvalidatedByClearing () { 153 | Scheduler.WaitFor(Tangle.Set("a", 1)); 154 | Scheduler.WaitFor(Tangle.Set("b", 2)); 155 | 156 | var itemRef = Scheduler.WaitFor(Tangle.Find("a")); 157 | Scheduler.WaitFor(Tangle.Clear()); 158 | 159 | Assert.Throws( 160 | () => Scheduler.WaitFor(itemRef.SetValue(3)) 161 | ); 162 | Assert.Throws( 163 | () => Scheduler.WaitFor(itemRef.GetValue()) 164 | ); 165 | Assert.Throws( 166 | () => Scheduler.WaitFor(itemRef.LockData()) 167 | ); 168 | } 169 | 170 | [Test] 171 | public void FindThrowsIfKeyIsNotFound () { 172 | try { 173 | Scheduler.WaitFor(Tangle.Find("missing")); 174 | Assert.Fail("Should have thrown"); 175 | } catch (FutureException fe) { 176 | Assert.IsInstanceOf(fe.InnerException); 177 | } 178 | } 179 | 180 | [Test] 181 | public void AddOrUpdateInvokesCallbackWhenKeyIsFound () { 182 | Scheduler.WaitFor(Tangle.Add("a", 1)); 183 | Scheduler.WaitFor(Tangle.AddOrUpdate("a", 999, (oldValue) => oldValue + 1)); 184 | Scheduler.WaitFor(Tangle.AddOrUpdate("b", 128, (oldValue) => oldValue + 1)); 185 | 186 | Assert.AreEqual(2, Scheduler.WaitFor(Tangle.Get("a"))); 187 | Assert.AreEqual(128, Scheduler.WaitFor(Tangle.Get("b"))); 188 | } 189 | 190 | [Test] 191 | public void AddOrUpdateCallbackCanAbortUpdate () { 192 | Scheduler.WaitFor(Tangle.Add("a", 1)); 193 | Scheduler.WaitFor(Tangle.AddOrUpdate("a", 999, (ref int oldValue, ref int newValue) => false)); 194 | 195 | Assert.AreEqual(1, Scheduler.WaitFor(Tangle.Get("a"))); 196 | } 197 | 198 | [Test] 199 | public void AddOrUpdateCallbackCanMutateValue () { 200 | Scheduler.WaitFor(Tangle.Add("a", 1)); 201 | Scheduler.WaitFor(Tangle.AddOrUpdate("a", 999, (ref int oldValue, ref int newValue) => { 202 | newValue = oldValue + 1; 203 | return true; 204 | })); 205 | 206 | Assert.AreEqual(2, Scheduler.WaitFor(Tangle.Get("a"))); 207 | } 208 | 209 | [Test] 210 | public void InsertInSequentialOrder () { 211 | Scheduler.WaitFor(Tangle.Set("aa", 4)); 212 | Scheduler.WaitFor(Tangle.Set("ea", 3)); 213 | Scheduler.WaitFor(Tangle.Set("qa", 2)); 214 | Scheduler.WaitFor(Tangle.Set("za", 1)); 215 | 216 | Assert.AreEqual( 217 | new TangleKey[] { "aa", "ea", "qa", "za" }, 218 | Scheduler.WaitFor(Tangle.GetAllKeys()) 219 | ); 220 | } 221 | 222 | [Test] 223 | public void InsertInReverseOrder () { 224 | Scheduler.WaitFor(Tangle.Set("za", 4)); 225 | Scheduler.WaitFor(Tangle.Set("qa", 3)); 226 | Scheduler.WaitFor(Tangle.Set("ea", 2)); 227 | Scheduler.WaitFor(Tangle.Set("aa", 1)); 228 | 229 | Assert.AreEqual( 230 | new TangleKey[] { "aa", "ea", "qa", "za" }, 231 | Scheduler.WaitFor(Tangle.GetAllKeys()) 232 | ); 233 | } 234 | 235 | protected IEnumerator WriteLotsOfValues (Tangle tangle, int numIterations, int direction) { 236 | if (direction > 0) 237 | for (int i = 0; i < numIterations; i++) { 238 | yield return tangle.Set(i, i); 239 | } 240 | else 241 | for (int i = numIterations - 1; i >= 0; i--) { 242 | yield return tangle.Set(i, i); 243 | } 244 | } 245 | 246 | [Test] 247 | public void CanWriteLotsOfValuesSequentially () { 248 | const int numValues = 50000; 249 | 250 | long startTime = Time.Ticks; 251 | Scheduler.WaitFor(WriteLotsOfValues(Tangle, numValues, 1)); 252 | decimal elapsedSeconds = (decimal)(Time.Ticks - startTime) / Time.SecondInTicks; 253 | Console.WriteLine( 254 | "Wrote {0} values in ~{1:00.000} second(s) at ~{2:00000.00} values/sec.", 255 | numValues, elapsedSeconds, numValues / elapsedSeconds 256 | ); 257 | 258 | startTime = Time.Ticks; 259 | Scheduler.WaitFor(CheckLotsOfValues(Tangle, numValues)); 260 | elapsedSeconds = (decimal)(Time.Ticks - startTime) / Time.SecondInTicks; 261 | Console.WriteLine( 262 | "Read {0} values in ~{1:00.000} second(s) at ~{2:00000.00} values/sec.", 263 | numValues, elapsedSeconds, numValues / elapsedSeconds 264 | ); 265 | } 266 | 267 | [Test] 268 | public void CanWriteLotsOfValuesInReverse () { 269 | const int numValues = 500000; 270 | 271 | long startTime = Time.Ticks; 272 | Scheduler.WaitFor(WriteLotsOfValues(Tangle, numValues, -1)); 273 | decimal elapsedSeconds = (decimal)(Time.Ticks - startTime) / Time.SecondInTicks; 274 | Console.WriteLine( 275 | "Wrote {0} values in ~{1:00.000} second(s) at ~{2:00000.00} values/sec.", 276 | numValues, elapsedSeconds, numValues / elapsedSeconds 277 | ); 278 | 279 | startTime = Time.Ticks; 280 | Scheduler.WaitFor(CheckLotsOfValues(Tangle, numValues)); 281 | elapsedSeconds = (decimal)(Time.Ticks - startTime) / Time.SecondInTicks; 282 | Console.WriteLine( 283 | "Read {0} values in ~{1:00.000} second(s) at ~{2:00000.00} values/sec.", 284 | numValues, elapsedSeconds, numValues / elapsedSeconds 285 | ); 286 | } 287 | 288 | protected IEnumerator WriteLotsOfValuesInBatch (Tangle tangle, int numIterations, int direction) { 289 | int batchSize = 256; 290 | Batch batch = null; 291 | 292 | if (direction > 0) 293 | for (int i = 0; i < numIterations; i++) { 294 | if (batch == null) 295 | batch = Tangle.CreateBatch(batchSize); 296 | 297 | batch.Add(i, i); 298 | 299 | if (batch.Count == batchSize) { 300 | yield return batch.Execute(); 301 | batch = null; 302 | } 303 | } 304 | else 305 | for (int i = numIterations - 1; i >= 0; i--) { 306 | if (batch == null) 307 | batch = Tangle.CreateBatch(batchSize); 308 | 309 | batch.Add(i, i); 310 | 311 | if (batch.Count == batchSize) { 312 | yield return batch.Execute(); 313 | batch = null; 314 | } 315 | } 316 | 317 | if (batch != null) 318 | yield return batch.Execute(); 319 | } 320 | 321 | [Test] 322 | public void BatchValuesInReverse () { 323 | const int numValues = 500000; 324 | 325 | long startTime = Time.Ticks; 326 | Scheduler.WaitFor(WriteLotsOfValuesInBatch(Tangle, numValues, -1)); 327 | decimal elapsedSeconds = (decimal)(Time.Ticks - startTime) / Time.SecondInTicks; 328 | Console.WriteLine( 329 | "Wrote {0} values in ~{1:00.000} second(s) at ~{2:00000.00} values/sec.", 330 | numValues, elapsedSeconds, numValues / elapsedSeconds 331 | ); 332 | 333 | startTime = Time.Ticks; 334 | Scheduler.WaitFor(CheckLotsOfValues(Tangle, numValues)); 335 | elapsedSeconds = (decimal)(Time.Ticks - startTime) / Time.SecondInTicks; 336 | Console.WriteLine( 337 | "Read {0} values in ~{1:00.000} second(s) at ~{2:00000.00} values/sec.", 338 | numValues, elapsedSeconds, numValues / elapsedSeconds 339 | ); 340 | } 341 | 342 | protected IEnumerator CheckLotsOfValues (Tangle tangle, int numIterations) { 343 | for (int i = 0; i < numIterations; i++) { 344 | var f = tangle.Get(i); 345 | yield return f; 346 | Assert.AreEqual(i, f.Result); 347 | } 348 | } 349 | 350 | [Test] 351 | public void TestCount () { 352 | Assert.AreEqual(0, Tangle.Count); 353 | Scheduler.WaitFor(Tangle.Add(1, 1)); 354 | Assert.AreEqual(1, Tangle.Count); 355 | Scheduler.WaitFor(Tangle.Add(2, 2)); 356 | Assert.AreEqual(2, Tangle.Count); 357 | Scheduler.WaitFor(Tangle.Add(2, 2)); 358 | Assert.AreEqual(2, Tangle.Count); 359 | Scheduler.WaitFor(Tangle.Set(1, 3)); 360 | Assert.AreEqual(2, Tangle.Count); 361 | } 362 | 363 | [Test] 364 | public void TestForeach () { 365 | const int numValues = 100000; 366 | 367 | Scheduler.WaitFor(WriteLotsOfValuesInBatch(Tangle, numValues, -1)); 368 | 369 | var keys = new List(); 370 | for (int i = 0; i < numValues; i += 2) 371 | keys.Add(i); 372 | 373 | long startTime = Time.Ticks; 374 | var bag = new ConcurrentBag(); 375 | var fForeach = Tangle.ForEach(keys, (key, value) => bag.Add(value)); 376 | Scheduler.WaitFor(fForeach); 377 | decimal elapsedSeconds = (decimal)(Time.Ticks - startTime) / Time.SecondInTicks; 378 | Console.WriteLine( 379 | "{0} values in ~{1:00.000} second(s) at ~{2:00000.00} values/sec.", 380 | keys.Count, elapsedSeconds, keys.Count / elapsedSeconds 381 | ); 382 | 383 | Assert.AreEqual(keys.Count, bag.Count()); 384 | 385 | Assert.AreEqual( 386 | keys.OrderBy((k) => k).ToArray(), 387 | bag.OrderBy((k) => k).ToArray() 388 | ); 389 | } 390 | 391 | [Test] 392 | public void TestMapReduce () { 393 | const int numValues = 100000; 394 | 395 | Scheduler.WaitFor(WriteLotsOfValuesInBatch(Tangle, numValues, -1)); 396 | 397 | var keys = new List(); 398 | for (int i = 0; i < numValues; i += 2) 399 | keys.Add(i); 400 | 401 | long startTime = Time.Ticks; 402 | var fReduced = Tangle.MapReduce( 403 | keys, 404 | (key, value) => (long)(value * 2), 405 | (lhs, rhs) => lhs + rhs 406 | ); 407 | var reduced = Scheduler.WaitFor(fReduced); 408 | decimal elapsedSeconds = (decimal)(Time.Ticks - startTime) / Time.SecondInTicks; 409 | Console.WriteLine( 410 | "{0} values in ~{1:00.000} second(s) at ~{2:00000.00} values/sec.", 411 | keys.Count, elapsedSeconds, keys.Count / elapsedSeconds 412 | ); 413 | } 414 | 415 | [Test] 416 | public void TestMultiGet () { 417 | const int numValues = 500000; 418 | 419 | Scheduler.WaitFor(WriteLotsOfValuesInBatch(Tangle, numValues, -1)); 420 | 421 | var keys = new List(); 422 | for (int i = 0; i < numValues; i += 2) 423 | keys.Add(i); 424 | 425 | long startTime = Time.Ticks; 426 | var fMultiGet = Tangle.Select(keys); 427 | var results = Scheduler.WaitFor(fMultiGet); 428 | decimal elapsedSeconds = (decimal)(Time.Ticks - startTime) / Time.SecondInTicks; 429 | Console.WriteLine( 430 | "Fetched {0} values in ~{1:00.000} second(s) at ~{2:00000.00} values/sec.", 431 | keys.Count, elapsedSeconds, keys.Count / elapsedSeconds 432 | ); 433 | 434 | Assert.AreEqual(keys.Count, results.Count()); 435 | 436 | Assert.AreEqual( 437 | keys.OrderBy((k) => k).ToArray(), 438 | results.ToArray() 439 | ); 440 | } 441 | 442 | [Test] 443 | public void TestMultiGetMissingValuesStillProduceAKeyValuePair () { 444 | var fMultiGet = Tangle.Select(new[] { 1, 2 }); 445 | var results = Scheduler.WaitFor(fMultiGet); 446 | 447 | Assert.AreEqual(default(int), results[0]); 448 | Assert.AreEqual(default(int), results[1]); 449 | } 450 | 451 | [Test] 452 | public void TestGetAllValues () { 453 | const int numValues = 100000; 454 | 455 | Scheduler.WaitFor(WriteLotsOfValuesInBatch(Tangle, numValues, -1)); 456 | 457 | long startTime = Time.Ticks; 458 | var fValues = Tangle.GetAllValues(); 459 | var values = Scheduler.WaitFor(fValues); 460 | decimal elapsedSeconds = (decimal)(Time.Ticks - startTime) / Time.SecondInTicks; 461 | Console.WriteLine( 462 | "Fetched {0} values in ~{1:00.000} second(s) at ~{2:00000.00} values/sec.", 463 | numValues, elapsedSeconds, numValues / elapsedSeconds 464 | ); 465 | 466 | Assert.AreEqual(numValues, values.Length); 467 | Assert.AreEqual( 468 | Enumerable.Range(0, numValues).ToArray(), 469 | values.OrderBy((v) => v).ToArray() 470 | ); 471 | } 472 | 473 | [Test] 474 | public void TestJoin () { 475 | for (int i = 0; i < 8; i++) 476 | Scheduler.WaitFor(Tangle.Set(i, i * 2)); 477 | 478 | using (var otherTangle = new Tangle(Scheduler, new SubStreamSource(Storage, "2_", false))) { 479 | var keys = new List(); 480 | for (int i = 0; i < 8; i++) { 481 | var key = new String((char)('a' + i), 1); 482 | keys.Add(key); 483 | Scheduler.WaitFor(otherTangle.Set(key, i)); 484 | } 485 | 486 | var joinResult = Scheduler.WaitFor( 487 | otherTangle.Join( 488 | Tangle, keys, 489 | (string leftKey, ref int leftValue) => 490 | leftValue, 491 | (string leftKey, ref int leftValue, int rightKey, ref int rightValue) => 492 | new { leftKey, leftValue, rightKey, rightValue } 493 | ) 494 | ); 495 | 496 | for (int i = 0; i < 8; i++) { 497 | var item = joinResult[i]; 498 | Assert.AreEqual(keys[i], item.leftKey); 499 | Assert.AreEqual(i, item.leftValue); 500 | Assert.AreEqual(i, item.rightKey); 501 | Assert.AreEqual(i * 2, item.rightValue); 502 | } 503 | } 504 | } 505 | 506 | [Test] 507 | public void TestSimpleJoin () { 508 | for (int i = 0; i < 8; i++) 509 | Scheduler.WaitFor(Tangle.Set(i, i * 2)); 510 | 511 | using (var otherTangle = new Tangle(Scheduler, new SubStreamSource(Storage, "2_", false))) { 512 | var keys = new List(); 513 | for (int i = 0; i < 8; i++) { 514 | var key = new String((char)('a' + i), 1); 515 | keys.Add(key); 516 | Scheduler.WaitFor(otherTangle.Set(key, i)); 517 | } 518 | 519 | var joinResult = Scheduler.WaitFor( 520 | otherTangle.Join( 521 | Tangle, keys, 522 | (leftValue) => leftValue 523 | ) 524 | ); 525 | 526 | for (int i = 0; i < 8; i++) { 527 | var item = joinResult[i]; 528 | Assert.AreEqual(i, item.Key); 529 | Assert.AreEqual(i * 2, item.Value); 530 | } 531 | } 532 | } 533 | 534 | [Test] 535 | public void TestCascadingSelect () { 536 | using (var otherTangle1 = new Tangle(Scheduler, new SubStreamSource(Storage, "2_", false))) 537 | using (var otherTangle2 = new Tangle(Scheduler, new SubStreamSource(Storage, "3_", false))) { 538 | Scheduler.WaitFor(Tangle.Set(1, 1)); 539 | Scheduler.WaitFor(otherTangle1.Set(2, 3)); 540 | Scheduler.WaitFor(otherTangle2.Set(2, 5)); 541 | Scheduler.WaitFor(otherTangle2.Set(3, 4)); 542 | 543 | var result = Scheduler.WaitFor(Tangle.CascadingSelect( 544 | new [] { otherTangle1, otherTangle2 }, 545 | new [] { 1, 2, 3, 4 } 546 | )); 547 | 548 | Assert.AreEqual(new [] { 1, 3, 4, default(int) }, result); 549 | } 550 | } 551 | 552 | [Test] 553 | public void TestSelfJoin () { 554 | Scheduler.WaitFor(Tangle.Set(1, 2)); 555 | Scheduler.WaitFor(Tangle.Set(2, 8)); 556 | Scheduler.WaitFor(Tangle.Set(3, 4)); 557 | Scheduler.WaitFor(Tangle.Set(4, 16)); 558 | 559 | var joinResult = Scheduler.WaitFor(Tangle.Join(Tangle, new[] { 1, 3 }, (left) => left)); 560 | 561 | Assert.AreEqual(2, joinResult[0].Key); 562 | Assert.AreEqual(8, joinResult[0].Value); 563 | Assert.AreEqual(4, joinResult[1].Key); 564 | Assert.AreEqual(16, joinResult[1].Value); 565 | } 566 | 567 | [Test] 568 | public void TestClear () { 569 | for (int i = 0; i < 100; i++) 570 | Scheduler.WaitFor(Tangle.Set(i, i)); 571 | 572 | Scheduler.WaitFor(Tangle.Clear()); 573 | 574 | for (int i = 0; i < 100; i++) { 575 | try { 576 | Scheduler.WaitFor(Tangle.Find(i)); 577 | Assert.Fail("Expected to throw"); 578 | } catch (FutureException fe) { 579 | Assert.IsInstanceOf(fe.InnerException); 580 | } 581 | 582 | try { 583 | Scheduler.WaitFor(Tangle.Get(i)); 584 | Assert.Fail("Expected to throw"); 585 | } catch (FutureException fe) { 586 | Assert.IsInstanceOf(fe.InnerException); 587 | } 588 | } 589 | 590 | Scheduler.WaitFor(Tangle.Set(2, 4)); 591 | Assert.AreEqual( 592 | 4, Scheduler.WaitFor(Tangle.Get(2)) 593 | ); 594 | } 595 | } 596 | 597 | [TestFixture] 598 | public class SynchronizationTests : BasicTestFixture { 599 | public Tangle Tangle; 600 | 601 | [SetUp] 602 | public override void SetUp () { 603 | base.SetUp(); 604 | 605 | Tangle = new Tangle( 606 | Scheduler, Storage, 607 | serializer: BlittableSerializer.Serialize, 608 | deserializer: BlittableSerializer.Deserialize, 609 | ownsStorage: true 610 | ); 611 | } 612 | 613 | [TearDown] 614 | public override void TearDown () { 615 | Tangle.Dispose(); 616 | base.TearDown(); 617 | } 618 | 619 | [Test] 620 | public void BarrierPreventsOperationsLaterInTheQueueFromCompleting () { 621 | var barrier = Tangle.CreateBarrier(); 622 | var fOperation = Tangle.Add(1, 1); 623 | Scheduler.WaitFor(barrier); 624 | Assert.AreEqual(0, Tangle.Count); 625 | barrier.Open(); 626 | Scheduler.WaitFor(fOperation); 627 | Assert.AreEqual(1, Tangle.Count); 628 | } 629 | 630 | [Test] 631 | public void DisposingOperationFutureCancelsTheOperation () { 632 | var barrier1 = Tangle.CreateBarrier(false); 633 | var fOperation = Tangle.Add(1, 1); 634 | var barrier2 = Tangle.CreateBarrier(true); 635 | fOperation.Dispose(); 636 | barrier1.Open(); 637 | Scheduler.WaitFor(barrier2); 638 | Assert.AreEqual(0, Tangle.Count); 639 | } 640 | 641 | [Test] 642 | public void DisposingTangleFailsPendingOperations () { 643 | var barrier = Tangle.CreateBarrier(false); 644 | var fOperation = Tangle.Add(1, 1); 645 | Scheduler.WaitFor(barrier); 646 | Tangle.Dispose(); 647 | 648 | Assert.Throws( 649 | () => Scheduler.WaitFor(fOperation) 650 | ); 651 | } 652 | 653 | [Test] 654 | public void BarrierCollectionOpensAllContainedBarriersWhenOpened () { 655 | var bc = new BarrierCollection(false, Tangle, Tangle); 656 | var fOperation = Tangle.Add(1, 1); 657 | 658 | Scheduler.WaitFor(bc); 659 | Assert.IsFalse(fOperation.Completed); 660 | bc.Open(); 661 | Scheduler.WaitFor(fOperation); 662 | Assert.AreEqual(1, Tangle.Count); 663 | } 664 | } 665 | 666 | [TestFixture] 667 | public class StringTests : BasicTestFixture { 668 | public Tangle Tangle; 669 | 670 | [SetUp] 671 | public unsafe override void SetUp () { 672 | base.SetUp(); 673 | 674 | var serializer = new Squared.Data.Mangler.Serialization.StringSerializer( 675 | Encoding.UTF8 676 | ); 677 | 678 | Tangle = new Tangle( 679 | Scheduler, Storage, 680 | serializer: serializer.Serialize, 681 | deserializer: serializer.Deserialize, 682 | ownsStorage: true 683 | ); 684 | } 685 | 686 | [TearDown] 687 | public override void TearDown () { 688 | Tangle.Dispose(); 689 | base.TearDown(); 690 | } 691 | 692 | [Test] 693 | public void OverwritingWithShorterStringWorks () { 694 | Scheduler.WaitFor(Tangle.Set("hello", "long string")); 695 | Assert.AreEqual("long string", Scheduler.WaitFor(Tangle.Get("hello"))); 696 | Scheduler.WaitFor(Tangle.Set("hello", "world")); 697 | Assert.AreEqual("world", Scheduler.WaitFor(Tangle.Get("hello"))); 698 | } 699 | 700 | [Test] 701 | public void OverwritingWithLongerStringWorks () { 702 | Scheduler.WaitFor(Tangle.Set("hello", "world")); 703 | Assert.AreEqual("world", Scheduler.WaitFor(Tangle.Get("hello"))); 704 | Scheduler.WaitFor(Tangle.Set("hello", "long string")); 705 | Assert.AreEqual("long string", Scheduler.WaitFor(Tangle.Get("hello"))); 706 | } 707 | 708 | [Test] 709 | public void UpdateGrowthWorks () { 710 | var s = "a"; 711 | Scheduler.WaitFor(Tangle.Set("test", s)); 712 | 713 | UpdateCallback callback = (str) => str + "a"; 714 | 715 | for (int i = 0; i < 10; i++) { 716 | s = s + "a"; 717 | 718 | Scheduler.WaitFor(Tangle.AddOrUpdate("test", null, callback)); 719 | 720 | Assert.AreEqual(s, Scheduler.WaitFor(Tangle.Get("test"))); 721 | } 722 | } 723 | 724 | [Test] 725 | public void UpdateShrinkageWorks () { 726 | var s = new String('a', 11); 727 | Scheduler.WaitFor(Tangle.Set("test", s)); 728 | 729 | UpdateCallback callback = (str) => str.Substring(0, str.Length - 1); 730 | 731 | for (int i = 0; i < 10; i++) { 732 | s = s.Substring(0, s.Length - 1); 733 | 734 | Scheduler.WaitFor(Tangle.AddOrUpdate("test", null, callback)); 735 | 736 | Assert.AreEqual(s, Scheduler.WaitFor(Tangle.Get("test"))); 737 | } 738 | } 739 | 740 | [Test] 741 | public void TestWastedDataBytes () { 742 | Assert.AreEqual(0, Tangle.WastedDataBytes); 743 | Scheduler.WaitFor(Tangle.Set(1, "abcd")); 744 | Assert.AreEqual(0, Tangle.WastedDataBytes); 745 | Scheduler.WaitFor(Tangle.Set(1, "abcdefgh")); 746 | Assert.AreEqual(0, Tangle.WastedDataBytes); 747 | Scheduler.WaitFor(Tangle.Set(1, "abcdefghijkl")); 748 | Assert.AreEqual(8, Tangle.WastedDataBytes); 749 | Scheduler.WaitFor(Tangle.Set(1, "abc")); 750 | Assert.AreEqual(8, Tangle.WastedDataBytes); 751 | Scheduler.WaitFor(Tangle.Set(1, "abcdefghijk")); 752 | Assert.AreEqual(8, Tangle.WastedDataBytes); 753 | } 754 | 755 | [Test] 756 | public void TestStoringHugeValue () { 757 | var hugeString = new String('a', 1024 * 1024 * 32); 758 | Scheduler.WaitFor(Tangle.Set(1, hugeString)); 759 | Assert.AreEqual(hugeString, Scheduler.WaitFor(Tangle.Get(1))); 760 | } 761 | 762 | [Test] 763 | public void TestZeroByteValue () { 764 | var emptyString = ""; 765 | Scheduler.WaitFor(Tangle.Set(1, emptyString)); 766 | Assert.AreEqual(emptyString, Scheduler.WaitFor(Tangle.Get(1))); 767 | } 768 | 769 | [Test] 770 | public void TestGrowFromZeroBytes () { 771 | string emptyString = "", largerString = "abcdefgh"; 772 | Scheduler.WaitFor(Tangle.Set(1, emptyString)); 773 | Assert.AreEqual(emptyString, Scheduler.WaitFor(Tangle.Get(1))); 774 | Scheduler.WaitFor(Tangle.Set(1, largerString)); 775 | Assert.AreEqual(largerString, Scheduler.WaitFor(Tangle.Get(1))); 776 | Assert.AreEqual(0, Tangle.WastedDataBytes); 777 | } 778 | 779 | [Test] 780 | public void TestRepeatedGrowAndShrink () { 781 | var wasted1 = Tangle.WastedDataBytes; 782 | 783 | for (int i = 1; i < 50; i += 10) { 784 | for (int j = 0; j < 20; j++) { 785 | var key = new TangleKey(String.Format("test{0}", j)); 786 | var text = new String((char)(j + 63), i); 787 | 788 | Scheduler.WaitFor(Tangle.Set(key, text)); 789 | 790 | Assert.AreEqual( 791 | text, Scheduler.WaitFor(Tangle.Get(key)) 792 | ); 793 | } 794 | } 795 | 796 | var wasted2 = Tangle.WastedDataBytes; 797 | Assert.Greater(wasted2, wasted1); 798 | 799 | for (int j = 0; j < 20; j++) { 800 | var key = new TangleKey(String.Format("test{0}", j)); 801 | var text = new String((char)(j + 63), 5); 802 | 803 | Scheduler.WaitFor(Tangle.Set(key, text)); 804 | 805 | Assert.AreEqual( 806 | text, Scheduler.WaitFor(Tangle.Get(key)) 807 | ); 808 | } 809 | 810 | var wasted3 = Tangle.WastedDataBytes; 811 | Assert.AreEqual(wasted3, wasted2); 812 | 813 | for (int j = 20; j < 40; j++) { 814 | var key = new TangleKey(String.Format("test{0}", j)); 815 | var text = new String((char)(j + 63), 10); 816 | 817 | Scheduler.WaitFor(Tangle.Set(key, text)); 818 | 819 | Assert.AreEqual( 820 | text, Scheduler.WaitFor(Tangle.Get(key)) 821 | ); 822 | } 823 | 824 | // This will fail if the freelist is not being used 825 | Assert.Less(Tangle.WastedDataBytes, wasted3, "Freelist not functioning"); 826 | } 827 | 828 | [Test] 829 | // Tests simple freelist behavior including pulling nodes from the tail 830 | public void TestFreelist () { 831 | Scheduler.WaitFor(Tangle.Set(1, "abcdefghijkl")); 832 | Scheduler.WaitFor(Tangle.Set(2, "abcdefgh")); 833 | Assert.AreEqual(0, Tangle.WastedDataBytes); 834 | 835 | Scheduler.WaitFor(Tangle.Set(1, "abcdefghijklmnop")); 836 | Assert.AreEqual(12, Tangle.WastedDataBytes); 837 | 838 | Scheduler.WaitFor(Tangle.Set(2, "abcdefghijklmnopqrst")); 839 | Assert.AreEqual(20, Tangle.WastedDataBytes); 840 | 841 | Scheduler.WaitFor(Tangle.Set(3, "abcdefghijkl")); 842 | Assert.AreEqual(8, Tangle.WastedDataBytes); 843 | 844 | Scheduler.WaitFor(Tangle.Set(4, "abcdefgh")); 845 | Assert.AreEqual(0, Tangle.WastedDataBytes); 846 | } 847 | 848 | [Test] 849 | public unsafe void TestLockExistingData () { 850 | var key = new TangleKey("test"); 851 | var value = "abcdefgh"; 852 | Scheduler.WaitFor(Tangle.Set(key, value)); 853 | 854 | var findResult = Scheduler.WaitFor(Tangle.Find(key)); 855 | var expectedBytes = Encoding.UTF8.GetBytes(value); 856 | 857 | fixed (byte * pExpected = expectedBytes) 858 | using (var data = Scheduler.WaitFor(findResult.LockData())) { 859 | Assert.AreEqual(expectedBytes.Length, data.Size); 860 | Assert.AreEqual(0, Native.memcmp( 861 | pExpected, data.Pointer, 862 | new UIntPtr((uint)Math.Min(data.Size, expectedBytes.Length)) 863 | )); 864 | } 865 | } 866 | 867 | [Test] 868 | public unsafe void TestGrowAndAlterDataByLockingIt () { 869 | var key = new TangleKey("test"); 870 | var value = "abcdefgh"; 871 | var newValue = "acdefghijkl"; 872 | Scheduler.WaitFor(Tangle.Set(key, value)); 873 | 874 | var findResult = Scheduler.WaitFor(Tangle.Find(key)); 875 | var newBytes = Encoding.UTF8.GetBytes(newValue); 876 | 877 | fixed (byte* pNewBytes = newBytes) 878 | using (var data = Scheduler.WaitFor(findResult.LockData(newBytes.Length))) { 879 | Assert.GreaterOrEqual(data.Size, newBytes.Length); 880 | Native.memmove(data.Pointer, pNewBytes, new UIntPtr((uint)newBytes.Length)); 881 | } 882 | 883 | Assert.AreEqual(newValue, Scheduler.WaitFor(Tangle.Get(key))); 884 | } 885 | 886 | [Test] 887 | public void TestCopyDataToStream () { 888 | var key = new TangleKey("test"); 889 | var value = new String('a', 1024 * 1024 * 2); 890 | 891 | Scheduler.WaitFor(Tangle.Set(key, value)); 892 | 893 | var findResult = Scheduler.WaitFor(Tangle.Find(key)); 894 | var ms = new MemoryStream(); 895 | 896 | Scheduler.WaitFor(findResult.CopyTo(ms, bufferSize: 64 * 1024)); 897 | 898 | var expectedBytes = Encoding.UTF8.GetBytes(value); 899 | var actualBytes = new byte[expectedBytes.Length]; 900 | 901 | ms.Seek(0, SeekOrigin.Begin); 902 | ms.Read(actualBytes, 0, expectedBytes.Length); 903 | 904 | Assert.AreEqual(expectedBytes, actualBytes); 905 | } 906 | 907 | [Test] 908 | public void TestCopyDataFromStream () { 909 | var key = new TangleKey("test"); 910 | var value = new String('a', 1024 * 1024 * 2); 911 | 912 | Scheduler.WaitFor(Tangle.Set(key, "")); 913 | 914 | var findResult = Scheduler.WaitFor(Tangle.Find(key)); 915 | 916 | var expectedBytes = Encoding.UTF8.GetBytes(value); 917 | var ms = new MemoryStream(expectedBytes); 918 | 919 | Scheduler.WaitFor(findResult.CopyFrom(ms, bufferSize: 64 * 1024)); 920 | 921 | var actualBytes = new byte[expectedBytes.Length]; 922 | ms.Seek(0, SeekOrigin.Begin); 923 | ms.Read(actualBytes, 0, expectedBytes.Length); 924 | 925 | Assert.AreEqual(expectedBytes, actualBytes); 926 | } 927 | } 928 | 929 | [TestFixture] 930 | public class KeyTests { 931 | [Test] 932 | public void TestKeyEquals () { 933 | var keyA = new TangleKey("abcd"); 934 | var keyB = new TangleKey("abcd"); 935 | var keyC = new TangleKey(2); 936 | var keyD = new TangleKey(2); 937 | 938 | Assert.IsTrue(keyA.Equals(keyA)); 939 | Assert.IsTrue(keyA.Equals(keyB)); 940 | Assert.IsTrue(keyC.Equals(keyC)); 941 | Assert.IsTrue(keyC.Equals(keyD)); 942 | Assert.IsFalse(keyA.Equals(keyC)); 943 | } 944 | } 945 | } 946 | --------------------------------------------------------------------------------