├── .gitignore ├── .nuget ├── NuGet.Config ├── NuGet.exe └── NuGet.targets ├── README.md ├── Rachis.Tests ├── BasicTests.cs ├── CommandsTests.cs ├── DictionaryCommand.cs ├── DictionaryStateMachine.cs ├── ElectionRelatedTests.cs ├── Http │ ├── ElectionRelatedTests.cs │ ├── HttpRaftTestsBase.cs │ └── HttpTransportPingTest.cs ├── NLog.config ├── Properties │ └── AssemblyInfo.cs ├── Rachis.Tests.csproj ├── RaftTestBaseWithHttp.cs ├── RaftTestsBase.cs ├── SnapshotTests.cs ├── TopologyChangesTests.cs └── packages.config ├── Rachis.sln ├── Rachis.userprefs ├── Rachis ├── Behaviors │ ├── AbstractRaftStateBehavior.cs │ ├── CandidateStateBehavior.cs │ ├── FollowerStateBehavior.cs │ ├── LeaderStateBehavior.cs │ ├── SnapshotInstallationStateBehavior.cs │ └── SteppingDownStateBehavior.cs ├── Commands │ ├── Command.cs │ ├── NopCommand.cs │ └── TopologyChangeCommand.cs ├── Interfaces │ ├── ICommandSerializer.cs │ ├── IRaftStateMachine.cs │ ├── ISnapshotWriter.cs │ └── ITransport.cs ├── JsonCommandSerializer.cs ├── MessageAbsolutePath.cs ├── Messages │ ├── AppendEntriesRequest.cs │ ├── AppendEntriesResponse.cs │ ├── BaseMessage.cs │ ├── CanInstallSnapshotRequest.cs │ ├── CanInstallSnapshotResponse.cs │ ├── InstallSnapshotRequest.cs │ ├── InstallSnapshotResponse.cs │ ├── LogEntry.cs │ ├── MessageContext.cs │ ├── RequestVoteRequest.cs │ ├── RequestVoteResponse.cs │ └── TimeoutNowRequest.cs ├── Properties │ └── AssemblyInfo.cs ├── Rachis.csproj ├── RaftEngine.cs ├── RaftEngineOptions.cs ├── RaftEngineState.cs ├── Storage │ ├── PersistentState.cs │ └── Topology.cs ├── Transport │ ├── HttpTransport.cs │ ├── HttpTransportBus.cs │ ├── HttpTransportSender.cs │ ├── InMemoryTransportHub.cs │ ├── NodeConnectionInfo.cs │ ├── RaftController.cs │ └── RaftWebApiConfig.cs ├── Utils │ └── NotLeadingException.cs └── packages.config ├── TailFeather.Client ├── Properties │ └── AssemblyInfo.cs ├── TailFeather.Client.csproj ├── TailFeatherClient.cs ├── TailFeatherTopology.cs └── packages.config ├── TailFeather ├── Controllers │ ├── AdminController.cs │ ├── KeyValueController.cs │ └── TailFeatherController.cs ├── NLog.config ├── Program.cs ├── Properties │ └── AssemblyInfo.cs ├── Storage │ ├── KeyValueOperation.cs │ ├── KeyValueOperationTypes.cs │ ├── KeyValueStateMachine.cs │ └── OperationBatchCommand.cs ├── TailFeather.csproj ├── TailFeatherCommandLineOptions.cs ├── app.config └── packages.config ├── Tryouts ├── App.config ├── Program.cs ├── Properties │ └── AssemblyInfo.cs ├── Tryouts.csproj └── packages.config └── license.txt /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.sln.docstates 8 | 9 | # Build results 10 | [Dd]ebug/ 11 | [Dd]ebugPublic/ 12 | [Rr]elease/ 13 | x64/ 14 | build/ 15 | bld/ 16 | [Bb]in/ 17 | [Oo]bj/ 18 | 19 | # MSTest test Results 20 | [Tt]est[Rr]esult*/ 21 | [Bb]uild[Ll]og.* 22 | 23 | #NUNIT 24 | *.VisualState.xml 25 | TestResult.xml 26 | 27 | # Build Results of an ATL Project 28 | [Dd]ebugPS/ 29 | [Rr]eleasePS/ 30 | dlldata.c 31 | 32 | *_i.c 33 | *_p.c 34 | *_i.h 35 | *.ilk 36 | *.meta 37 | *.obj 38 | *.pch 39 | *.pdb 40 | *.pgc 41 | *.pgd 42 | *.rsp 43 | *.sbr 44 | *.tlb 45 | *.tli 46 | *.tlh 47 | *.tmp 48 | *.tmp_proj 49 | *.log 50 | *.vspscc 51 | *.vssscc 52 | .builds 53 | *.pidb 54 | *.svclog 55 | *.scc 56 | 57 | # Chutzpah Test files 58 | _Chutzpah* 59 | 60 | # Visual C++ cache files 61 | ipch/ 62 | *.aps 63 | *.ncb 64 | *.opensdf 65 | *.sdf 66 | *.cachefile 67 | 68 | # Visual Studio profiler 69 | *.psess 70 | *.vsp 71 | *.vspx 72 | 73 | # TFS 2012 Local Workspace 74 | $tf/ 75 | 76 | # Guidance Automation Toolkit 77 | *.gpState 78 | 79 | # ReSharper is a .NET coding add-in 80 | _ReSharper*/ 81 | *.[Rr]e[Ss]harper 82 | *.DotSettings.user 83 | 84 | # JustCode is a .NET coding addin-in 85 | .JustCode 86 | 87 | # TeamCity is a build add-in 88 | _TeamCity* 89 | 90 | # DotCover is a Code Coverage Tool 91 | *.dotCover 92 | 93 | # NCrunch 94 | *.ncrunch* 95 | _NCrunch_* 96 | .*crunch*.local.xml 97 | 98 | # MightyMoose 99 | *.mm.* 100 | AutoTest.Net/ 101 | 102 | # Web workbench (sass) 103 | .sass-cache/ 104 | 105 | # Installshield output folder 106 | [Ee]xpress/ 107 | 108 | # DocProject is a documentation generator add-in 109 | DocProject/buildhelp/ 110 | DocProject/Help/*.HxT 111 | DocProject/Help/*.HxC 112 | DocProject/Help/*.hhc 113 | DocProject/Help/*.hhk 114 | DocProject/Help/*.hhp 115 | DocProject/Help/Html2 116 | DocProject/Help/html 117 | 118 | # Click-Once directory 119 | publish/ 120 | 121 | # Publish Web Output 122 | *.[Pp]ublish.xml 123 | *.azurePubxml 124 | 125 | # NuGet Packages Directory 126 | packages/ 127 | ## TODO: If the tool you use requires repositories.config uncomment the next line 128 | #!packages/repositories.config 129 | 130 | # Enable "build/" folder in the NuGet Packages folder since NuGet packages use it for MSBuild targets 131 | # This line needs to be after the ignore of the build folder (and the packages folder if the line above has been uncommented) 132 | !packages/build/ 133 | 134 | # Windows Azure Build Output 135 | csx/ 136 | *.build.csdef 137 | 138 | # Windows Store app package directory 139 | AppPackages/ 140 | 141 | # Others 142 | sql/ 143 | *.Cache 144 | ClientBin/ 145 | [Ss]tyle[Cc]op.* 146 | ~$* 147 | *~ 148 | *.dbmdl 149 | *.dbproj.schemaview 150 | *.pfx 151 | *.publishsettings 152 | node_modules/ 153 | 154 | # RIA/Silverlight projects 155 | Generated_Code/ 156 | 157 | # Backup & report files from converting an old project file to a newer 158 | # Visual Studio version. Backup files are not needed, because we have git ;-) 159 | _UpgradeReport_Files/ 160 | Backup*/ 161 | UpgradeLog*.XML 162 | UpgradeLog*.htm 163 | 164 | # SQL Server files 165 | *.mdf 166 | *.ldf 167 | 168 | # Business Intelligence projects 169 | *.rdl.data 170 | *.bim.layout 171 | *.bim_*.settings 172 | 173 | # Microsoft Fakes 174 | FakesAssemblies/ 175 | -------------------------------------------------------------------------------- /.nuget/NuGet.Config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.nuget/NuGet.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ayende/Rachis/fc623a48f3ee2f170f2f178d0ff04fe085c84a96/.nuget/NuGet.exe -------------------------------------------------------------------------------- /.nuget/NuGet.targets: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | $(MSBuildProjectDirectory)\..\ 5 | 6 | 7 | false 8 | 9 | 10 | false 11 | 12 | 13 | true 14 | 15 | 16 | false 17 | 18 | 19 | 20 | 21 | 22 | 26 | 27 | 28 | 29 | 30 | $([System.IO.Path]::Combine($(SolutionDir), ".nuget")) 31 | 32 | 33 | 34 | 35 | $(SolutionDir).nuget 36 | 37 | 38 | 39 | $(MSBuildProjectDirectory)\packages.$(MSBuildProjectName.Replace(' ', '_')).config 40 | $(MSBuildProjectDirectory)\packages.$(MSBuildProjectName).config 41 | 42 | 43 | 44 | $(MSBuildProjectDirectory)\packages.config 45 | $(PackagesProjectConfig) 46 | 47 | 48 | 49 | 50 | $(NuGetToolsPath)\NuGet.exe 51 | @(PackageSource) 52 | 53 | "$(NuGetExePath)" 54 | mono --runtime=v4.0.30319 "$(NuGetExePath)" 55 | 56 | $(TargetDir.Trim('\\')) 57 | 58 | -RequireConsent 59 | -NonInteractive 60 | 61 | "$(SolutionDir) " 62 | "$(SolutionDir)" 63 | 64 | 65 | $(NuGetCommand) install "$(PackagesConfig)" -source "$(PackageSources)" $(NonInteractiveSwitch) $(RequireConsentSwitch) -solutionDir $(PaddedSolutionDir) 66 | $(NuGetCommand) pack "$(ProjectPath)" -Properties "Configuration=$(Configuration);Platform=$(Platform)" $(NonInteractiveSwitch) -OutputDirectory "$(PackageOutputDir)" -symbols 67 | 68 | 69 | 70 | RestorePackages; 71 | $(BuildDependsOn); 72 | 73 | 74 | 75 | 76 | $(BuildDependsOn); 77 | BuildPackage; 78 | 79 | 80 | 81 | 82 | 83 | 84 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 99 | 100 | 103 | 104 | 105 | 106 | 108 | 109 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 141 | 142 | 143 | 144 | 145 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Rhino.Raft 2 | ---------- 3 | Implementation of Raft protocol for use with RavenDB 4 | Reference on the protocol can be found here https://ramcloud.stanford.edu/wiki/download/attachments/11370504/raft.pdf 5 | -------------------------------------------------------------------------------- /Rachis.Tests/BasicTests.cs: -------------------------------------------------------------------------------- 1 | using Xunit; 2 | using Xunit.Extensions; 3 | 4 | namespace Rachis.Tests 5 | { 6 | public class BasicTests : RaftTestsBase 7 | { 8 | [Theory] 9 | [InlineData(2)] 10 | [InlineData(3)] 11 | [InlineData(5)] 12 | [InlineData(7)] 13 | public void CanApplyCommitAcrossAllCluster(int amount) 14 | { 15 | var leader = CreateNetworkAndGetLeader(amount); 16 | var commits = WaitForCommitsOnCluster(machine => 17 | machine.Data.ContainsKey("4")); 18 | for (int i = 0; i < 5; i++) 19 | { 20 | leader.AppendCommand(new DictionaryCommand.Set 21 | { 22 | Key = i.ToString(), 23 | Value = i 24 | }); 25 | } 26 | commits.Wait(); 27 | 28 | foreach (var node in Nodes) 29 | { 30 | for (int i = 0; i < 5; i++) 31 | { 32 | var dictionary = ((DictionaryStateMachine)node.StateMachine).Data; 33 | Assert.Equal(i, dictionary[i.ToString()]); 34 | } 35 | } 36 | } 37 | } 38 | } -------------------------------------------------------------------------------- /Rachis.Tests/CommandsTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | using FizzWare.NBuilder; 7 | using FluentAssertions; 8 | using Rachis.Messages; 9 | using Xunit; 10 | using Xunit.Extensions; 11 | 12 | namespace Rachis.Tests 13 | { 14 | public class CommandsTests : RaftTestsBase 15 | { 16 | [Fact] 17 | public void When_command_committed_CompletionTaskSource_is_notified() 18 | { 19 | const int CommandCount = 5; 20 | var leader = CreateNetworkAndGetLeader(3); 21 | var commands = Builder.CreateListOfSize(CommandCount) 22 | .All() 23 | .With(x => x.Completion = new TaskCompletionSource()) 24 | .With(x => x.AssignedIndex = -1) 25 | .Build() 26 | .ToList(); 27 | 28 | 29 | var nonLeaderNode = Nodes.First(x => x.State != RaftEngineState.Leader); 30 | var commitsAppliedEvent = new ManualResetEventSlim(); 31 | 32 | nonLeaderNode.CommitIndexChanged += (oldIndex, newIndex) => 33 | { 34 | //CommandCount + 1 --> take into account NOP command that leader sends after election 35 | if (newIndex == CommandCount + 1) 36 | commitsAppliedEvent.Set(); 37 | }; 38 | 39 | commands.ForEach(leader.AppendCommand); 40 | 41 | Assert.True(commitsAppliedEvent.Wait(nonLeaderNode.Options.ElectionTimeout * 2)); 42 | commands.Should().OnlyContain(cmd => cmd.Completion.Task.Status == TaskStatus.RanToCompletion); 43 | } 44 | 45 | //this test is a show-case of how to check for command commit time-out 46 | [Fact] 47 | public void Command_not_committed_after_timeout_CompletionTaskSource_is_notified() 48 | { 49 | const int CommandCount = 5; 50 | var leader = CreateNetworkAndGetLeader(3); 51 | var commands = Builder.CreateListOfSize(CommandCount) 52 | .All() 53 | .With(x => x.Completion = new TaskCompletionSource()) 54 | .With(x => x.AssignedIndex = -1) 55 | .Build() 56 | .ToList(); 57 | 58 | 59 | var nonLeaderNode = Nodes.First(x => x.State != RaftEngineState.Leader); 60 | var commitsAppliedEvent = new ManualResetEventSlim(); 61 | 62 | nonLeaderNode.CommitIndexChanged += (oldIndex, newIndex) => 63 | { 64 | //essentially fire event for (CommandCount - 1) + Nop command 65 | if (newIndex == CommandCount) 66 | commitsAppliedEvent.Set(); 67 | }; 68 | 69 | //don't append the last command yet 70 | commands.Take(CommandCount - 1).ToList().ForEach(leader.AppendCommand); 71 | //make sure commands that were appended before network leader disconnection are replicated 72 | Assert.True(commitsAppliedEvent.Wait(nonLeaderNode.Options.ElectionTimeout * 3)); 73 | 74 | DisconnectNode(leader.Name); 75 | 76 | var lastCommand = commands.Last(); 77 | var commandCompletionTask = lastCommand.Completion.Task; 78 | 79 | leader.AppendCommand(lastCommand); 80 | 81 | var aggregateException = Assert.Throws(() => commandCompletionTask.Wait(leader.Options.ElectionTimeout * 2)); 82 | Assert.IsType(aggregateException.InnerException); 83 | } 84 | 85 | [Theory] 86 | [InlineData(2)] 87 | [InlineData(3)] 88 | public void Leader_AppendCommand_for_first_time_should_distribute_commands_between_nodes(int nodeCount) 89 | { 90 | const int CommandCount = 5; 91 | var commandsToDistribute = Builder.CreateListOfSize(CommandCount) 92 | .All() 93 | .With(x => x.Completion = null) 94 | .Build() 95 | .ToList(); 96 | 97 | var leader = CreateNetworkAndGetLeader(nodeCount); 98 | var entriesAppended = new Dictionary>(); 99 | Nodes.ToList().ForEach(node => 100 | { 101 | entriesAppended.Add(node.Name, new List()); 102 | node.EntriesAppended += logEntries => entriesAppended[node.Name].AddRange(logEntries); 103 | }); 104 | 105 | 106 | var nonLeaderNode = Nodes.First(x => x.State != RaftEngineState.Leader); 107 | var commitsAppliedEvent = new ManualResetEventSlim(); 108 | if (nonLeaderNode.CommitIndex == CommandCount + 1) //precaution 109 | commitsAppliedEvent.Set(); 110 | nonLeaderNode.CommitIndexChanged += (oldIndex, newIndex) => 111 | { 112 | //CommandCount + 1 --> take into account NOP command that leader sends after election 113 | if (newIndex == CommandCount + 1) 114 | commitsAppliedEvent.Set(); 115 | }; 116 | 117 | commandsToDistribute.ForEach(leader.AppendCommand); 118 | 119 | var millisecondsTimeout = 10000 * nodeCount; 120 | Assert.True(commitsAppliedEvent.Wait(millisecondsTimeout), "within " + millisecondsTimeout + " sec. non leader node should have all relevant commands committed"); 121 | } 122 | 123 | [Theory] 124 | [InlineData(3)] 125 | [InlineData(2)] 126 | [InlineData(10)] 127 | public void Leader_AppendCommand_several_times_should_distribute_commands_between_nodes(int nodeCount) 128 | { 129 | const int CommandCount = 5; 130 | var commands = Builder.CreateListOfSize(CommandCount * 2) 131 | .All() 132 | .With(x => x.Completion = null) 133 | .Build() 134 | .ToList(); 135 | 136 | var leader = CreateNetworkAndGetLeader(nodeCount, messageTimeout: 10000); 137 | var entriesAppended = new Dictionary>(); 138 | Nodes.ToList().ForEach(node => 139 | { 140 | entriesAppended.Add(node.Name, new List()); 141 | node.EntriesAppended += logEntries => entriesAppended[node.Name].AddRange(logEntries); 142 | }); 143 | 144 | var nonLeaderNode = Nodes.First(x => x.State != RaftEngineState.Leader); 145 | var commitsAppliedEvent = new ManualResetEventSlim(); 146 | nonLeaderNode.CommitApplied += (cmd) => 147 | { 148 | if (cmd.AssignedIndex == commands.Last().AssignedIndex) 149 | commitsAppliedEvent.Set(); 150 | }; 151 | 152 | commands.Take(CommandCount).ToList().ForEach(leader.AppendCommand); 153 | commands.Skip(CommandCount).ToList().ForEach(leader.AppendCommand); 154 | 155 | var millisecondsTimeout = 10000 * nodeCount; 156 | Assert.True(commitsAppliedEvent.Wait(millisecondsTimeout), "within " + millisecondsTimeout + " sec. non leader node should have all relevant commands committed"); 157 | 158 | var committedCommands = nonLeaderNode.PersistentState.LogEntriesAfter(0).Select(x => nonLeaderNode.PersistentState.CommandSerializer.Deserialize(x.Data)) 159 | .OfType().ToList(); 160 | 161 | Assert.Equal(10, committedCommands.Count); 162 | for (int i = 0; i < 10; i++) 163 | { 164 | Assert.Equal(commands[i].Value, committedCommands[i].Value); 165 | Assert.Equal(commands[i].AssignedIndex, committedCommands[i].AssignedIndex); 166 | } 167 | } 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /Rachis.Tests/DictionaryCommand.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Concurrent; 2 | using Rachis.Commands; 3 | 4 | namespace Rachis.Tests 5 | { 6 | public abstract class DictionaryCommand : Command 7 | { 8 | public abstract void Apply(ConcurrentDictionary data); 9 | 10 | public string Key { get; set; } 11 | 12 | public class Set : DictionaryCommand 13 | { 14 | public int Value { get; set; } 15 | 16 | public override void Apply(ConcurrentDictionary data) 17 | { 18 | data[Key] = Value; 19 | } 20 | } 21 | 22 | public class Inc : DictionaryCommand 23 | { 24 | public int Value { get; set; } 25 | 26 | public override void Apply(ConcurrentDictionary data) 27 | { 28 | int value; 29 | data.TryGetValue(Key, out value); 30 | data[Key] = value + Value; 31 | } 32 | } 33 | 34 | 35 | public class Del : DictionaryCommand 36 | { 37 | public override void Apply(ConcurrentDictionary data) 38 | { 39 | int value; 40 | data.TryRemove(Key, out value); 41 | } 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /Rachis.Tests/DictionaryStateMachine.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.IO; 5 | using System.Threading; 6 | using Newtonsoft.Json; 7 | using Rachis.Commands; 8 | using Rachis.Interfaces; 9 | using Rachis.Messages; 10 | 11 | namespace Rachis.Tests 12 | { 13 | public class DictionaryStateMachine : IRaftStateMachine 14 | { 15 | private readonly JsonSerializer _serializer = new JsonSerializer(); 16 | 17 | public long LastAppliedIndex 18 | { 19 | get { return _lastAppliedIndex; } 20 | private set { Thread.VolatileWrite(ref _lastAppliedIndex, value); } 21 | } 22 | 23 | private class SnapshotWriter : ISnapshotWriter 24 | { 25 | private readonly Dictionary _snapshot; 26 | private readonly DictionaryStateMachine _parent; 27 | 28 | public SnapshotWriter(DictionaryStateMachine parent, Dictionary snapshot) 29 | { 30 | _parent = parent; 31 | _snapshot = snapshot; 32 | } 33 | 34 | public void Dispose() 35 | { 36 | } 37 | 38 | public long Index { get; set; } 39 | public long Term { get; set; } 40 | public void WriteSnapshot(Stream stream) 41 | { 42 | var streamWriter = new StreamWriter(stream); 43 | _parent._serializer.Serialize(streamWriter, _snapshot); 44 | streamWriter.Flush(); 45 | } 46 | } 47 | 48 | public ConcurrentDictionary Data = new ConcurrentDictionary(); 49 | private SnapshotWriter _snapshot; 50 | private long _lastAppliedIndex; 51 | 52 | public void Apply(LogEntry entry, Command cmd) 53 | { 54 | if (LastAppliedIndex >= entry.Index) 55 | throw new InvalidOperationException("Already applied " + entry.Index); 56 | 57 | LastAppliedIndex = entry.Index; 58 | 59 | var dicCommand = cmd as DictionaryCommand; 60 | 61 | if (dicCommand != null) 62 | dicCommand.Apply(Data); 63 | } 64 | 65 | public bool SupportSnapshots { get { return true; }} 66 | 67 | public void CreateSnapshot(long index, long term, ManualResetEventSlim allowFurtherModifications) 68 | { 69 | _snapshot = new SnapshotWriter(this, new Dictionary(Data)) 70 | { 71 | Term = term, 72 | Index = index 73 | }; 74 | allowFurtherModifications.Set(); 75 | } 76 | 77 | public ISnapshotWriter GetSnapshotWriter() 78 | { 79 | return _snapshot; 80 | } 81 | 82 | public void ApplySnapshot(long term, long index, Stream stream) 83 | { 84 | if(stream.CanSeek) 85 | stream.Position = 0; 86 | 87 | using (var streamReader = new StreamReader(stream)) 88 | Data = new ConcurrentDictionary(_serializer.Deserialize>(new JsonTextReader(streamReader))); 89 | 90 | _snapshot = new SnapshotWriter(this, new Dictionary(Data)) 91 | { 92 | Term = term, 93 | Index = index 94 | }; 95 | } 96 | 97 | public void Dispose() 98 | { 99 | //nothing to do 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /Rachis.Tests/ElectionRelatedTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | using FizzWare.NBuilder; 8 | using FluentAssertions; 9 | using Rachis.Storage; 10 | using Rachis.Transport; 11 | using Voron; 12 | using Xunit; 13 | using Xunit.Extensions; 14 | 15 | namespace Rachis.Tests 16 | { 17 | public class ElectionRelatedTests : RaftTestsBase 18 | { 19 | [Fact] 20 | public void Follower_as_a_single_node_becomes_leader_automatically() 21 | { 22 | var hub = new InMemoryTransportHub(); 23 | var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly(); 24 | storageEnvironmentOptions.OwnsPagers = false; 25 | 26 | var raftEngineOptions = new RaftEngineOptions( 27 | new NodeConnectionInfo { Name = "node1" }, 28 | storageEnvironmentOptions, 29 | hub.CreateTransportFor("node1"), 30 | new DictionaryStateMachine() 31 | ) 32 | { 33 | ElectionTimeout = 1000, 34 | HeartbeatTimeout = 1000/6 35 | }; 36 | 37 | PersistentState.ClusterBootstrap(raftEngineOptions); 38 | storageEnvironmentOptions.OwnsPagers = true; 39 | 40 | using (var raftNode = new RaftEngine(raftEngineOptions)) 41 | { 42 | Assert.Equal(RaftEngineState.Leader, raftNode.State); 43 | } 44 | } 45 | 46 | 47 | [Fact] 48 | public void Network_partition_should_cause_message_resend() 49 | { 50 | var leader = CreateNetworkAndGetLeader(3, messageTimeout: 300); 51 | 52 | var countdown = new CountdownEvent(2); 53 | leader.ElectionStarted += () => 54 | { 55 | if (countdown.CurrentCount > 0) 56 | countdown.Signal(); 57 | }; 58 | WriteLine("Disconnecting network"); 59 | for (int i = 0; i < 3; i++) 60 | { 61 | DisconnectNode("node" + i); 62 | DisconnectNodeSending("node" + i); 63 | } 64 | 65 | for (int i = 0; i < 5; i++) 66 | { 67 | ForceTimeout(leader.Name); 68 | 69 | } 70 | Assert.True(countdown.Wait(1500)); 71 | 72 | for (int i = 0; i < 3; i++) 73 | { 74 | ReconnectNode("node" + i); 75 | ReconnectNodeSending("node" + i); 76 | } 77 | 78 | Assert.True(Nodes.First().WaitForLeader()); 79 | } 80 | 81 | /* 82 | * This test deals with network "partition" -> leader is detached from the rest of the nodes (simulation of network issues) 83 | * Before the network is partitioned the leader distributes the first three commands, then the partition happens. 84 | * Then the detached leader has 2 more commands appended - but because of network partition, they are not distributed to other nodes 85 | * When communication is restored, the leader from before becomes follower, and the new leader makes roll back on log of former leader, 86 | * so only the first three commands are in the log of former leader node 87 | */ 88 | [Theory] 89 | [InlineData(2)] 90 | [InlineData(3)] 91 | [InlineData(5)] 92 | [InlineData(7)] 93 | public void Network_partition_for_more_time_than_timeout_can_be_healed(int nodeCount) 94 | { 95 | const int CommandCount = 5; 96 | var commands = Builder.CreateListOfSize(CommandCount) 97 | .All() 98 | .With(x => x.Completion = new TaskCompletionSource()) 99 | .With(x => x.AssignedIndex = -1) 100 | .Build() 101 | .ToList(); 102 | 103 | var leader = CreateNetworkAndGetLeader(nodeCount); 104 | 105 | var nonLeaderNode = Nodes.First(x => x.State != RaftEngineState.Leader); 106 | var commitsAppliedEvent = new ManualResetEventSlim(); 107 | nonLeaderNode.CommitIndexChanged += (oldIndex, newIndex) => 108 | { 109 | if (newIndex == commands[2].AssignedIndex) 110 | commitsAppliedEvent.Set(); 111 | }; 112 | 113 | commands.Take(3).ToList().ForEach(leader.AppendCommand); 114 | var waitForCommitsOnCluster = WaitForCommitsOnCluster(machine => machine.LastAppliedIndex == commands[2].AssignedIndex); 115 | Assert.True(commitsAppliedEvent.Wait(5000)); //with in-memory transport it shouldn't take more than 5 sec 116 | 117 | var steppedDown = WaitForStateChange(leader, RaftEngineState.FollowerAfterStepDown); 118 | var candidancies = Nodes.Where(x=>x!=leader).Select(node => WaitForStateChange(node, RaftEngineState.Candidate)).ToArray(); 119 | 120 | WriteLine(" (" + leader.Name + ")"); 121 | DisconnectNode(leader.Name); 122 | 123 | commands.Skip(3).ToList().ForEach(leader.AppendCommand); 124 | var formerLeader = leader; 125 | 126 | Assert.True(steppedDown.Wait(leader.Options.ElectionTimeout * 2)); 127 | Assert.True(WaitHandle.WaitAny(candidancies.Select(x => x.WaitHandle).ToArray(), leader.Options.ElectionTimeout*2) != WaitHandle.WaitTimeout); 128 | WriteLine(" (" + leader.Name + ")"); 129 | ReconnectNode(leader.Name); 130 | 131 | foreach (var raftEngine in Nodes) 132 | { 133 | Assert.True(raftEngine.WaitForLeader()); 134 | } 135 | 136 | leader = Nodes.FirstOrDefault(x => x.State == RaftEngineState.Leader); 137 | Assert.NotNull(leader); 138 | 139 | Assert.True(waitForCommitsOnCluster.Wait(3000)); 140 | var committedCommands = formerLeader.PersistentState.LogEntriesAfter(0).Select(x => nonLeaderNode.PersistentState.CommandSerializer.Deserialize(x.Data)) 141 | .OfType() 142 | .ToList(); 143 | for (int i = 0; i < 3; i++) 144 | { 145 | commands[i].Value.Should().Be(committedCommands[i].Value); 146 | commands[i].AssignedIndex.Should().Be(committedCommands[i].AssignedIndex); 147 | } 148 | } 149 | 150 | [Theory] 151 | [InlineData(2)] 152 | [InlineData(3)] 153 | [InlineData(4)] 154 | [InlineData(5)] 155 | public void Network_partition_for_less_time_than_timeout_can_be_healed_without_elections(int nodeCount) 156 | { 157 | const int CommandCount = 5; 158 | var commands = Builder.CreateListOfSize(CommandCount) 159 | .All() 160 | .With(x => x.Completion = new TaskCompletionSource()) 161 | .With(x => x.AssignedIndex = -1) 162 | .Build() 163 | .ToList(); 164 | 165 | var leader = CreateNetworkAndGetLeader(nodeCount, messageTimeout: 1500); 166 | 167 | var nonLeaderNode = Nodes.First(x => x.State != RaftEngineState.Leader); 168 | 169 | commands.Take(CommandCount - 1).ToList().ForEach(leader.AppendCommand); 170 | while (nonLeaderNode.CommitIndex < 2) //make sure at least one command is committed 171 | Thread.Sleep(50); 172 | 173 | WriteLine(" (" + leader.Name + ")"); 174 | DisconnectNode(leader.Name); 175 | 176 | DictionaryCommand.Set command = commands.Last(); 177 | leader.AppendCommand(command); 178 | 179 | var waitForCommitsOnCluster = WaitForCommitsOnCluster(machine => machine.LastAppliedIndex == command.AssignedIndex); 180 | 181 | WriteLine(" (" + leader.Name + ")"); 182 | ReconnectNode(leader.Name); 183 | Assert.Equal(RaftEngineState.Leader, leader.State); 184 | Assert.True(waitForCommitsOnCluster.Wait(3000)); 185 | 186 | var committedCommands = nonLeaderNode.PersistentState.LogEntriesAfter(0).Select(x => nonLeaderNode.PersistentState.CommandSerializer.Deserialize(x.Data)) 187 | .OfType() 188 | .ToList(); 189 | for (int i = 0; i < CommandCount; i++) 190 | { 191 | commands[i].Value.Should().Be(committedCommands[i].Value); 192 | commands[i].AssignedIndex.Should().Be(committedCommands[i].AssignedIndex); 193 | } 194 | } 195 | 196 | [Theory] 197 | [InlineData(2)] 198 | [InlineData(3)] 199 | public void On_many_node_network_after_leader_establishment_all_nodes_know_who_is_leader(int nodeCount) 200 | { 201 | var leader = CreateNetworkAndGetLeader(nodeCount); 202 | var raftNodes = Nodes.ToList(); 203 | 204 | var leadersOfNodes = raftNodes.Select(x => x.CurrentLeader).ToList(); 205 | 206 | leadersOfNodes.Should().NotContainNulls("After leader is established, all nodes should know that leader exists"); 207 | leadersOfNodes.Should().OnlyContain(l => l.Equals(leader.Name, StringComparison.InvariantCultureIgnoreCase), 208 | "after leader establishment, all nodes should know only one, selected leader"); 209 | } 210 | 211 | [Fact] 212 | public void Follower_on_timeout_should_become_candidate() 213 | { 214 | var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly(); 215 | storageEnvironmentOptions.OwnsPagers = false; 216 | 217 | var nodeOptions = new RaftEngineOptions(new NodeConnectionInfo { Name = "real" }, storageEnvironmentOptions, _inMemoryTransportHub.CreateTransportFor("real"), new DictionaryStateMachine()); 218 | 219 | PersistentState.SetTopologyExplicitly(nodeOptions, 220 | new Topology( 221 | new Guid("355a589b-cadc-463d-a515-5add2ea47205"), 222 | new[] 223 | { 224 | new NodeConnectionInfo {Name = "real"}, new NodeConnectionInfo {Name = "u2"}, new NodeConnectionInfo {Name = "pj"}, 225 | }, new NodeConnectionInfo[0], new NodeConnectionInfo[0]), throwIfTopologyExists: true); 226 | storageEnvironmentOptions.OwnsPagers = true; 227 | 228 | using (var node = new RaftEngine(nodeOptions)) 229 | { 230 | var timeoutEvent = new ManualResetEventSlim(); 231 | node.StateTimeout += timeoutEvent.Set; 232 | 233 | ForceTimeout("real"); 234 | 235 | timeoutEvent.Wait(); 236 | Assert.Equal(RaftEngineState.Candidate, node.State); 237 | } 238 | } 239 | 240 | [Fact] 241 | public void AllPeers_and_AllVotingPeers_can_be_persistantly_saved_and_loaded() 242 | { 243 | var cancellationTokenSource = new CancellationTokenSource(); 244 | 245 | var path = "test" + Guid.NewGuid(); 246 | try 247 | { 248 | var expectedAllVotingPeers = new List { "Node123", "Node1", "Node2", "NodeG", "NodeB", "NodeABC" }; 249 | 250 | using (var options = StorageEnvironmentOptions.ForPath(path)) 251 | { 252 | using (var persistentState = new PersistentState("self",options, cancellationTokenSource.Token) 253 | { 254 | CommandSerializer = new JsonCommandSerializer() 255 | }) 256 | { 257 | var currentConfiguration = persistentState.GetCurrentTopology(); 258 | Assert.Empty(currentConfiguration.AllVotingNodes); 259 | 260 | var currentTopology = new Topology(new Guid("355a589b-cadc-463d-a515-5add2ea47205"), 261 | expectedAllVotingPeers.Select(x => new NodeConnectionInfo { Name = x }), Enumerable.Empty(), Enumerable.Empty()); 262 | persistentState.SetCurrentTopology(currentTopology, 1); 263 | } 264 | } 265 | using (var options = StorageEnvironmentOptions.ForPath(path)) 266 | { 267 | using (var persistentState = new PersistentState("self", options, cancellationTokenSource.Token) 268 | { 269 | CommandSerializer = new JsonCommandSerializer() 270 | }) 271 | { 272 | var currentConfiguration = persistentState.GetCurrentTopology(); 273 | Assert.Equal(expectedAllVotingPeers.Count, currentConfiguration.AllVotingNodes.Count()); 274 | foreach (var nodeConnectionInfo in currentConfiguration.AllVotingNodes) 275 | { 276 | Assert.True(expectedAllVotingPeers.Contains(nodeConnectionInfo.Name)); 277 | } 278 | 279 | } 280 | } 281 | } 282 | finally 283 | { 284 | new DirectoryInfo(path).Delete(true); 285 | } 286 | } 287 | 288 | [Fact] 289 | public void Request_vote_when_leader_exists_will_be_rejected() 290 | { 291 | var node = CreateNetworkAndGetLeader(3); 292 | 293 | node.State.Should().Be(RaftEngineState.Leader); 294 | 295 | 296 | 297 | } 298 | } 299 | } 300 | -------------------------------------------------------------------------------- /Rachis.Tests/Http/ElectionRelatedTests.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using System.Threading; 3 | 4 | using Xunit; 5 | using Xunit.Extensions; 6 | 7 | namespace Rachis.Tests.Http 8 | { 9 | public class ElectionRelatedTests : HttpRaftTestsBase 10 | { 11 | [Theory] 12 | [InlineData(11)] 13 | [InlineData(13)] 14 | [InlineData(15)] 15 | public void LeaderShouldStayLeader(int numberOfNodes) 16 | { 17 | var leader = CreateNetworkAndGetLeader(numberOfNodes); 18 | 19 | Thread.Sleep(numberOfNodes * 2000); 20 | 21 | Assert.Equal(RaftEngineState.Leader, leader.RaftEngine.State); 22 | 23 | foreach (var node in AllNodes.Where(node => node != leader)) 24 | Assert.Equal(RaftEngineState.Follower, node.RaftEngine.State); 25 | } 26 | } 27 | } -------------------------------------------------------------------------------- /Rachis.Tests/Http/HttpRaftTestsBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading; 5 | using System.Web.Http; 6 | 7 | using Microsoft.Owin.Hosting; 8 | 9 | using Owin; 10 | 11 | using Rachis.Commands; 12 | using Rachis.Storage; 13 | using Rachis.Transport; 14 | 15 | using Voron; 16 | 17 | using Xunit; 18 | 19 | namespace Rachis.Tests.Http 20 | { 21 | public abstract class HttpRaftTestsBase : IDisposable 22 | { 23 | private const int PortRangeStart = 9000; 24 | 25 | private static int numberOfPortRequests; 26 | 27 | protected readonly List AllNodes = new List(); 28 | 29 | public static IEnumerable Nodes 30 | { 31 | get 32 | { 33 | return new[] 34 | { 35 | new object[] { 1 }, 36 | new object[] { 3 }, 37 | new object[] { 5 }, 38 | new object[] { 7 }, 39 | new object[] { 11 } 40 | }; 41 | } 42 | } 43 | 44 | protected RaftNode CreateNetworkAndGetLeader(int numberOfNodes) 45 | { 46 | var nodes = Enumerable 47 | .Range(0, numberOfNodes) 48 | .Select(i => new RaftNode(GetPort())) 49 | .ToList(); 50 | 51 | AllNodes.AddRange(nodes); 52 | 53 | var allNodesFinishedJoining = new ManualResetEventSlim(); 54 | 55 | var random = new Random(); 56 | var leader = nodes[random.Next(0, numberOfNodes - 1)]; 57 | 58 | Console.WriteLine("Leader: " + leader.RaftEngine.Options.SelfConnection.Uri); 59 | 60 | InitializeTopology(leader); 61 | 62 | Assert.True(leader.RaftEngine.WaitForLeader()); 63 | Assert.Equal(RaftEngineState.Leader, leader.RaftEngine.State); 64 | 65 | leader.RaftEngine.TopologyChanged += command => 66 | { 67 | if (command.Requested.AllNodeNames.All(command.Requested.IsVoter)) 68 | { 69 | allNodesFinishedJoining.Set(); 70 | } 71 | }; 72 | 73 | for (var i = 0; i < numberOfNodes; i++) 74 | { 75 | var n = nodes[i]; 76 | 77 | if (n == leader) 78 | continue; 79 | 80 | Assert.Equal(RaftEngineState.Leader, leader.RaftEngine.State); 81 | Assert.True(leader.RaftEngine.AddToClusterAsync(new NodeConnectionInfo 82 | { 83 | Name = n.Name, 84 | Uri = new Uri(n.Url) 85 | }).Wait(3000)); 86 | } 87 | 88 | if (numberOfNodes == 1) 89 | allNodesFinishedJoining.Set(); 90 | 91 | Assert.True(allNodesFinishedJoining.Wait(10000 * numberOfNodes), "Not all nodes become voters. " + leader.RaftEngine.CurrentTopology); 92 | Assert.True(leader.RaftEngine.WaitForLeader()); 93 | 94 | return leader; 95 | } 96 | 97 | private static void InitializeTopology(RaftNode node) 98 | { 99 | var topologyId = Guid.NewGuid(); 100 | var topology = new Topology(topologyId, new List { node.RaftEngine.Options.SelfConnection }, Enumerable.Empty(), Enumerable.Empty()); 101 | 102 | var tcc = new TopologyChangeCommand 103 | { 104 | Requested = topology 105 | }; 106 | 107 | node.RaftEngine.PersistentState.SetCurrentTopology(tcc.Requested, 0); 108 | node.RaftEngine.StartTopologyChange(tcc); 109 | node.RaftEngine.CommitTopologyChange(tcc); 110 | node.RaftEngine.CurrentLeader = null; 111 | } 112 | 113 | private static int GetPort() 114 | { 115 | var portRequest = Interlocked.Increment(ref numberOfPortRequests); 116 | return PortRangeStart - (portRequest % 25); 117 | } 118 | 119 | public void Dispose() 120 | { 121 | var exceptions = new List(); 122 | 123 | foreach (var node in AllNodes) 124 | { 125 | try 126 | { 127 | node.Dispose(); 128 | } 129 | catch (Exception e) 130 | { 131 | exceptions.Add(e); 132 | } 133 | } 134 | 135 | if (exceptions.Count > 0) 136 | throw new AggregateException(exceptions); 137 | } 138 | } 139 | 140 | public class RaftNode : IDisposable 141 | { 142 | private RaftEngine _raftEngine; 143 | 144 | private IDisposable _server; 145 | 146 | private string _name; 147 | 148 | private string _url; 149 | 150 | public RaftEngine RaftEngine 151 | { 152 | get 153 | { 154 | return _raftEngine; 155 | } 156 | } 157 | 158 | public string Name 159 | { 160 | get 161 | { 162 | return _name; 163 | } 164 | } 165 | 166 | public string Url 167 | { 168 | get 169 | { 170 | return _url; 171 | } 172 | } 173 | 174 | public RaftNode(int port) 175 | { 176 | _name = "node-" + port; 177 | _url = string.Format("http://{0}:{1}", Environment.MachineName, port); 178 | 179 | var nodeTransport = new HttpTransport(_name); 180 | 181 | var node1 = new NodeConnectionInfo { Name = _name, Uri = new Uri(_url) }; 182 | var engineOptions = new RaftEngineOptions( 183 | node1, 184 | StorageEnvironmentOptions.CreateMemoryOnly(), 185 | nodeTransport, 186 | new DictionaryStateMachine()); 187 | 188 | engineOptions.ElectionTimeout *= 2; 189 | engineOptions.HeartbeatTimeout *= 2; 190 | 191 | _raftEngine = new RaftEngine(engineOptions); 192 | 193 | _server = WebApp.Start(new StartOptions 194 | { 195 | Urls = { string.Format("http://+:{0}/", port) } 196 | }, builder => 197 | { 198 | var httpConfiguration = new HttpConfiguration(); 199 | RaftWebApiConfig.Load(); 200 | httpConfiguration.MapHttpAttributeRoutes(); 201 | httpConfiguration.Properties[typeof(HttpTransportBus)] = nodeTransport.Bus; 202 | builder.UseWebApi(httpConfiguration); 203 | }); 204 | } 205 | 206 | public void Dispose() 207 | { 208 | var toDispose = new[] { _raftEngine, _server }; 209 | var exceptions = new List(); 210 | 211 | foreach (var disposable in toDispose) 212 | { 213 | if (disposable == null) 214 | continue; 215 | 216 | try 217 | { 218 | disposable.Dispose(); 219 | } 220 | catch (Exception e) 221 | { 222 | exceptions.Add(e); 223 | } 224 | } 225 | 226 | if (exceptions.Count > 0) 227 | throw new AggregateException(exceptions); 228 | } 229 | } 230 | } -------------------------------------------------------------------------------- /Rachis.Tests/Http/HttpTransportPingTest.cs: -------------------------------------------------------------------------------- 1 | // ----------------------------------------------------------------------- 2 | // 3 | // Copyright (c) Hibernating Rhinos LTD. All rights reserved. 4 | // 5 | // ----------------------------------------------------------------------- 6 | 7 | using System; 8 | using System.Collections.Generic; 9 | using System.Diagnostics; 10 | using System.IO; 11 | using System.Threading; 12 | using System.Web.Http; 13 | 14 | using Microsoft.Owin.Hosting; 15 | 16 | using Newtonsoft.Json; 17 | 18 | using Owin; 19 | 20 | using Rachis.Messages; 21 | using Rachis.Storage; 22 | using Rachis.Transport; 23 | 24 | using Voron; 25 | 26 | using Xunit; 27 | 28 | namespace Rachis.Tests.Http 29 | { 30 | public class HttpTransportPingTest : IDisposable 31 | { 32 | private readonly IDisposable _server; 33 | private readonly RaftEngine _raftEngine; 34 | private readonly int _timeout = Debugger.IsAttached ? 50 * 1000 : 5*1000; 35 | private readonly HttpTransport _node1Transport; 36 | 37 | public HttpTransportPingTest() 38 | { 39 | _node1Transport = new HttpTransport("node1"); 40 | 41 | var node1 = new NodeConnectionInfo { Name = "node1", Uri = new Uri("http://localhost:9079") }; 42 | var engineOptions = new RaftEngineOptions(node1, StorageEnvironmentOptions.CreateMemoryOnly(), _node1Transport, new DictionaryStateMachine()) 43 | { 44 | ElectionTimeout = 60 * 1000, 45 | HeartbeatTimeout = 10 * 1000 46 | }; 47 | _raftEngine = new RaftEngine(engineOptions); 48 | 49 | _server = WebApp.Start(new StartOptions 50 | { 51 | Urls = { "http://+:9079/" } 52 | }, builder => 53 | { 54 | var httpConfiguration = new HttpConfiguration(); 55 | RaftWebApiConfig.Load(); 56 | httpConfiguration.MapHttpAttributeRoutes(); 57 | httpConfiguration.Properties[typeof(HttpTransportBus)] = _node1Transport.Bus; 58 | builder.UseWebApi(httpConfiguration); 59 | }); 60 | } 61 | 62 | [Fact] 63 | public void CanSendRequestVotesAndGetReply() 64 | { 65 | using (var node2Transport = new HttpTransport("node2")) 66 | { 67 | var node1 = new NodeConnectionInfo { Name = "node1", Uri = new Uri("http://localhost:9079") }; 68 | node2Transport.Send(node1, new RequestVoteRequest 69 | { 70 | TrialOnly = true, 71 | From = "node2", 72 | ClusterTopologyId = new Guid("355a589b-cadc-463d-a515-5add2ea47205"), 73 | Term = 3, 74 | LastLogIndex = 2, 75 | LastLogTerm = 2, 76 | }); 77 | 78 | MessageContext context; 79 | var gotIt = node2Transport.TryReceiveMessage(_timeout, CancellationToken.None, out context); 80 | 81 | Assert.True(gotIt); 82 | 83 | Assert.True(context.Message is RequestVoteResponse); 84 | } 85 | } 86 | 87 | 88 | [Fact] 89 | public void CanSendTimeoutNow() 90 | { 91 | using (var node2Transport = new HttpTransport("node2")) 92 | { 93 | var node1 = new NodeConnectionInfo { Name = "node1", Uri = new Uri("http://localhost:9079") }; 94 | node2Transport.Send(node1, new AppendEntriesRequest 95 | { 96 | From = "node2", 97 | ClusterTopologyId = new Guid("355a589b-cadc-463d-a515-5add2ea47205"), 98 | Term = 2, 99 | PrevLogIndex = 0, 100 | PrevLogTerm = 0, 101 | LeaderCommit = 1, 102 | Entries = new[] 103 | { 104 | new LogEntry 105 | { 106 | Term = 2, 107 | Index = 1, 108 | Data = new JsonCommandSerializer().Serialize(new DictionaryCommand.Set 109 | { 110 | Key = "a", 111 | Value = 2 112 | }) 113 | }, 114 | } 115 | }); 116 | MessageContext context; 117 | var gotIt = node2Transport.TryReceiveMessage(_timeout, CancellationToken.None, out context); 118 | 119 | Assert.True(gotIt); 120 | Assert.True(((AppendEntriesResponse)context.Message).Success); 121 | 122 | var mres = new ManualResetEventSlim(); 123 | _raftEngine.StateChanged += state => 124 | { 125 | if (state == RaftEngineState.CandidateByRequest) 126 | mres.Set(); 127 | }; 128 | 129 | node2Transport.Send(node1, new TimeoutNowRequest 130 | { 131 | Term = 4, 132 | From = "node2", 133 | ClusterTopologyId = new Guid("355a589b-cadc-463d-a515-5add2ea47205"), 134 | }); 135 | 136 | gotIt = node2Transport.TryReceiveMessage(_timeout, CancellationToken.None, out context); 137 | 138 | Assert.True(gotIt); 139 | 140 | Assert.True(context.Message is NothingToDo); 141 | 142 | Assert.True(mres.Wait(_timeout)); 143 | } 144 | } 145 | 146 | [Fact] 147 | public void CanAskIfCanInstallSnapshot() 148 | { 149 | using (var node2Transport = new HttpTransport("node2")) 150 | { 151 | var node1 = new NodeConnectionInfo { Name = "node1", Uri = new Uri("http://localhost:9079") }; 152 | 153 | node2Transport.Send(node1, new CanInstallSnapshotRequest 154 | { 155 | From = "node2", 156 | ClusterTopologyId = new Guid("355a589b-cadc-463d-a515-5add2ea47205"), 157 | Term = 2, 158 | Index = 3, 159 | }); 160 | 161 | 162 | MessageContext context; 163 | var gotIt = node2Transport.TryReceiveMessage(_timeout, CancellationToken.None, out context); 164 | 165 | Assert.True(gotIt); 166 | var msg = (CanInstallSnapshotResponse)context.Message; 167 | Assert.True(msg.Success); 168 | } 169 | } 170 | 171 | [Fact] 172 | public void CanSendEntries() 173 | { 174 | using (var node2Transport = new HttpTransport("node2")) 175 | { 176 | var node1 = new NodeConnectionInfo { Name = "node1", Uri = new Uri("http://localhost:9079") }; 177 | 178 | 179 | node2Transport.Send(node1, new AppendEntriesRequest 180 | { 181 | From = "node2", 182 | ClusterTopologyId = new Guid("355a589b-cadc-463d-a515-5add2ea47205"), 183 | Term = 2, 184 | PrevLogIndex = 0, 185 | PrevLogTerm = 0, 186 | LeaderCommit = 1, 187 | Entries = new LogEntry[] 188 | { 189 | new LogEntry 190 | { 191 | Term = 2, 192 | Index = 1, 193 | Data = new JsonCommandSerializer().Serialize(new DictionaryCommand.Set 194 | { 195 | Key = "a", 196 | Value = 2 197 | }) 198 | }, 199 | } 200 | }); 201 | 202 | 203 | MessageContext context; 204 | var gotIt = node2Transport.TryReceiveMessage(_timeout, CancellationToken.None, out context); 205 | 206 | Assert.True(gotIt); 207 | 208 | var appendEntriesResponse = (AppendEntriesResponse)context.Message; 209 | Assert.True(appendEntriesResponse.Success); 210 | 211 | Assert.Equal(2, ((DictionaryStateMachine)_raftEngine.StateMachine).Data["a"]); 212 | } 213 | } 214 | 215 | [Fact] 216 | public void CanInstallSnapshot() 217 | { 218 | using (var node2Transport = new HttpTransport("node2")) 219 | { 220 | var node1 = new NodeConnectionInfo { Name = "node1", Uri = new Uri("http://localhost:9079") }; 221 | 222 | 223 | node2Transport.Send(node1, new CanInstallSnapshotRequest 224 | { 225 | From = "node2", 226 | ClusterTopologyId = new Guid("355a589b-cadc-463d-a515-5add2ea47205"), 227 | Term = 2, 228 | Index = 3, 229 | }); 230 | 231 | MessageContext context; 232 | var gotIt = node2Transport.TryReceiveMessage(_timeout, CancellationToken.None, out context); 233 | Assert.True(gotIt); 234 | Assert.True(context.Message is CanInstallSnapshotResponse); 235 | 236 | node2Transport.Stream(node1, new InstallSnapshotRequest 237 | { 238 | From = "node2", 239 | ClusterTopologyId = new Guid("355a589b-cadc-463d-a515-5add2ea47205"), 240 | Term = 2, 241 | Topology = new Topology(new Guid("355a589b-cadc-463d-a515-5add2ea47205")), 242 | LastIncludedIndex = 2, 243 | LastIncludedTerm = 2, 244 | }, stream => 245 | { 246 | var streamWriter = new StreamWriter(stream); 247 | var data = new Dictionary { { "a", 2 } }; 248 | new JsonSerializer().Serialize(streamWriter, data); 249 | streamWriter.Flush(); 250 | }); 251 | 252 | 253 | gotIt = node2Transport.TryReceiveMessage(_timeout, CancellationToken.None, out context); 254 | 255 | Assert.True(gotIt); 256 | 257 | var appendEntriesResponse = (InstallSnapshotResponse)context.Message; 258 | Assert.True(appendEntriesResponse.Success); 259 | 260 | Assert.Equal(2, ((DictionaryStateMachine)_raftEngine.StateMachine).Data["a"]); 261 | } 262 | } 263 | 264 | public void Dispose() 265 | { 266 | _server.Dispose(); 267 | _raftEngine.Dispose(); 268 | _node1Transport.Dispose(); 269 | 270 | } 271 | } 272 | } -------------------------------------------------------------------------------- /Rachis.Tests/NLog.config: -------------------------------------------------------------------------------- 1 |  2 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /Rachis.Tests/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("Rhino.Raft.Tests")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("Rhino.Raft.Tests")] 13 | [assembly: AssemblyCopyright("Copyright © 2014")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("9cb4564d-01c0-4a1a-9a70-e91fe6648867")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /Rachis.Tests/Rachis.Tests.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {61B3D01A-F286-42B1-AA6B-83D9D6DF0873} 8 | Library 9 | Properties 10 | Rachis.Tests 11 | Rachis.Tests 12 | 512 13 | 14 | ..\ 15 | true 16 | v4.5 17 | 18 | 19 | true 20 | full 21 | false 22 | bin\Debug\ 23 | DEBUG;TRACE 24 | prompt 25 | 4 26 | 27 | 28 | pdbonly 29 | true 30 | bin\Release\ 31 | TRACE 32 | prompt 33 | 4 34 | 35 | 36 | 37 | ..\packages\NBuilder.3.0.1.1\lib\FizzWare.NBuilder.dll 38 | 39 | 40 | ..\packages\FluentAssertions.3.0.107\lib\net45\FluentAssertions.dll 41 | 42 | 43 | ..\packages\FluentAssertions.3.0.107\lib\net45\FluentAssertions.Core.dll 44 | 45 | 46 | ..\packages\Microsoft.Owin.2.0.2\lib\net45\Microsoft.Owin.dll 47 | 48 | 49 | ..\packages\Microsoft.Owin.Host.HttpListener.2.0.2\lib\net45\Microsoft.Owin.Host.HttpListener.dll 50 | 51 | 52 | ..\packages\Microsoft.Owin.Hosting.2.0.2\lib\net45\Microsoft.Owin.Hosting.dll 53 | 54 | 55 | False 56 | ..\packages\Newtonsoft.Json.6.0.4\lib\net45\Newtonsoft.Json.dll 57 | 58 | 59 | False 60 | ..\packages\NLog.3.1.0.0\lib\net45\NLog.dll 61 | 62 | 63 | ..\packages\Owin.1.0\lib\net40\Owin.dll 64 | 65 | 66 | 67 | 68 | 69 | False 70 | ..\packages\Microsoft.AspNet.WebApi.Client.5.2.2\lib\net45\System.Net.Http.Formatting.dll 71 | 72 | 73 | ..\packages\Rx-Core.2.2.5\lib\net45\System.Reactive.Core.dll 74 | 75 | 76 | ..\packages\Rx-Interfaces.2.2.5\lib\net45\System.Reactive.Interfaces.dll 77 | 78 | 79 | ..\packages\Rx-Linq.2.2.5\lib\net45\System.Reactive.Linq.dll 80 | 81 | 82 | ..\packages\Rx-PlatformServices.2.2.5\lib\net45\System.Reactive.PlatformServices.dll 83 | 84 | 85 | False 86 | ..\packages\Microsoft.AspNet.WebApi.Core.5.2.2\lib\net45\System.Web.Http.dll 87 | 88 | 89 | ..\packages\Microsoft.AspNet.WebApi.Owin.5.2.2\lib\net45\System.Web.Http.Owin.dll 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | ..\packages\xunit.1.9.2\lib\net20\xunit.dll 98 | 99 | 100 | ..\packages\xunit.extensions.1.9.2\lib\net20\xunit.extensions.dll 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | PreserveNewest 120 | 121 | 122 | 123 | 124 | 125 | {FF83C7C2-BC7B-4DCC-A782-49EF9BBD9390} 126 | Voron 127 | 128 | 129 | {F796F69F-D17B-4260-92D6-65CB94C0E05C} 130 | Rachis 131 | 132 | 133 | 134 | 135 | 136 | 137 | This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. 138 | 139 | 140 | 141 | 148 | -------------------------------------------------------------------------------- /Rachis.Tests/RaftTestBaseWithHttp.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | using Rhino.Raft.Interfaces; 8 | using Xunit; 9 | 10 | namespace Rhino.Raft.Tests 11 | { 12 | public class RaftTestBaseWithHttp : RaftTestsBase 13 | { 14 | private const short FirstPort = 8100; 15 | 16 | private readonly string _localHost; 17 | private readonly List _disposables; 18 | 19 | 20 | public RaftTestBaseWithHttp() 21 | { 22 | _disposables = new List(); 23 | var isFiddlerActive = 24 | Process.GetProcesses().Any(p => p.ProcessName.Equals("fiddler", StringComparison.InvariantCultureIgnoreCase)); 25 | _localHost = isFiddlerActive ? "localhost.fiddler" : "localhost"; 26 | } 27 | 28 | [Fact] 29 | public void Nodes_should_be_able_to_elect_via_http_transport() 30 | { 31 | 32 | } 33 | 34 | public override void Dispose() 35 | { 36 | base.Dispose(); 37 | foreach(var disposable in _disposables) 38 | disposable.Dispose(); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /Rachis.Tests/RaftTestsBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Diagnostics; 5 | using System.Linq; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | using NLog; 9 | using Rachis.Storage; 10 | using Rachis.Transport; 11 | using Voron; 12 | using Xunit; 13 | 14 | namespace Rachis.Tests 15 | { 16 | public class RaftTestsBase : IDisposable 17 | { 18 | private readonly List _nodes = new List(); 19 | 20 | private readonly Logger _log = LogManager.GetCurrentClassLogger(); 21 | protected readonly InMemoryTransportHub _inMemoryTransportHub; 22 | 23 | protected void ForceTimeout(string name) 24 | { 25 | ((InMemoryTransportHub.InMemoryTransport)_inMemoryTransportHub.CreateTransportFor(name)).ForceTimeout(); 26 | } 27 | 28 | protected void DisconnectNodeSending(string name) 29 | { 30 | _inMemoryTransportHub.DisconnectNodeSending(name); 31 | } 32 | 33 | protected void DisconnectNode(string name) 34 | { 35 | _inMemoryTransportHub.DisconnectNode(name); 36 | } 37 | 38 | protected void ReconnectNodeSending(string name) 39 | { 40 | _inMemoryTransportHub.ReconnectNodeSending(name); 41 | } 42 | 43 | protected void ReconnectNode(string name) 44 | { 45 | _inMemoryTransportHub.ReconnectNode(name); 46 | } 47 | 48 | public RaftTestsBase() 49 | { 50 | _inMemoryTransportHub = new InMemoryTransportHub(); 51 | } 52 | 53 | protected void WriteLine(string format, params object[] args) 54 | { 55 | _log.Error(format, args); 56 | } 57 | 58 | public IEnumerable Nodes { get { return _nodes; } } 59 | 60 | protected ManualResetEventSlim WaitForStateChange(RaftEngine node, RaftEngineState requestedState) 61 | { 62 | var mre = new ManualResetEventSlim(); 63 | node.StateChanged += state => 64 | { 65 | if (state == requestedState) 66 | mre.Set(); 67 | }; 68 | return mre; 69 | } 70 | 71 | protected ManualResetEventSlim WaitForToplogyChange(RaftEngine node) 72 | { 73 | var mre = new ManualResetEventSlim(); 74 | node.TopologyChanged += state => 75 | { 76 | if (node.CurrentTopology.HasVoters) 77 | mre.Set(); 78 | }; 79 | return mre; 80 | } 81 | 82 | protected ManualResetEventSlim WaitForCommit(RaftEngine node, Func predicate) 83 | { 84 | var cde = new ManualResetEventSlim(); 85 | node.CommitApplied += command => 86 | { 87 | if (predicate((DictionaryStateMachine)node.StateMachine)) 88 | cde.Set(); 89 | }; 90 | node.SnapshotInstalled += () => 91 | { 92 | var state = (DictionaryStateMachine)node.StateMachine; 93 | if (predicate(state)) 94 | { 95 | cde.Set(); 96 | } 97 | }; 98 | return cde; 99 | } 100 | 101 | protected ManualResetEventSlim WaitForSnapshot(RaftEngine node) 102 | { 103 | var cde = new ManualResetEventSlim(); 104 | node.CreatedSnapshot += cde.Set; 105 | return cde; 106 | } 107 | 108 | protected CountdownEvent WaitForCommitsOnCluster(int numberOfCommits) 109 | { 110 | var cde = new CountdownEvent(_nodes.Count); 111 | foreach (var node in _nodes) 112 | { 113 | var n = node; 114 | if (n.CommitIndex == numberOfCommits && cde.CurrentCount > 0) 115 | { 116 | cde.Signal(); 117 | continue; 118 | } 119 | n.CommitApplied += command => 120 | { 121 | if (n.CommitIndex == numberOfCommits && cde.CurrentCount > 0) 122 | cde.Signal(); 123 | }; 124 | n.SnapshotInstalled += () => 125 | { 126 | if (n.CommitIndex == numberOfCommits && cde.CurrentCount > 0) 127 | cde.Signal(); 128 | }; 129 | } 130 | 131 | return cde; 132 | } 133 | 134 | protected CountdownEvent WaitForCommitsOnCluster(Func predicate) 135 | { 136 | var cde = new CountdownEvent(_nodes.Count); 137 | var votedAlready = new ConcurrentDictionary(); 138 | 139 | foreach (var node in _nodes) 140 | { 141 | var n = node; 142 | n.CommitApplied += command => 143 | { 144 | var state = (DictionaryStateMachine)n.StateMachine; 145 | if (predicate(state) && cde.CurrentCount > 0) 146 | { 147 | if (votedAlready.ContainsKey(n)) 148 | return; 149 | votedAlready.TryAdd(n, n); 150 | _log.Debug("WaitForCommitsOnCluster match " + n.Name + " " + state.Data.Count); 151 | cde.Signal(); 152 | } 153 | }; 154 | n.SnapshotInstalled += () => 155 | { 156 | var state = (DictionaryStateMachine)n.StateMachine; 157 | if (predicate(state) && cde.CurrentCount > 0) 158 | { 159 | if (votedAlready.ContainsKey(n)) 160 | return; 161 | votedAlready.TryAdd(n, n); 162 | 163 | _log.Debug("WaitForCommitsOnCluster match"); 164 | cde.Signal(); 165 | } 166 | }; 167 | } 168 | 169 | return cde; 170 | } 171 | 172 | 173 | protected CountdownEvent WaitForToplogyChangeOnCluster(List raftNodes = null) 174 | { 175 | raftNodes = raftNodes ?? _nodes; 176 | var cde = new CountdownEvent(raftNodes.Count); 177 | foreach (var node in raftNodes) 178 | { 179 | var n = node; 180 | n.TopologyChanged += (a) => 181 | { 182 | if (cde.CurrentCount > 0) 183 | { 184 | cde.Signal(); 185 | } 186 | }; 187 | } 188 | 189 | return cde; 190 | } 191 | protected ManualResetEventSlim WaitForSnapshotInstallation(RaftEngine node) 192 | { 193 | var cde = new ManualResetEventSlim(); 194 | node.SnapshotInstalled += cde.Set; 195 | return cde; 196 | } 197 | 198 | protected Task WaitForNewLeaderAsync() 199 | { 200 | var rcs = new TaskCompletionSource(); 201 | foreach (var node in _nodes) 202 | { 203 | var n = node; 204 | 205 | n.ElectedAsLeader += () => rcs.TrySetResult(n); 206 | } 207 | 208 | return rcs.Task; 209 | } 210 | 211 | protected void RestartAllNodes() 212 | { 213 | foreach (var raftEngine in _nodes) 214 | { 215 | raftEngine.Options.StorageOptions.OwnsPagers = false; 216 | raftEngine.Dispose(); 217 | } 218 | for (int i = 0; i < _nodes.Count; i++) 219 | { 220 | _nodes[i] = new RaftEngine(_nodes[i].Options); 221 | } 222 | } 223 | 224 | protected RaftEngine CreateNetworkAndGetLeader(int nodeCount, int messageTimeout = -1) 225 | { 226 | var leaderIndex = new Random().Next(0, nodeCount); 227 | if (messageTimeout == -1) 228 | messageTimeout = Debugger.IsAttached ? 3 * 1000 : 500; 229 | var nodeNames = new string[nodeCount]; 230 | for (int i = 0; i < nodeCount; i++) 231 | { 232 | nodeNames[i] = "node" + i; 233 | } 234 | 235 | WriteLine("{0} selected as seed", nodeNames[leaderIndex]); 236 | var allNodesFinishedJoining = new ManualResetEventSlim(); 237 | for (int index = 0; index < nodeNames.Length; index++) 238 | { 239 | var nodeName = nodeNames[index]; 240 | var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly(); 241 | storageEnvironmentOptions.OwnsPagers = false; 242 | var options = CreateNodeOptions(nodeName, messageTimeout, storageEnvironmentOptions, nodeNames); 243 | if (leaderIndex == index) 244 | { 245 | PersistentState.ClusterBootstrap(options); 246 | } 247 | storageEnvironmentOptions.OwnsPagers = true; 248 | var engine = new RaftEngine(options); 249 | _nodes.Add(engine); 250 | if (leaderIndex == index) 251 | { 252 | engine.TopologyChanged += command => 253 | { 254 | if (command.Requested.AllNodeNames.All(command.Requested.IsVoter)) 255 | { 256 | allNodesFinishedJoining.Set(); 257 | } 258 | }; 259 | for (int i = 0; i < nodeNames.Length; i++) 260 | { 261 | if (i == leaderIndex) 262 | continue; 263 | Assert.True(engine.AddToClusterAsync(new NodeConnectionInfo { Name = nodeNames[i] }).Wait(3000)); 264 | } 265 | } 266 | } 267 | if (nodeCount == 1) 268 | allNodesFinishedJoining.Set(); 269 | Assert.True(allNodesFinishedJoining.Wait(5000 * nodeCount)); 270 | 271 | var raftEngine = _nodes[leaderIndex]; 272 | 273 | 274 | var transport = (InMemoryTransportHub.InMemoryTransport)_inMemoryTransportHub.CreateTransportFor(raftEngine.Name); 275 | transport.ForceTimeout(); 276 | Assert.True(_nodes[leaderIndex].WaitForLeader()); 277 | var leader = _nodes.FirstOrDefault(x => x.State == RaftEngineState.Leader); 278 | Assert.NotNull(leader); 279 | 280 | return _nodes[leaderIndex]; 281 | } 282 | 283 | private RaftEngineOptions CreateNodeOptions(string nodeName, int messageTimeout, StorageEnvironmentOptions storageOptions, params string[] peers) 284 | { 285 | var nodeOptions = new RaftEngineOptions(new NodeConnectionInfo { Name = nodeName }, 286 | storageOptions, 287 | _inMemoryTransportHub.CreateTransportFor(nodeName), 288 | new DictionaryStateMachine()) 289 | { 290 | ElectionTimeout = messageTimeout, 291 | HeartbeatTimeout = messageTimeout / 6, 292 | Stopwatch = Stopwatch.StartNew() 293 | }; 294 | return nodeOptions; 295 | } 296 | 297 | protected bool AreEqual(byte[] array1, byte[] array2) 298 | { 299 | if (array1.Length != array2.Length) 300 | return false; 301 | 302 | return !array1.Where((t, i) => t != array2[i]).Any(); 303 | } 304 | 305 | 306 | protected RaftEngine NewNodeFor(RaftEngine leader) 307 | { 308 | var raftEngine = new RaftEngine(CreateNodeOptions("node" + _nodes.Count, leader.Options.ElectionTimeout, StorageEnvironmentOptions.CreateMemoryOnly())); 309 | _nodes.Add(raftEngine); 310 | return raftEngine; 311 | } 312 | 313 | public virtual void Dispose() 314 | { 315 | _nodes.ForEach(node => node.Dispose()); 316 | } 317 | } 318 | } -------------------------------------------------------------------------------- /Rachis.Tests/SnapshotTests.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics; 2 | using System.Linq; 3 | using System.Threading; 4 | using FizzWare.NBuilder; 5 | using FluentAssertions; 6 | using FluentAssertions.Events; 7 | using Rachis.Commands; 8 | using Rachis.Storage; 9 | using Rachis.Transport; 10 | using Voron; 11 | using Xunit; 12 | using Xunit.Extensions; 13 | 14 | namespace Rachis.Tests 15 | { 16 | public class SnapshotTests : RaftTestsBase 17 | { 18 | 19 | [Fact] 20 | public void CanProperlySnapshot() 21 | { 22 | using (var state = new PersistentState("self", StorageEnvironmentOptions.CreateMemoryOnly(), CancellationToken.None) 23 | { 24 | CommandSerializer = new JsonCommandSerializer() 25 | }) 26 | { 27 | state.UpdateTermTo(null, 1); 28 | state.AppendToLeaderLog(new NopCommand()); 29 | for (int i = 0; i < 5; i++) 30 | { 31 | state.AppendToLeaderLog(new DictionaryCommand.Set 32 | { 33 | Key = i.ToString(), 34 | Value = i 35 | }); 36 | } 37 | 38 | state.MarkSnapshotFor(6, 1, 5); 39 | 40 | state.AppendToLeaderLog(new DictionaryCommand.Set 41 | { 42 | Key = "1", 43 | Value = 4 44 | }); 45 | 46 | var lastLogEntry = state.LastLogEntry(); 47 | 48 | Assert.Equal(7, lastLogEntry.Index); 49 | } 50 | } 51 | 52 | [Theory] 53 | [InlineData(1)] 54 | [InlineData(2)] 55 | [InlineData(3)] 56 | [InlineData(5)] 57 | [InlineData(7)] 58 | public void AfterSnapshotInstalled_CanContinueGettingLogEntriesNormally(int amount) 59 | { 60 | var leader = CreateNetworkAndGetLeader(amount); 61 | leader.Options.MaxLogLengthBeforeCompaction = 5; 62 | var snapshot = WaitForSnapshot(leader); 63 | var commits = WaitForCommitsOnCluster( 64 | machine => machine.Data.Count == 5); 65 | for (int i = 0; i < 5; i++) 66 | { 67 | leader.AppendCommand(new DictionaryCommand.Set 68 | { 69 | Key = i.ToString(), 70 | Value = i 71 | }); 72 | } 73 | Assert.True(snapshot.Wait(3000)); 74 | Assert.True(commits.Wait(3000)); 75 | 76 | Assert.NotNull(leader.StateMachine.GetSnapshotWriter()); 77 | 78 | var newNode = NewNodeFor(leader); 79 | WriteLine("<-- adding node"); 80 | var waitForSnapshotInstallation = WaitForSnapshotInstallation(newNode); 81 | 82 | Assert.True(leader.AddToClusterAsync(new NodeConnectionInfo { Name = newNode.Name }).Wait(3000)); 83 | 84 | Assert.True(waitForSnapshotInstallation.Wait(3000)); 85 | 86 | Assert.Equal(newNode.CurrentLeader, leader.Name); 87 | 88 | var commit = WaitForCommit(newNode, 89 | machine => machine.Data.ContainsKey("c")); 90 | 91 | leader.AppendCommand(new DictionaryCommand.Set 92 | { 93 | Key = "c", 94 | Value = 1 95 | }); 96 | 97 | Assert.True(commit.Wait(3000)); 98 | 99 | var dictionary = ((DictionaryStateMachine)newNode.StateMachine).Data; 100 | for (int i = 0; i < 5; i++) 101 | { 102 | Assert.Equal(i, dictionary[i.ToString()]); 103 | } 104 | Assert.Equal(1, dictionary["c"]); 105 | } 106 | 107 | [Fact] 108 | public void Snapshot_after_enough_command_applies_snapshot_is_applied_only_once() 109 | { 110 | var snapshotCreationEndedEvent = new ManualResetEventSlim(); 111 | const int commandsCount = 5; 112 | var commands = Builder.CreateListOfSize(commandsCount) 113 | .All() 114 | .With(x => x.Completion = null) 115 | .Build() 116 | .ToList(); 117 | var appliedAllCommandsEvent = new CountdownEvent(commandsCount); 118 | 119 | var leader = CreateNetworkAndGetLeader(3); 120 | 121 | leader.MonitorEvents(); 122 | leader.CreatedSnapshot += snapshotCreationEndedEvent.Set; 123 | leader.CommitIndexChanged += (old, @new) => appliedAllCommandsEvent.Signal(); 124 | 125 | leader.Options.MaxLogLengthBeforeCompaction = commandsCount - 3; 126 | commands.ForEach(leader.AppendCommand); 127 | 128 | Assert.True(appliedAllCommandsEvent.Wait(3000)); 129 | Assert.True(snapshotCreationEndedEvent.Wait(3000)); 130 | 131 | //should only raise the event once 132 | leader.ShouldRaise("CreatedSnapshot"); 133 | leader.GetRecorderForEvent("CreatedSnapshot") 134 | .Should().HaveCount(1); 135 | } 136 | 137 | 138 | [Fact] 139 | public void Snaphot_after_enough_command_applies_snapshot_is_created() 140 | { 141 | var snapshotCreationEndedEvent = new ManualResetEventSlim(); 142 | const int commandsCount = 9; 143 | var commands = Builder.CreateListOfSize(commandsCount) 144 | .All() 145 | .With(x => x.Completion = null) 146 | .Build() 147 | .ToList(); 148 | 149 | var leader = CreateNetworkAndGetLeader(3); 150 | var lastLogEntry = leader.PersistentState.LastLogEntry(); 151 | leader.Options.MaxLogLengthBeforeCompaction = commandsCount - 4; 152 | 153 | var appliedAllCommandsEvent = new CountdownEvent(commandsCount); 154 | leader.CreatedSnapshot += snapshotCreationEndedEvent.Set; 155 | 156 | leader.CommitApplied += cmd => 157 | { 158 | if (cmd is DictionaryCommand.Set) 159 | { 160 | appliedAllCommandsEvent.Signal(); 161 | } 162 | }; 163 | 164 | WriteLine("<--- Started appending commands.."); 165 | commands.ForEach(leader.AppendCommand); 166 | WriteLine("<--- Ended appending commands.."); 167 | 168 | var millisecondsTimeout = Debugger.IsAttached ? 600000 : 4000; 169 | Assert.True(snapshotCreationEndedEvent.Wait(millisecondsTimeout)); 170 | Assert.True(appliedAllCommandsEvent.Wait(millisecondsTimeout), "Not all commands were applied, there are still " + appliedAllCommandsEvent.CurrentCount + " commands left"); 171 | 172 | var entriesAfterSnapshotCreation = leader.PersistentState.LogEntriesAfter(0).ToList(); 173 | Assert.Empty(entriesAfterSnapshotCreation.Where(x=>x.Index == lastLogEntry.Index)); 174 | } 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /Rachis.Tests/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /Rachis.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 2012 4 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Rachis", "Rachis\Rachis.csproj", "{F796F69F-D17B-4260-92D6-65CB94C0E05C}" 5 | EndProject 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Rachis.Tests", "Rachis.Tests\Rachis.Tests.csproj", "{61B3D01A-F286-42B1-AA6B-83D9D6DF0873}" 7 | EndProject 8 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tryouts", "Tryouts\Tryouts.csproj", "{54412F82-A711-4AF6-931B-D8AF808E57C5}" 9 | EndProject 10 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{279C1A0A-49E4-4EB2-96DB-AC1E9C20DB0E}" 11 | ProjectSection(SolutionItems) = preProject 12 | .nuget\NuGet.Config = .nuget\NuGet.Config 13 | .nuget\NuGet.exe = .nuget\NuGet.exe 14 | .nuget\NuGet.targets = .nuget\NuGet.targets 15 | EndProjectSection 16 | EndProject 17 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Voron", "..\ravendb\Raven.Voron\Voron\Voron.csproj", "{FF83C7C2-BC7B-4DCC-A782-49EF9BBD9390}" 18 | EndProject 19 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TailFeather", "TailFeather\TailFeather.csproj", "{832A4C02-72AE-4F30-9691-E286457DD39D}" 20 | EndProject 21 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TailFeather.Client", "TailFeather.Client\TailFeather.Client.csproj", "{1E267CF9-EAE8-4F87-BE37-BB1E7C9E86D6}" 22 | EndProject 23 | Global 24 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 25 | Debug|Any CPU = Debug|Any CPU 26 | Release|Any CPU = Release|Any CPU 27 | EndGlobalSection 28 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 29 | {1E267CF9-EAE8-4F87-BE37-BB1E7C9E86D6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 30 | {1E267CF9-EAE8-4F87-BE37-BB1E7C9E86D6}.Debug|Any CPU.Build.0 = Debug|Any CPU 31 | {1E267CF9-EAE8-4F87-BE37-BB1E7C9E86D6}.Release|Any CPU.ActiveCfg = Release|Any CPU 32 | {1E267CF9-EAE8-4F87-BE37-BB1E7C9E86D6}.Release|Any CPU.Build.0 = Release|Any CPU 33 | {54412F82-A711-4AF6-931B-D8AF808E57C5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 34 | {54412F82-A711-4AF6-931B-D8AF808E57C5}.Debug|Any CPU.Build.0 = Debug|Any CPU 35 | {54412F82-A711-4AF6-931B-D8AF808E57C5}.Release|Any CPU.ActiveCfg = Release|Any CPU 36 | {54412F82-A711-4AF6-931B-D8AF808E57C5}.Release|Any CPU.Build.0 = Release|Any CPU 37 | {61B3D01A-F286-42B1-AA6B-83D9D6DF0873}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 38 | {61B3D01A-F286-42B1-AA6B-83D9D6DF0873}.Debug|Any CPU.Build.0 = Debug|Any CPU 39 | {61B3D01A-F286-42B1-AA6B-83D9D6DF0873}.Release|Any CPU.ActiveCfg = Release|Any CPU 40 | {61B3D01A-F286-42B1-AA6B-83D9D6DF0873}.Release|Any CPU.Build.0 = Release|Any CPU 41 | {832A4C02-72AE-4F30-9691-E286457DD39D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 42 | {832A4C02-72AE-4F30-9691-E286457DD39D}.Debug|Any CPU.Build.0 = Debug|Any CPU 43 | {832A4C02-72AE-4F30-9691-E286457DD39D}.Release|Any CPU.ActiveCfg = Release|Any CPU 44 | {832A4C02-72AE-4F30-9691-E286457DD39D}.Release|Any CPU.Build.0 = Release|Any CPU 45 | {F796F69F-D17B-4260-92D6-65CB94C0E05C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 46 | {F796F69F-D17B-4260-92D6-65CB94C0E05C}.Debug|Any CPU.Build.0 = Debug|Any CPU 47 | {F796F69F-D17B-4260-92D6-65CB94C0E05C}.Release|Any CPU.ActiveCfg = Release|Any CPU 48 | {F796F69F-D17B-4260-92D6-65CB94C0E05C}.Release|Any CPU.Build.0 = Release|Any CPU 49 | {FF83C7C2-BC7B-4DCC-A782-49EF9BBD9390}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 50 | {FF83C7C2-BC7B-4DCC-A782-49EF9BBD9390}.Debug|Any CPU.Build.0 = Debug|Any CPU 51 | {FF83C7C2-BC7B-4DCC-A782-49EF9BBD9390}.Release|Any CPU.ActiveCfg = Release|Any CPU 52 | {FF83C7C2-BC7B-4DCC-A782-49EF9BBD9390}.Release|Any CPU.Build.0 = Release|Any CPU 53 | EndGlobalSection 54 | GlobalSection(NestedProjects) = preSolution 55 | EndGlobalSection 56 | GlobalSection(SolutionProperties) = preSolution 57 | HideSolutionNode = FALSE 58 | EndGlobalSection 59 | EndGlobal 60 | -------------------------------------------------------------------------------- /Rachis.userprefs: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /Rachis/Behaviors/CandidateStateBehavior.cs: -------------------------------------------------------------------------------- 1 | // ----------------------------------------------------------------------- 2 | // 3 | // Copyright (c) Hibernating Rhinos LTD. All rights reserved. 4 | // 5 | // ----------------------------------------------------------------------- 6 | 7 | using System; 8 | using System.Collections.Generic; 9 | using System.Linq; 10 | using Rachis.Messages; 11 | 12 | namespace Rachis.Behaviors 13 | { 14 | public class CandidateStateBehavior : AbstractRaftStateBehavior 15 | { 16 | private readonly bool _forcedElection; 17 | private readonly HashSet _votesForMyLeadership = new HashSet(); 18 | private readonly Random _random; 19 | private bool _wonTrialElection; 20 | private bool _termIncreaseMightGetMyVote; 21 | 22 | public CandidateStateBehavior(RaftEngine engine, bool forcedElection) 23 | : base(engine) 24 | { 25 | _forcedElection = forcedElection; 26 | _wonTrialElection = forcedElection; 27 | _random = new Random((int)(engine.Name.GetHashCode() + DateTime.UtcNow.Ticks)); 28 | Timeout = _random.Next(engine.Options.ElectionTimeout / 2, engine.Options.ElectionTimeout); 29 | if (forcedElection) 30 | StartElection(); 31 | } 32 | 33 | public override void HandleTimeout() 34 | { 35 | _log.Info("Timeout ({1:#,#;;0} ms) for elections in term {0}", Engine.PersistentState.CurrentTerm, 36 | Timeout); 37 | 38 | Timeout = _random.Next(Engine.Options.ElectionTimeout / 2, Engine.Options.ElectionTimeout); 39 | _wonTrialElection = false; 40 | StartElection(); 41 | } 42 | 43 | private void StartElection() 44 | { 45 | LastHeartbeatTime = DateTime.UtcNow; 46 | _votesForMyLeadership.Clear(); 47 | 48 | long currentTerm = Engine.PersistentState.CurrentTerm + 1; 49 | if (_wonTrialElection || _termIncreaseMightGetMyVote) // only in the real election (or if we have to), we increment the current term 50 | { 51 | Engine.PersistentState.UpdateTermTo(Engine, currentTerm); 52 | } 53 | if (_wonTrialElection)// and only if we won an election do we record a firm vote for ourselves 54 | Engine.PersistentState.RecordVoteFor(Engine.Name, currentTerm); 55 | 56 | _termIncreaseMightGetMyVote = false; 57 | 58 | Engine.CurrentLeader = null; 59 | _log.Info("Calling for {0} election in term {1}", 60 | _wonTrialElection ? "an" : "a trial", currentTerm); 61 | 62 | var lastLogEntry = Engine.PersistentState.LastLogEntry(); 63 | var rvr = new RequestVoteRequest 64 | { 65 | LastLogIndex = lastLogEntry.Index, 66 | LastLogTerm = lastLogEntry.Term, 67 | Term = currentTerm, 68 | From = Engine.Name, 69 | ClusterTopologyId = Engine.CurrentTopology.TopologyId, 70 | TrialOnly = _wonTrialElection == false, 71 | ForcedElection = _forcedElection 72 | }; 73 | 74 | var allVotingNodes = Engine.CurrentTopology.AllVotingNodes; 75 | 76 | // don't send to yourself the message 77 | foreach (var votingPeer in allVotingNodes) 78 | { 79 | if (votingPeer.Name == Engine.Name) 80 | continue; 81 | Engine.Transport.Send(votingPeer, rvr); 82 | } 83 | 84 | Engine.OnCandidacyAnnounced(); 85 | _log.Info("Voting for myself in {1}election for term {0}", currentTerm, _wonTrialElection ? " " : " trial "); 86 | Handle(new RequestVoteResponse 87 | { 88 | CurrentTerm = Engine.PersistentState.CurrentTerm, 89 | VoteGranted = true, 90 | Message = String.Format("{0} -> Voting for myself", Engine.Name), 91 | From = Engine.Name, 92 | ClusterTopologyId = Engine.CurrentTopology.TopologyId, 93 | VoteTerm = currentTerm, 94 | TrialOnly = _wonTrialElection == false 95 | }); 96 | } 97 | 98 | public override RaftEngineState State 99 | { 100 | get { return RaftEngineState.Candidate; } 101 | } 102 | 103 | public override void Handle(RequestVoteResponse resp) 104 | { 105 | if (FromOurTopology(resp) == false) 106 | { 107 | _log.Info("Got a request vote response message outside my cluster topology (id: {0}), ignoring", resp.ClusterTopologyId); 108 | return; 109 | } 110 | 111 | long currentTerm = _wonTrialElection ? Engine.PersistentState.CurrentTerm : Engine.PersistentState.CurrentTerm + 1; 112 | if (resp.VoteTerm != currentTerm) 113 | { 114 | _log.Info("Got a vote for {2}election term {0} but current term is {1}, ignoring", resp.VoteTerm, currentTerm, 115 | _wonTrialElection ? " " : "trial "); 116 | return; 117 | } 118 | if (resp.CurrentTerm > currentTerm) 119 | { 120 | _log.Info("CandidateStateBehavior -> UpdateCurrentTerm called, there is a new leader, moving to follower state"); 121 | Engine.UpdateCurrentTerm(resp.CurrentTerm, null); 122 | return; 123 | } 124 | 125 | if (resp.VoteGranted == false) 126 | { 127 | if (resp.TermIncreaseMightGetMyVote) 128 | _termIncreaseMightGetMyVote = true; 129 | _log.Info("Vote rejected from {0} trial: {1}", resp.From, resp.TrialOnly); 130 | return; 131 | } 132 | 133 | if (Engine.CurrentTopology.IsVoter(resp.From) == false) //precaution 134 | { 135 | _log.Info("Vote accepted from {0}, which isn't a voting node in our cluster", resp.From); 136 | return; 137 | } 138 | 139 | if (resp.TrialOnly && _wonTrialElection) // note that we can't get a vote for real election when we get a trail, because the terms would be different 140 | { 141 | _log.Info("Got a vote for trial only from {0} but we already won the trial election for this round, ignoring", resp.From); 142 | return; 143 | } 144 | 145 | _votesForMyLeadership.Add(resp.From); 146 | _log.Info("Adding to my votes: {0} (current votes: {1})", resp.From, string.Join(", ", _votesForMyLeadership)); 147 | 148 | if (Engine.CurrentTopology.HasQuorum(_votesForMyLeadership) == false) 149 | { 150 | _log.Info("Not enough votes for leadership, votes = {0}", _votesForMyLeadership.Any() ? string.Join(", ", _votesForMyLeadership) : "empty"); 151 | return; 152 | } 153 | 154 | if (_wonTrialElection == false) 155 | { 156 | _wonTrialElection = true; 157 | _log.Info("Won trial election with {0} votes from {1}, now running for real", _votesForMyLeadership.Count, string.Join(", ", _votesForMyLeadership)); 158 | StartElection(); 159 | return; 160 | } 161 | 162 | Engine.SetState(RaftEngineState.Leader); 163 | _log.Info("Selected as leader, term = {0}", resp.CurrentTerm); 164 | } 165 | 166 | } 167 | } -------------------------------------------------------------------------------- /Rachis/Behaviors/FollowerStateBehavior.cs: -------------------------------------------------------------------------------- 1 | // ----------------------------------------------------------------------- 2 | // 3 | // Copyright (c) Hibernating Rhinos LTD. All rights reserved. 4 | // 5 | // ----------------------------------------------------------------------- 6 | 7 | using System; 8 | 9 | namespace Rachis.Behaviors 10 | { 11 | public class FollowerStateBehavior : AbstractRaftStateBehavior 12 | { 13 | private bool _avoidLeadership; 14 | private readonly long _currentTermWhenWeBecameFollowers; 15 | 16 | public FollowerStateBehavior(RaftEngine engine, bool avoidLeadership) : base(engine) 17 | { 18 | _avoidLeadership = avoidLeadership; 19 | _currentTermWhenWeBecameFollowers = engine.PersistentState.CurrentTerm + 1;// we are going to have a new term immediately. 20 | var random = new Random(Engine.Name.GetHashCode() ^ (int)DateTime.Now.Ticks); 21 | Timeout = random.Next(engine.Options.ElectionTimeout / 2, engine.Options.ElectionTimeout); 22 | } 23 | 24 | public override RaftEngineState State 25 | { 26 | get { return RaftEngineState.Follower; } 27 | } 28 | 29 | public override void HandleTimeout() 30 | { 31 | LastHeartbeatTime = DateTime.UtcNow; 32 | 33 | if (Engine.CurrentTopology.IsVoter(Engine.Name) == false) 34 | { 35 | _log.Info("Not a leader material, can't become a candidate. (This will change the first time we'll get a append entries request)."); 36 | return; 37 | } 38 | 39 | if (_avoidLeadership && _currentTermWhenWeBecameFollowers >= Engine.PersistentState.CurrentTerm) 40 | { 41 | _log.Info("Got timeout in follower mode in term {0}, but we are in avoid leadership mode following a step down, so we'll let this one slide. Next time, I'm going to be the leader again!", 42 | Engine.PersistentState.CurrentTerm); 43 | _avoidLeadership = false; 44 | return; 45 | } 46 | _log.Info("Got timeout in follower mode in term {0}", Engine.PersistentState.CurrentTerm); 47 | Engine.SetState(RaftEngineState.Candidate); 48 | } 49 | } 50 | } -------------------------------------------------------------------------------- /Rachis/Behaviors/SnapshotInstallationStateBehavior.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Threading.Tasks; 4 | using Rachis.Commands; 5 | using Rachis.Messages; 6 | 7 | namespace Rachis.Behaviors 8 | { 9 | public class SnapshotInstallationStateBehavior : AbstractRaftStateBehavior 10 | { 11 | private readonly Random _random; 12 | 13 | private Task _installingSnapshot; 14 | 15 | public SnapshotInstallationStateBehavior(RaftEngine engine) : base(engine) 16 | { 17 | _random = new Random((int)(engine.Name.GetHashCode() + DateTime.UtcNow.Ticks)); 18 | Timeout = _random.Next(engine.Options.ElectionTimeout / 2, engine.Options.ElectionTimeout); 19 | } 20 | 21 | public override RaftEngineState State 22 | { 23 | get { return RaftEngineState.SnapshotInstallation; } 24 | } 25 | 26 | public override CanInstallSnapshotResponse Handle(CanInstallSnapshotRequest req) 27 | { 28 | if (_installingSnapshot == null) 29 | { 30 | return base.Handle(req); 31 | } 32 | return new CanInstallSnapshotResponse 33 | { 34 | From = Engine.Name, 35 | ClusterTopologyId = Engine.CurrentTopology.TopologyId, 36 | IsCurrentlyInstalling = true, 37 | Message = "The node is in the process of installing a snapshot", 38 | Success = false 39 | }; 40 | } 41 | 42 | public override InstallSnapshotResponse Handle(MessageContext context, InstallSnapshotRequest req, Stream stream) 43 | { 44 | if (_installingSnapshot != null) 45 | { 46 | return new InstallSnapshotResponse 47 | { 48 | Success = false, 49 | Message = "Cannot install snapshot because we are already installing a snapshot", 50 | CurrentTerm = Engine.PersistentState.CurrentTerm, 51 | From = Engine.Name, 52 | ClusterTopologyId = Engine.CurrentTopology.TopologyId, 53 | LastLogIndex = Engine.PersistentState.LastLogEntry().Index 54 | }; 55 | } 56 | 57 | 58 | if (FromOurTopology(req) == false) 59 | { 60 | _log.Info("Got an install snapshot message outside my cluster topology (id: {0}), ignoring", req.ClusterTopologyId); 61 | 62 | return new InstallSnapshotResponse 63 | { 64 | Success = false, 65 | Message = "Cannot install snapshot because the cluster topology id doesn't match, mine is: " + Engine.CurrentTopology.TopologyId, 66 | CurrentTerm = Engine.PersistentState.CurrentTerm, 67 | From = Engine.Name, 68 | ClusterTopologyId = Engine.CurrentTopology.TopologyId, 69 | LastLogIndex = Engine.PersistentState.LastLogEntry().Index 70 | }; 71 | } 72 | 73 | var lastLogEntry = Engine.PersistentState.LastLogEntry(); 74 | if (req.Term < lastLogEntry.Term || req.LastIncludedIndex < lastLogEntry.Index) 75 | { 76 | stream.Dispose(); 77 | 78 | return new InstallSnapshotResponse 79 | { 80 | From = Engine.Name, 81 | ClusterTopologyId = Engine.CurrentTopology.TopologyId, 82 | CurrentTerm = lastLogEntry.Term, 83 | LastLogIndex = lastLogEntry.Index, 84 | Message = string.Format("Snapshot is too old (term {0} index {1}) while we have (term {2} index {3})", 85 | req.Term, req.LastIncludedIndex, lastLogEntry.Term, lastLogEntry.Index), 86 | Success = false 87 | }; 88 | } 89 | 90 | _log.Info("Received InstallSnapshotRequest from {0} until term {1} / {2}", req.From, req.LastIncludedTerm, req.LastIncludedIndex); 91 | 92 | Engine.OnSnapshotInstallationStarted(); 93 | 94 | // this can be a long running task 95 | _installingSnapshot = Task.Run(() => 96 | { 97 | try 98 | { 99 | Engine.StateMachine.ApplySnapshot(req.LastIncludedTerm, req.LastIncludedIndex, stream); 100 | Engine.PersistentState.MarkSnapshotFor(req.LastIncludedIndex, req.LastIncludedTerm, int.MaxValue); 101 | Engine.PersistentState.SetCurrentTopology(req.Topology, req.LastIncludedIndex); 102 | var tcc = new TopologyChangeCommand { Requested = req.Topology }; 103 | Engine.StartTopologyChange(tcc); 104 | Engine.CommitTopologyChange(tcc); 105 | } 106 | catch (Exception e) 107 | { 108 | _log.Warn(string.Format("Failed to install snapshot term {0} index {1}", req.LastIncludedIndex, req.LastIncludedIndex), e); 109 | context.ExecuteInEventLoop(() => 110 | { 111 | _installingSnapshot = null; 112 | }); 113 | } 114 | 115 | // we are doing it this way to ensure that we are single threaded 116 | context.ExecuteInEventLoop(() => 117 | { 118 | Engine.UpdateCurrentTerm(req.Term, req.From); // implicitly put us in follower state 119 | _log.Info("Updating the commit index to the snapshot last included index of {0}", req.LastIncludedIndex); 120 | Engine.OnSnapshotInstallationEnded(req.Term); 121 | 122 | context.Reply(new InstallSnapshotResponse 123 | { 124 | From = Engine.Name, 125 | ClusterTopologyId = Engine.CurrentTopology.TopologyId, 126 | CurrentTerm = req.Term, 127 | LastLogIndex = req.LastIncludedIndex, 128 | Success = true 129 | }); 130 | }); 131 | }); 132 | 133 | return null; 134 | } 135 | 136 | public override AppendEntriesResponse Handle(AppendEntriesRequest req) 137 | { 138 | if (_installingSnapshot == null) 139 | { 140 | return base.Handle(req); 141 | } 142 | 143 | var lastLogEntry = Engine.PersistentState.LastLogEntry(); 144 | return new AppendEntriesResponse 145 | { 146 | From = Engine.Name, 147 | ClusterTopologyId = Engine.CurrentTopology.TopologyId, 148 | CurrentTerm = Engine.PersistentState.CurrentTerm, 149 | LastLogIndex = lastLogEntry.Index, 150 | LeaderId = Engine.CurrentLeader, 151 | Message = "I am in the process of receiving a snapshot, so I cannot accept new entries at the moment", 152 | Success = false 153 | }; 154 | } 155 | 156 | 157 | public override void HandleTimeout() 158 | { 159 | Timeout = _random.Next(Engine.Options.ElectionTimeout / 2, Engine.Options.ElectionTimeout); 160 | LastHeartbeatTime = DateTime.UtcNow;// avoid busy loop while waiting for the snapshot 161 | _log.Info("Received timeout during installation of a snapshot. Doing nothing, since the node should finish receiving snapshot before it could change into candidate"); 162 | //do nothing during timeout --> this behavior will go on until the snapshot installation is finished 163 | } 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /Rachis/Behaviors/SteppingDownStateBehavior.cs: -------------------------------------------------------------------------------- 1 | // ----------------------------------------------------------------------- 2 | // 3 | // Copyright (c) Hibernating Rhinos LTD. All rights reserved. 4 | // 5 | // ----------------------------------------------------------------------- 6 | 7 | using System.Diagnostics; 8 | using System.Linq; 9 | using Rachis.Messages; 10 | 11 | namespace Rachis.Behaviors 12 | { 13 | public class SteppingDownStateBehavior : LeaderStateBehavior 14 | { 15 | private readonly Stopwatch _stepdownDuration; 16 | 17 | public SteppingDownStateBehavior(RaftEngine engine) 18 | : base(engine) 19 | { 20 | _stepdownDuration = Stopwatch.StartNew(); 21 | // we are sending this to ourselves because we want to make 22 | // sure that we immediately check if we can step down 23 | Engine.Transport.SendToSelf(new AppendEntriesResponse 24 | { 25 | CurrentTerm = Engine.PersistentState.CurrentTerm, 26 | From = Engine.Name, 27 | ClusterTopologyId = Engine.CurrentTopology.TopologyId, 28 | LastLogIndex = Engine.PersistentState.LastLogEntry().Index, 29 | LeaderId = Engine.Name, 30 | Message = "Forcing step down evaluation", 31 | Success = true 32 | }); 33 | } 34 | 35 | public override RaftEngineState State 36 | { 37 | get { return RaftEngineState.SteppingDown; } 38 | } 39 | 40 | public override void Handle(AppendEntriesResponse resp) 41 | { 42 | base.Handle(resp); 43 | 44 | var maxIndexOnQuorom = GetMaxIndexOnQuorum(); 45 | 46 | var lastLogEntry = Engine.PersistentState.LastLogEntry(); 47 | 48 | if (maxIndexOnQuorom >= lastLogEntry.Index) 49 | { 50 | _log.Info("Done sending all events to the cluster, can step down gracefully now"); 51 | TransferToBestMatch(); 52 | } 53 | } 54 | 55 | private void TransferToBestMatch() 56 | { 57 | var bestMatch = _matchIndexes.OrderByDescending(x => x.Value) 58 | .Select(x => x.Key).FirstOrDefault(x => x != Engine.Name); 59 | 60 | if (bestMatch != null) // this should always be the case, but... 61 | { 62 | var nodeConnectionInfo = Engine.CurrentTopology.GetNodeByName(bestMatch); 63 | if (nodeConnectionInfo != null) 64 | { 65 | Engine.Transport.Send(nodeConnectionInfo, new TimeoutNowRequest 66 | { 67 | Term = Engine.PersistentState.CurrentTerm, 68 | From = Engine.Name, 69 | ClusterTopologyId = Engine.CurrentTopology.TopologyId, 70 | }); 71 | _log.Info("Transferring cluster leadership to {0}", bestMatch); 72 | } 73 | } 74 | 75 | Engine.SetState(RaftEngineState.FollowerAfterStepDown); 76 | } 77 | 78 | public override void HandleTimeout() 79 | { 80 | base.HandleTimeout(); 81 | if (_stepdownDuration.Elapsed > Engine.Options.MaxStepDownDrainTime) 82 | { 83 | _log.Info("Step down has aborted after {0} because this is greater than the max step down time", _stepdownDuration.Elapsed); 84 | TransferToBestMatch(); 85 | } 86 | } 87 | 88 | public override void Dispose() 89 | { 90 | base.Dispose(); 91 | Engine.FinishSteppingDown(); 92 | } 93 | } 94 | } -------------------------------------------------------------------------------- /Rachis/Commands/Command.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using Newtonsoft.Json; 3 | 4 | namespace Rachis.Commands 5 | { 6 | public abstract class Command 7 | { 8 | public long AssignedIndex { get; set; } 9 | 10 | [JsonIgnore] 11 | public TaskCompletionSource Completion { get; set; } 12 | 13 | [JsonIgnore] 14 | public object CommandResult { get; set; } 15 | 16 | public bool BufferCommand { get; set; } 17 | 18 | public void Complete() 19 | { 20 | if (Completion == null) 21 | return; 22 | Completion.SetResult(CommandResult); 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /Rachis/Commands/NopCommand.cs: -------------------------------------------------------------------------------- 1 | namespace Rachis.Commands 2 | { 3 | public class NopCommand : Command 4 | { 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /Rachis/Commands/TopologyChangeCommand.cs: -------------------------------------------------------------------------------- 1 | using Rachis.Storage; 2 | 3 | namespace Rachis.Commands 4 | { 5 | public class TopologyChangeCommand : Command 6 | { 7 | public Topology Requested { get; set; } 8 | public Topology Previous { get; set; } 9 | } 10 | } -------------------------------------------------------------------------------- /Rachis/Interfaces/ICommandSerializer.cs: -------------------------------------------------------------------------------- 1 | using Rachis.Commands; 2 | 3 | namespace Rachis.Interfaces 4 | { 5 | public interface ICommandSerializer 6 | { 7 | byte[] Serialize(Command cmd); 8 | Command Deserialize(byte[] cmd); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /Rachis/Interfaces/IRaftStateMachine.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Threading; 4 | using Rachis.Commands; 5 | using Rachis.Messages; 6 | 7 | namespace Rachis.Interfaces 8 | { 9 | public interface IRaftStateMachine : IDisposable 10 | { 11 | /// 12 | /// This is a thread safe operation, since this is being used by both the leader's message processing thread 13 | /// and the leader's heartbeat thread 14 | /// 15 | long LastAppliedIndex { get; } 16 | 17 | void Apply(LogEntry entry, Command cmd); 18 | 19 | bool SupportSnapshots { get; } 20 | 21 | /// 22 | /// Create a snapshot, can be called concurrently with GetSnapshotWriter, can also be called concurrently 23 | /// with calls to Apply. 24 | /// 25 | void CreateSnapshot(long index, long term, ManualResetEventSlim allowFurtherModifications); 26 | 27 | /// 28 | /// Can be called concurrently with CreateSnapshot 29 | /// Should be cheap unless WriteSnapshot is called 30 | /// 31 | ISnapshotWriter GetSnapshotWriter(); 32 | 33 | /// 34 | /// Nothing else may access the state machine when this is running, this is guranteed by Raft. 35 | /// 36 | void ApplySnapshot(long term, long index, Stream stream); 37 | } 38 | } -------------------------------------------------------------------------------- /Rachis/Interfaces/ISnapshotWriter.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | 4 | namespace Rachis.Interfaces 5 | { 6 | public interface ISnapshotWriter : IDisposable 7 | { 8 | long Index { get; } 9 | long Term { get; } 10 | void WriteSnapshot(Stream stream); 11 | } 12 | } -------------------------------------------------------------------------------- /Rachis/Interfaces/ITransport.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Threading; 4 | using Rachis.Messages; 5 | using Rachis.Transport; 6 | 7 | namespace Rachis.Interfaces 8 | { 9 | 10 | /// 11 | /// abstraction for transport between Raft nodes. 12 | /// 13 | public interface ITransport 14 | { 15 | bool TryReceiveMessage(int timeout, CancellationToken cancellationToken, out MessageContext messageContext); 16 | 17 | void Stream(NodeConnectionInfo dest, InstallSnapshotRequest snapshotRequest, Action streamWriter); 18 | 19 | void Send(NodeConnectionInfo dest, CanInstallSnapshotRequest req); 20 | void Send(NodeConnectionInfo dest, TimeoutNowRequest req); 21 | void Send(NodeConnectionInfo dest, DisconnectedFromCluster req); 22 | void Send(NodeConnectionInfo dest, AppendEntriesRequest req); 23 | void Send(NodeConnectionInfo dest, RequestVoteRequest req); 24 | 25 | void SendToSelf(AppendEntriesResponse resp); 26 | } 27 | } -------------------------------------------------------------------------------- /Rachis/JsonCommandSerializer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using Newtonsoft.Json; 4 | using Rachis.Commands; 5 | using Rachis.Interfaces; 6 | 7 | namespace Rachis 8 | { 9 | public class JsonCommandSerializer : ICommandSerializer 10 | { 11 | private readonly JsonSerializer _serializer; 12 | 13 | public JsonCommandSerializer() 14 | { 15 | _serializer = new JsonSerializer 16 | { 17 | TypeNameHandling = TypeNameHandling.Objects 18 | }; 19 | } 20 | 21 | public byte[] Serialize(Command cmd) 22 | { 23 | using (var memoryStream = new MemoryStream()) 24 | { 25 | using (var streamWriter = new StreamWriter(memoryStream)) 26 | { 27 | _serializer.Serialize(streamWriter, cmd); 28 | streamWriter.Flush(); 29 | } 30 | return memoryStream.ToArray(); 31 | } 32 | } 33 | 34 | public Command Deserialize(byte[] cmd) 35 | { 36 | lock(this) 37 | using (var memoryStream = new MemoryStream(cmd)) 38 | using (var streamReader = new StreamReader(memoryStream)) 39 | { 40 | Command command; 41 | try 42 | { 43 | var jsonTextReader = new JsonTextReader(streamReader); 44 | var deserialized = _serializer.Deserialize(jsonTextReader); 45 | command = deserialized as Command; 46 | } 47 | catch (Exception e) 48 | { 49 | throw new InvalidOperationException("Failed to deserialize command. This is not supposed to happen and it is probably a bug",e); 50 | } 51 | 52 | if(command == null) 53 | throw new ArgumentException("JsonCommandSerializer should only receive Command implementations to deserialize","cmd"); 54 | 55 | return command; 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /Rachis/MessageAbsolutePath.cs: -------------------------------------------------------------------------------- 1 | namespace Rachis 2 | { 3 | public static class MessageAbsolutePath 4 | { 5 | public const string AppendEntriesRequest = "appendentriesrequest"; 6 | public const string AppendEntriesResponse = "appendentriesresponse"; 7 | public const string CanInstallSnapshotRequest = "caninstallsnapshotrequest"; 8 | public const string CanInstallSnapshotResponse = "caninstallsnapshotresponse"; 9 | public const string InstallSnapshotRequest = "installsnapshotrequest"; 10 | public const string RequestVoteRequest = "requestvoterequest"; 11 | public const string RequestVoteResponse = "requestvoteresponse"; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /Rachis/Messages/AppendEntriesRequest.cs: -------------------------------------------------------------------------------- 1 | using Newtonsoft.Json; 2 | 3 | namespace Rachis.Messages 4 | { 5 | public class AppendEntriesRequest : BaseMessage 6 | { 7 | public long Term { get; set; } 8 | public long PrevLogIndex { get; set; } 9 | public long PrevLogTerm { get; set; } 10 | [JsonIgnore] 11 | public LogEntry[] Entries { get; set; } 12 | public int EntriesCount { get { return Entries == null ? 0 : Entries.Length; } } 13 | public long LeaderCommit { get; set; } 14 | } 15 | } -------------------------------------------------------------------------------- /Rachis/Messages/AppendEntriesResponse.cs: -------------------------------------------------------------------------------- 1 | namespace Rachis.Messages 2 | { 3 | public class AppendEntriesResponse : BaseMessage 4 | { 5 | public long CurrentTerm { get; set; } 6 | 7 | public long LastLogIndex { get; set; } 8 | 9 | public bool Success { get; set; } 10 | 11 | public string Message { get; set; } 12 | public string LeaderId { get; set; } 13 | } 14 | } -------------------------------------------------------------------------------- /Rachis/Messages/BaseMessage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Rachis.Messages 4 | { 5 | public abstract class BaseMessage 6 | { 7 | public string From { get; set; } 8 | public Guid ClusterTopologyId { get; set; } 9 | } 10 | 11 | public class DisconnectedFromCluster : BaseMessage 12 | { 13 | public long Term { get; set; } 14 | } 15 | 16 | public class NothingToDo { } 17 | } -------------------------------------------------------------------------------- /Rachis/Messages/CanInstallSnapshotRequest.cs: -------------------------------------------------------------------------------- 1 | namespace Rachis.Messages 2 | { 3 | public class CanInstallSnapshotRequest : BaseMessage 4 | { 5 | public long Index { get; set; } 6 | 7 | public long Term { get; set; } 8 | } 9 | } -------------------------------------------------------------------------------- /Rachis/Messages/CanInstallSnapshotResponse.cs: -------------------------------------------------------------------------------- 1 | namespace Rachis.Messages 2 | { 3 | public class CanInstallSnapshotResponse : BaseMessage 4 | { 5 | public bool IsCurrentlyInstalling { get; set; } 6 | public long Term { get; set; } 7 | public long Index { get; set; } 8 | public bool Success { get; set; } 9 | public string Message { get; set; } 10 | } 11 | } -------------------------------------------------------------------------------- /Rachis/Messages/InstallSnapshotRequest.cs: -------------------------------------------------------------------------------- 1 | using Rachis.Storage; 2 | 3 | namespace Rachis.Messages 4 | { 5 | public class InstallSnapshotRequest : BaseMessage 6 | { 7 | public long Term { get; set; } 8 | 9 | public long LastIncludedIndex { get; set; } 10 | 11 | public long LastIncludedTerm { get; set; } 12 | 13 | public Topology Topology { get; set; } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Rachis/Messages/InstallSnapshotResponse.cs: -------------------------------------------------------------------------------- 1 | namespace Rachis.Messages 2 | { 3 | public class InstallSnapshotResponse : BaseMessage 4 | { 5 | public string Message { get; set; } 6 | public bool Success { get; set; } 7 | public long CurrentTerm { get; set; } 8 | public long LastLogIndex { get; set; } 9 | 10 | } 11 | } -------------------------------------------------------------------------------- /Rachis/Messages/LogEntry.cs: -------------------------------------------------------------------------------- 1 | namespace Rachis.Messages 2 | { 3 | public class LogEntry 4 | { 5 | public long Index { get; set; } 6 | public long Term { get; set; } 7 | public bool? IsTopologyChange { get; set; } 8 | public byte[] Data { get; set; } 9 | } 10 | } -------------------------------------------------------------------------------- /Rachis/Messages/MessageContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | 4 | namespace Rachis.Messages 5 | { 6 | public abstract class MessageContext 7 | { 8 | public object Message { get; set; } 9 | public Stream Stream { get; set; } 10 | public bool AsyncResponse { get; set; } 11 | 12 | public abstract void Reply(CanInstallSnapshotResponse resp); 13 | public abstract void Reply(InstallSnapshotResponse resp); 14 | public abstract void Reply(AppendEntriesResponse resp); 15 | public abstract void Reply(RequestVoteResponse resp); 16 | 17 | public abstract void ExecuteInEventLoop(Action action); 18 | 19 | public abstract void Done(); 20 | public abstract void Error(Exception exception); 21 | } 22 | } -------------------------------------------------------------------------------- /Rachis/Messages/RequestVoteRequest.cs: -------------------------------------------------------------------------------- 1 | namespace Rachis.Messages 2 | { 3 | public class RequestVoteRequest : BaseMessage 4 | { 5 | public long Term { get; set; } 6 | public long LastLogIndex { get; set; } 7 | public long LastLogTerm { get; set; } 8 | 9 | public bool TrialOnly { get; set; } 10 | public bool ForcedElection { get; set; } 11 | } 12 | } -------------------------------------------------------------------------------- /Rachis/Messages/RequestVoteResponse.cs: -------------------------------------------------------------------------------- 1 | namespace Rachis.Messages 2 | { 3 | public class RequestVoteResponse : BaseMessage 4 | { 5 | public long CurrentTerm { get; set; } 6 | public long VoteTerm { get; set; } 7 | public bool VoteGranted { get; set; } 8 | public string Message { get; set; } 9 | public bool TrialOnly { get; set; } 10 | public bool TermIncreaseMightGetMyVote { get; set; } 11 | } 12 | } -------------------------------------------------------------------------------- /Rachis/Messages/TimeoutNowRequest.cs: -------------------------------------------------------------------------------- 1 | namespace Rachis.Messages 2 | { 3 | public class TimeoutNowRequest : BaseMessage 4 | { 5 | public long Term { get; set; } 6 | } 7 | } -------------------------------------------------------------------------------- /Rachis/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("Rhino.Raft")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("Rhino.Raft")] 13 | [assembly: AssemblyCopyright("Copyright © 2014")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("5376219e-e1b3-4ee4-80b7-c24c9d442d34")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /Rachis/Rachis.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {F796F69F-D17B-4260-92D6-65CB94C0E05C} 8 | Library 9 | Properties 10 | Rachis 11 | Rachis 12 | 512 13 | 14 | ..\ 15 | true 16 | v4.5 17 | 18 | 19 | true 20 | full 21 | false 22 | bin\Debug\ 23 | DEBUG;TRACE 24 | prompt 25 | 4 26 | 27 | 28 | pdbonly 29 | true 30 | bin\Release\ 31 | TRACE 32 | prompt 33 | 4 34 | 35 | 36 | 37 | ..\packages\AttributeRouting.Core.3.5.6\lib\net40\AttributeRouting.dll 38 | 39 | 40 | False 41 | ..\packages\Newtonsoft.Json.6.0.4\lib\net45\Newtonsoft.Json.dll 42 | 43 | 44 | ..\packages\NLog.3.1.0.0\lib\net45\NLog.dll 45 | 46 | 47 | 48 | 49 | 50 | False 51 | ..\packages\Microsoft.AspNet.WebApi.Client.5.2.2\lib\net45\System.Net.Http.Formatting.dll 52 | 53 | 54 | False 55 | ..\packages\Microsoft.AspNet.WebApi.Core.5.2.2\lib\net45\System.Web.Http.dll 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | {FF83C7C2-BC7B-4DCC-A782-49EF9BBD9390} 112 | Voron 113 | 114 | 115 | 116 | 117 | 118 | 119 | This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. 120 | 121 | 122 | 123 | 130 | -------------------------------------------------------------------------------- /Rachis/RaftEngineOptions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | using Rachis.Interfaces; 4 | using Rachis.Transport; 5 | using Voron; 6 | 7 | namespace Rachis 8 | { 9 | public class RaftEngineOptions 10 | { 11 | public RaftEngineOptions(NodeConnectionInfo connection, StorageEnvironmentOptions storageOptions, ITransport transport, IRaftStateMachine stateMachine) 12 | { 13 | if (connection == null) throw new ArgumentNullException("connection"); 14 | if (String.IsNullOrWhiteSpace(connection.Name)) throw new ArgumentNullException("connection.Name"); 15 | if (storageOptions == null) throw new ArgumentNullException("storageOptions"); 16 | if (transport == null) throw new ArgumentNullException("transport"); 17 | if (stateMachine == null) throw new ArgumentNullException("stateMachine"); 18 | 19 | SelfConnection = connection; 20 | StorageOptions = storageOptions; 21 | Transport = transport; 22 | StateMachine = stateMachine; 23 | ElectionTimeout = 1200; 24 | HeartbeatTimeout = 300; 25 | Stopwatch = new Stopwatch(); 26 | MaxLogLengthBeforeCompaction = 32 * 1024; 27 | MaxStepDownDrainTime = TimeSpan.FromSeconds(15); 28 | MaxEntriesPerRequest = 256; 29 | } 30 | 31 | public int MaxEntriesPerRequest { get; set; } 32 | public TimeSpan MaxStepDownDrainTime { get; set; } 33 | 34 | public int MaxLogLengthBeforeCompaction { get; set; } 35 | 36 | public Stopwatch Stopwatch { get; set; } 37 | 38 | public string Name { get { return SelfConnection.Name; } } 39 | 40 | public StorageEnvironmentOptions StorageOptions { get; private set; } 41 | 42 | public ITransport Transport { get; private set; } 43 | 44 | public IRaftStateMachine StateMachine { get; private set; } 45 | 46 | public int ElectionTimeout { get; set; } 47 | public int HeartbeatTimeout { get; set; } 48 | 49 | public NodeConnectionInfo SelfConnection { get; set; } 50 | } 51 | } -------------------------------------------------------------------------------- /Rachis/RaftEngineState.cs: -------------------------------------------------------------------------------- 1 | namespace Rachis 2 | { 3 | public enum RaftEngineState 4 | { 5 | None, 6 | Follower, 7 | Leader, 8 | Candidate, 9 | SteppingDown, 10 | SnapshotInstallation, 11 | 12 | FollowerAfterStepDown, 13 | CandidateByRequest, 14 | } 15 | } -------------------------------------------------------------------------------- /Rachis/Storage/Topology.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using Newtonsoft.Json; 5 | using Rachis.Transport; 6 | 7 | namespace Rachis.Storage 8 | { 9 | 10 | 11 | [JsonObject(MemberSerialization.OptIn)] 12 | public class Topology 13 | { 14 | [JsonProperty("AllNodes")] 15 | private readonly Dictionary _allNodes; 16 | [JsonProperty("AllVotingNodes")] 17 | private readonly Dictionary _allVotingNodes; 18 | [JsonProperty("NonVotingNodes")] 19 | private readonly Dictionary _nonVotingNodes; 20 | [JsonProperty("PromotableNodes")] 21 | private readonly Dictionary _promotableNodes; 22 | [JsonProperty] 23 | public Guid TopologyId { get; private set; } 24 | 25 | private string _topologyString; 26 | 27 | public Topology() 28 | { 29 | _allVotingNodes = new Dictionary(StringComparer.OrdinalIgnoreCase); 30 | _nonVotingNodes = new Dictionary(StringComparer.OrdinalIgnoreCase); 31 | _promotableNodes = new Dictionary(StringComparer.OrdinalIgnoreCase); 32 | _allNodes = new Dictionary(StringComparer.OrdinalIgnoreCase); 33 | } 34 | 35 | public Topology(Guid topologyId) : this() 36 | { 37 | TopologyId = topologyId; 38 | } 39 | 40 | public Topology(Guid topologyId,IEnumerable allVotingNodes, IEnumerable nonVotingNodes, 41 | IEnumerable promotableNodes) 42 | : this(topologyId) 43 | { 44 | foreach (NodeConnectionInfo nodeConnectionInfo in allVotingNodes) 45 | { 46 | _allVotingNodes[nodeConnectionInfo.Name] = nodeConnectionInfo; 47 | _allNodes[nodeConnectionInfo.Name] = nodeConnectionInfo; 48 | } 49 | foreach (NodeConnectionInfo nodeConnectionInfo in nonVotingNodes) 50 | { 51 | _nonVotingNodes[nodeConnectionInfo.Name] = nodeConnectionInfo; 52 | _allNodes[nodeConnectionInfo.Name] = nodeConnectionInfo; 53 | } 54 | foreach (NodeConnectionInfo nodeConnectionInfo in promotableNodes) 55 | { 56 | _promotableNodes[nodeConnectionInfo.Name] = nodeConnectionInfo; 57 | _allNodes[nodeConnectionInfo.Name] = nodeConnectionInfo; 58 | } 59 | 60 | 61 | CreateTopologyString(); 62 | } 63 | 64 | public IEnumerable AllVotingNodes 65 | { 66 | get { return _allVotingNodes.Values; } 67 | } 68 | 69 | public IEnumerable NonVotingNodes 70 | { 71 | get { return _nonVotingNodes.Values; } 72 | } 73 | 74 | public IEnumerable PromotableNodes 75 | { 76 | get { return _promotableNodes.Values; } 77 | } 78 | 79 | public int QuorumSize 80 | { 81 | get { return (_allVotingNodes.Count/2) + 1; } 82 | } 83 | 84 | public IEnumerable AllNodeNames 85 | { 86 | get { return _allNodes.Keys; } 87 | } 88 | 89 | public IEnumerable AllNodes 90 | { 91 | get { return _allNodes.Values; } 92 | } 93 | 94 | public bool HasVoters 95 | { 96 | get { return _allVotingNodes.Count > 0; } 97 | } 98 | 99 | private void CreateTopologyString() 100 | { 101 | if (_allNodes.Count == 0) 102 | { 103 | _topologyString = ""; 104 | return; 105 | } 106 | 107 | _topologyString = ""; 108 | if (_allVotingNodes.Count > 0) 109 | _topologyString += "Voting: [" + string.Join(", ", _allVotingNodes.Keys) + "] "; 110 | if (_nonVotingNodes.Count > 0) 111 | _topologyString += "Non voting: [" + string.Join(", ", _nonVotingNodes.Keys) + "] "; 112 | if (_promotableNodes.Count > 0) 113 | _topologyString += "Promotables: [" + string.Join(", ", _promotableNodes.Keys) + "] "; 114 | } 115 | 116 | public bool HasQuorum(HashSet votes) 117 | { 118 | return votes.Count(IsVoter) >= QuorumSize; 119 | } 120 | 121 | public override string ToString() 122 | { 123 | if (_topologyString == null) 124 | CreateTopologyString(); 125 | return _topologyString; 126 | } 127 | 128 | public bool Contains(string node) 129 | { 130 | return _allVotingNodes.ContainsKey(node) || _nonVotingNodes.ContainsKey(node) || _promotableNodes.ContainsKey(node); 131 | } 132 | 133 | public bool IsVoter(string node) 134 | { 135 | return _allVotingNodes.ContainsKey(node); 136 | } 137 | 138 | public bool IsPromotable(string node) 139 | { 140 | return _promotableNodes.ContainsKey(node); 141 | } 142 | 143 | public NodeConnectionInfo GetNodeByName(string node) 144 | { 145 | NodeConnectionInfo info; 146 | _allNodes.TryGetValue(node, out info); 147 | return info; 148 | } 149 | } 150 | } -------------------------------------------------------------------------------- /Rachis/Transport/HttpTransport.cs: -------------------------------------------------------------------------------- 1 | // ----------------------------------------------------------------------- 2 | // 3 | // Copyright (c) Hibernating Rhinos LTD. All rights reserved. 4 | // 5 | // ----------------------------------------------------------------------- 6 | 7 | using System; 8 | using System.IO; 9 | using System.Net.Http; 10 | using System.Threading; 11 | using System.Threading.Tasks; 12 | using Rachis.Interfaces; 13 | using Rachis.Messages; 14 | 15 | namespace Rachis.Transport 16 | { 17 | public class HttpTransport : ITransport, IDisposable 18 | { 19 | private readonly HttpTransportBus _bus; 20 | private readonly HttpTransportSender _sender; 21 | 22 | public HttpTransport(string name) 23 | { 24 | _bus = new HttpTransportBus(name); 25 | _sender = new HttpTransportSender(name,_bus); 26 | } 27 | 28 | public void Send(NodeConnectionInfo dest, DisconnectedFromCluster req) 29 | { 30 | _sender.Send(dest, req); 31 | } 32 | 33 | public void Send(NodeConnectionInfo dest, AppendEntriesRequest req) 34 | { 35 | _sender.Send(dest, req); 36 | } 37 | 38 | public void Stream(NodeConnectionInfo dest, InstallSnapshotRequest req, Action streamWriter) 39 | { 40 | _sender.Stream(dest, req, streamWriter); 41 | } 42 | 43 | public void Send(NodeConnectionInfo dest, CanInstallSnapshotRequest req) 44 | { 45 | _sender.Send(dest, req); 46 | } 47 | 48 | public void Send(NodeConnectionInfo dest, RequestVoteRequest req) 49 | { 50 | _sender.Send(dest, req); 51 | } 52 | 53 | public void Send(NodeConnectionInfo dest, TimeoutNowRequest req) 54 | { 55 | _sender.Send(dest, req); 56 | } 57 | 58 | public void SendToSelf(AppendEntriesResponse resp) 59 | { 60 | _bus.SendToSelf(resp); 61 | } 62 | 63 | public void Publish(object msg, TaskCompletionSource source, Stream stream = null) 64 | { 65 | _bus.Publish(msg, source, stream); 66 | } 67 | 68 | public bool TryReceiveMessage(int timeout, CancellationToken cancellationToken, out MessageContext messageContext) 69 | { 70 | return _bus.TryReceiveMessage(timeout, cancellationToken, out messageContext); 71 | } 72 | 73 | public void Dispose() 74 | { 75 | _bus.Dispose(); 76 | _sender.Dispose(); 77 | } 78 | 79 | public HttpTransportBus Bus 80 | { 81 | get { return _bus; } 82 | } 83 | } 84 | } -------------------------------------------------------------------------------- /Rachis/Transport/HttpTransportBus.cs: -------------------------------------------------------------------------------- 1 | // ----------------------------------------------------------------------- 2 | // 3 | // Copyright (c) Hibernating Rhinos LTD. All rights reserved. 4 | // 5 | // ----------------------------------------------------------------------- 6 | 7 | using System; 8 | using System.Collections.Concurrent; 9 | using System.IO; 10 | using System.Net; 11 | using System.Net.Http; 12 | using System.Net.Http.Formatting; 13 | using System.Threading; 14 | using System.Threading.Tasks; 15 | using JetBrains.Annotations; 16 | using NLog; 17 | using Rachis.Messages; 18 | 19 | namespace Rachis.Transport 20 | { 21 | public class HttpTransportBus : IDisposable 22 | { 23 | private readonly string _name; 24 | private readonly BlockingCollection _queue = new BlockingCollection(); 25 | 26 | public HttpTransportBus(string name) 27 | { 28 | _name = name; 29 | Log = LogManager.GetLogger(GetType().Name +"."+ name); 30 | } 31 | 32 | public Logger Log { get; private set; } 33 | 34 | public bool TryReceiveMessage(int timeout, CancellationToken cancellationToken, out MessageContext messageContext) 35 | { 36 | if (timeout < 0) 37 | timeout = 0; 38 | 39 | HttpTransportMessageContext item; 40 | if (_queue.TryTake(out item, timeout, cancellationToken) == false) 41 | { 42 | messageContext = null; 43 | return false; 44 | } 45 | messageContext = item; 46 | return true; 47 | } 48 | 49 | private class HttpTransportMessageContext : MessageContext 50 | { 51 | private readonly TaskCompletionSource _tcs; 52 | private readonly HttpTransportBus _parent; 53 | private bool sent; 54 | public HttpTransportMessageContext(TaskCompletionSource tcs, HttpTransportBus parent) 55 | { 56 | _tcs = tcs; 57 | sent = tcs == null; 58 | _parent = parent; 59 | } 60 | 61 | private void Reply(bool success, object msg) 62 | { 63 | if (_tcs == null) 64 | return; 65 | 66 | 67 | var httpResponseMessage = new HttpResponseMessage( 68 | success ? HttpStatusCode.OK : HttpStatusCode.NotAcceptable 69 | ); 70 | if (msg != null) 71 | { 72 | httpResponseMessage.Content = new ObjectContent(msg.GetType(), msg, new JsonMediaTypeFormatter()); 73 | } 74 | sent = true; 75 | _tcs.TrySetResult(httpResponseMessage); 76 | } 77 | 78 | public override void Reply(CanInstallSnapshotResponse resp) 79 | { 80 | Reply(resp.Success, resp); 81 | } 82 | 83 | public override void Reply(InstallSnapshotResponse resp) 84 | { 85 | Reply(resp.Success, resp); 86 | } 87 | 88 | public override void Reply(AppendEntriesResponse resp) 89 | { 90 | Reply(resp.Success, resp); 91 | } 92 | 93 | public override void Reply(RequestVoteResponse resp) 94 | { 95 | Reply(resp.VoteGranted, resp); 96 | } 97 | 98 | public override void ExecuteInEventLoop(Action action) 99 | { 100 | _parent.Publish(action, null); 101 | } 102 | 103 | public override void Done() 104 | { 105 | if (sent) 106 | return;// nothing to do 107 | 108 | var httpResponseMessage = new HttpResponseMessage(HttpStatusCode.OK); 109 | _tcs.TrySetResult(httpResponseMessage); 110 | } 111 | 112 | public override void Error(Exception exception) 113 | { 114 | _tcs.TrySetException(exception); 115 | } 116 | } 117 | 118 | public void SendToSelf(AppendEntriesResponse resp) 119 | { 120 | Publish(resp, null); 121 | } 122 | 123 | public void Publish(object msg, TaskCompletionSource source, Stream stream = null) 124 | { 125 | if (msg == null) throw new ArgumentNullException("msg"); 126 | _queue.Add(new HttpTransportMessageContext(source, this) 127 | { 128 | Message = msg, 129 | Stream = stream, 130 | }); 131 | } 132 | 133 | public void Dispose() 134 | { 135 | 136 | } 137 | } 138 | } -------------------------------------------------------------------------------- /Rachis/Transport/HttpTransportSender.cs: -------------------------------------------------------------------------------- 1 | // ----------------------------------------------------------------------- 2 | // 3 | // Copyright (c) Hibernating Rhinos LTD. All rights reserved. 4 | // 5 | // ----------------------------------------------------------------------- 6 | 7 | using System; 8 | using System.Collections.Concurrent; 9 | using System.IO; 10 | using System.Linq; 11 | using System.Net; 12 | using System.Net.Http; 13 | using System.Threading.Tasks; 14 | using Newtonsoft.Json; 15 | using NLog; 16 | using Rachis.Messages; 17 | 18 | namespace Rachis.Transport 19 | { 20 | /// 21 | /// All requests are fire & forget, with the reply coming in (if at all) 22 | /// from the resulting thread. 23 | /// 24 | public class HttpTransportSender : IDisposable 25 | { 26 | private readonly HttpTransportBus _bus; 27 | 28 | private readonly ConcurrentDictionary> _httpClientsCache = new ConcurrentDictionary>(); 29 | private readonly Logger _log; 30 | public HttpTransportSender(string name, HttpTransportBus bus) 31 | { 32 | _bus = bus; 33 | _log = LogManager.GetLogger(GetType().Name + "." + name); 34 | } 35 | 36 | 37 | public void Stream(NodeConnectionInfo dest, InstallSnapshotRequest req, Action streamWriter) 38 | { 39 | HttpClient client; 40 | using (GetConnection(dest, out client)) 41 | { 42 | LogStatus("install snapshot to " + dest, async () => 43 | { 44 | var requestUri = 45 | string.Format("raft/installSnapshot?term={0}&=lastIncludedIndex={1}&lastIncludedTerm={2}&from={3}&topology={4}&clusterTopologyId={5}", 46 | req.Term, req.LastIncludedIndex, req.LastIncludedTerm, req.From, Uri.EscapeDataString(JsonConvert.SerializeObject(req.Topology)), req.ClusterTopologyId); 47 | var httpResponseMessage = await client.PostAsync(requestUri, new SnapshotContent(streamWriter)); 48 | var reply = await httpResponseMessage.Content.ReadAsStringAsync(); 49 | if (httpResponseMessage.IsSuccessStatusCode == false && httpResponseMessage.StatusCode != HttpStatusCode.NotAcceptable) 50 | { 51 | _log.Warn("Error installing snapshot to {0}. Status: {1}\r\n{2}", dest.Name, httpResponseMessage.StatusCode, reply); 52 | return; 53 | } 54 | var installSnapshotResponse = JsonConvert.DeserializeObject(reply); 55 | SendToSelf(installSnapshotResponse); 56 | }); 57 | } 58 | } 59 | 60 | public class SnapshotContent : HttpContent 61 | { 62 | private readonly Action _streamWriter; 63 | 64 | public SnapshotContent(Action streamWriter) 65 | { 66 | _streamWriter = streamWriter; 67 | } 68 | 69 | protected override Task SerializeToStreamAsync(Stream stream, TransportContext context) 70 | { 71 | _streamWriter(stream); 72 | 73 | return Task.FromResult(1); 74 | } 75 | 76 | protected override bool TryComputeLength(out long length) 77 | { 78 | length = -1; 79 | return false; 80 | } 81 | } 82 | 83 | public void Send(NodeConnectionInfo dest, AppendEntriesRequest req) 84 | { 85 | HttpClient client; 86 | using (GetConnection(dest, out client)) 87 | { 88 | LogStatus("append entries to " + dest, async () => 89 | { 90 | var requestUri = string.Format("raft/appendEntries?term={0}&leaderCommit={1}&prevLogTerm={2}&prevLogIndex={3}&entriesCount={4}&from={5}&clusterTopologyId={6}", 91 | req.Term, req.LeaderCommit, req.PrevLogTerm, req.PrevLogIndex, req.EntriesCount, req.From, req.ClusterTopologyId); 92 | var httpResponseMessage = await client.PostAsync(requestUri,new EntriesContent(req.Entries)); 93 | var reply = await httpResponseMessage.Content.ReadAsStringAsync(); 94 | if (httpResponseMessage.IsSuccessStatusCode == false && httpResponseMessage.StatusCode != HttpStatusCode.NotAcceptable) 95 | { 96 | _log.Warn("Error appending entries to {0}. Status: {1}\r\n{2}", dest.Name, httpResponseMessage.StatusCode, reply); 97 | return; 98 | } 99 | var appendEntriesResponse = JsonConvert.DeserializeObject(reply); 100 | SendToSelf(appendEntriesResponse); 101 | }); 102 | } 103 | } 104 | 105 | private class EntriesContent : HttpContent 106 | { 107 | private readonly LogEntry[] _entries; 108 | 109 | public EntriesContent(LogEntry[] entries) 110 | { 111 | _entries = entries; 112 | } 113 | 114 | protected override Task SerializeToStreamAsync(Stream stream, TransportContext context) 115 | { 116 | foreach (var logEntry in _entries) 117 | { 118 | Write7BitEncodedInt64(stream, logEntry.Index); 119 | Write7BitEncodedInt64(stream, logEntry.Term); 120 | stream.WriteByte(logEntry.IsTopologyChange == true ? (byte)1 : (byte)0); 121 | Write7BitEncodedInt64(stream, logEntry.Data.Length); 122 | stream.Write(logEntry.Data, 0, logEntry.Data.Length); 123 | } 124 | return Task.FromResult(1); 125 | } 126 | 127 | private void Write7BitEncodedInt64(Stream stream, long value) 128 | { 129 | var v = (ulong)value; 130 | while (v >= 128) 131 | { 132 | stream.WriteByte((byte)(v | 128)); 133 | v >>= 7; 134 | } 135 | stream.WriteByte((byte)(v)); 136 | } 137 | 138 | protected override bool TryComputeLength(out long length) 139 | { 140 | length = -1; 141 | return false; 142 | } 143 | } 144 | 145 | public void Send(NodeConnectionInfo dest, CanInstallSnapshotRequest req) 146 | { 147 | HttpClient client; 148 | using (GetConnection(dest, out client)) 149 | { 150 | LogStatus("can install snapshot to " + dest, async () => 151 | { 152 | var requestUri = string.Format("raft/canInstallSnapshot?term={0}&=index{1}&from={2}&clusterTopologyId={3}", req.Term, req.Index, 153 | req.From, req.ClusterTopologyId); 154 | var httpResponseMessage = await client.GetAsync(requestUri); 155 | var reply = await httpResponseMessage.Content.ReadAsStringAsync(); 156 | if (httpResponseMessage.IsSuccessStatusCode == false && httpResponseMessage.StatusCode != HttpStatusCode.NotAcceptable) 157 | { 158 | _log.Warn("Error checking if can install snapshot to {0}. Status: {1}\r\n{2}", dest.Name, httpResponseMessage.StatusCode, reply); 159 | return; 160 | } 161 | var canInstallSnapshotResponse = JsonConvert.DeserializeObject(reply); 162 | SendToSelf(canInstallSnapshotResponse); 163 | }); 164 | } 165 | } 166 | 167 | public void Send(NodeConnectionInfo dest, RequestVoteRequest req) 168 | { 169 | HttpClient client; 170 | using (GetConnection(dest, out client)) 171 | { 172 | LogStatus("request vote from " + dest, async () => 173 | { 174 | var requestUri = string.Format("raft/requestVote?term={0}&lastLogIndex={1}&lastLogTerm={2}&trialOnly={3}&forcedElection={4}&from={5}&clusterTopologyId={6}", 175 | req.Term, req.LastLogIndex, req.LastLogTerm, req.TrialOnly, req.ForcedElection, req.From, req.ClusterTopologyId); 176 | var httpResponseMessage = await client.GetAsync(requestUri); 177 | var reply = await httpResponseMessage.Content.ReadAsStringAsync(); 178 | if (httpResponseMessage.IsSuccessStatusCode == false && httpResponseMessage.StatusCode != HttpStatusCode.NotAcceptable) 179 | { 180 | _log.Warn("Error requesting vote from {0}. Status: {1}\r\n{2}", dest.Name, httpResponseMessage.StatusCode, reply); 181 | return; 182 | } 183 | var requestVoteResponse = JsonConvert.DeserializeObject(reply); 184 | SendToSelf(requestVoteResponse); 185 | }); 186 | } 187 | } 188 | 189 | private void SendToSelf(object o) 190 | { 191 | _bus.Publish(o, source: null); 192 | } 193 | 194 | public void Send(NodeConnectionInfo dest, TimeoutNowRequest req) 195 | { 196 | HttpClient client; 197 | using (GetConnection(dest, out client)) 198 | { 199 | LogStatus("timeout to " + dest, async () => 200 | { 201 | var message = await client.GetAsync(string.Format("raft/timeoutNow?term={0}&from={1}&clusterTopologyId={2}", req.Term, req.From, req.ClusterTopologyId)); 202 | var reply = await message.Content.ReadAsStringAsync(); 203 | if (message.IsSuccessStatusCode == false) 204 | { 205 | _log.Warn("Error appending entries to {0}. Status: {1}\r\n{2}", dest.Name, message.StatusCode, message, reply); 206 | return; 207 | } 208 | SendToSelf(new NothingToDo()); 209 | }); 210 | } 211 | } 212 | 213 | public void Send(NodeConnectionInfo dest, DisconnectedFromCluster req) 214 | { 215 | HttpClient client; 216 | using (GetConnection(dest, out client)) 217 | { 218 | LogStatus("disconnect " + dest, async () => 219 | { 220 | var message = await client.GetAsync(string.Format("raft/disconnectFromCluster?term={0}&from={1}&clusterTopologyId={2}", req.Term, req.From, req.ClusterTopologyId)); 221 | var reply = await message.Content.ReadAsStringAsync(); 222 | if (message.IsSuccessStatusCode == false) 223 | { 224 | _log.Warn("Error sending disconnecton notification to {0}. Status: {1}\r\n{2}", dest.Name, message.StatusCode, message, reply); 225 | return; 226 | } 227 | SendToSelf(new NothingToDo()); 228 | }); 229 | } 230 | } 231 | 232 | private ConcurrentDictionary _runningOps = new ConcurrentDictionary(); 233 | 234 | private void LogStatus(string details, Func operation) 235 | { 236 | var op = operation(); 237 | _runningOps.TryAdd(op, op); 238 | op 239 | .ContinueWith(task => 240 | { 241 | object value; 242 | _runningOps.TryRemove(op, out value); 243 | if (task.Exception != null) 244 | { 245 | _log.Warn("Failed to send " + details + " " + InnerMostMessage(task.Exception), task.Exception); 246 | return; 247 | } 248 | _log.Info("Sent {0}", details); 249 | }); 250 | } 251 | 252 | private string InnerMostMessage(Exception exception) 253 | { 254 | if (exception.InnerException == null) 255 | return exception.Message; 256 | return InnerMostMessage(exception.InnerException); 257 | } 258 | 259 | 260 | public void Dispose() 261 | { 262 | foreach (var q in _httpClientsCache.Select(x=>x.Value)) 263 | { 264 | HttpClient result; 265 | while (q.TryDequeue(out result)) 266 | { 267 | result.Dispose(); 268 | } 269 | } 270 | _httpClientsCache.Clear(); 271 | var array = _runningOps.Keys.ToArray(); 272 | _runningOps.Clear(); 273 | try 274 | { 275 | Task.WaitAll(array); 276 | } 277 | catch (OperationCanceledException) 278 | { 279 | // nothing to do here 280 | } 281 | catch (AggregateException e) 282 | { 283 | if (e.InnerException is OperationCanceledException == false) 284 | throw; 285 | // nothing to do here 286 | } 287 | } 288 | 289 | 290 | private ReturnToQueue GetConnection(NodeConnectionInfo info, out HttpClient result) 291 | { 292 | var connectionQueue = _httpClientsCache.GetOrAdd(info.Name, _ => new ConcurrentQueue()); 293 | 294 | if (connectionQueue.TryDequeue(out result) == false) 295 | { 296 | result = new HttpClient 297 | { 298 | BaseAddress = info.Uri 299 | }; 300 | } 301 | 302 | return new ReturnToQueue(result, connectionQueue); 303 | } 304 | 305 | private struct ReturnToQueue : IDisposable 306 | { 307 | private readonly HttpClient client; 308 | private readonly ConcurrentQueue queue; 309 | 310 | public ReturnToQueue(HttpClient client, ConcurrentQueue queue) 311 | { 312 | this.client = client; 313 | this.queue = queue; 314 | } 315 | 316 | public void Dispose() 317 | { 318 | queue.Enqueue(client); 319 | } 320 | } 321 | 322 | } 323 | } -------------------------------------------------------------------------------- /Rachis/Transport/InMemoryTransportHub.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.IO; 5 | using System.Threading; 6 | using NLog; 7 | using Rachis.Interfaces; 8 | using Rachis.Messages; 9 | 10 | namespace Rachis.Transport 11 | { 12 | public class InMemoryTransportHub 13 | { 14 | private readonly ConcurrentDictionary> _messageQueue = 15 | new ConcurrentDictionary>(); 16 | 17 | private readonly HashSet _disconnectedNodes = new HashSet(); 18 | 19 | private readonly HashSet _disconnectedNodesFromSending = new HashSet(); 20 | 21 | private readonly Dictionary _transports = new Dictionary(); 22 | 23 | public ConcurrentDictionary> MessageQueue 24 | { 25 | get { return _messageQueue; } 26 | } 27 | 28 | public ITransport CreateTransportFor(string from) 29 | { 30 | InMemoryTransport value; 31 | if (_transports.TryGetValue(from, out value)) 32 | return value; 33 | value = new InMemoryTransport(this, from); 34 | _transports[from] = value; 35 | return value; 36 | } 37 | 38 | public class InMemoryTransport : ITransport 39 | { 40 | private readonly InMemoryTransportHub _parent; 41 | private readonly string _from; 42 | 43 | public readonly Logger Log; 44 | 45 | public InMemoryTransport(InMemoryTransportHub parent, string from) 46 | { 47 | _parent = parent; 48 | _from = from; 49 | Log = LogManager.GetLogger(typeof (InMemoryTransport).FullName + "." + from); 50 | } 51 | 52 | public string From 53 | { 54 | get { return _from; } 55 | } 56 | 57 | public bool TryReceiveMessage(int timeout, CancellationToken cancellationToken, out MessageContext messageContext) 58 | { 59 | return _parent.TryReceiveMessage(_from, timeout, cancellationToken, out messageContext); 60 | } 61 | 62 | 63 | public void Stream(NodeConnectionInfo dest, InstallSnapshotRequest snapshotRequest, Action streamWriter) 64 | { 65 | var stream = new MemoryStream(); 66 | streamWriter(stream); 67 | stream.Position = 0; 68 | 69 | _parent.AddToQueue(this, dest.Name, snapshotRequest, stream); 70 | } 71 | 72 | public void Send(NodeConnectionInfo dest, CanInstallSnapshotRequest req) 73 | { 74 | _parent.AddToQueue(this, dest.Name, req); 75 | } 76 | 77 | public void SendInternal(string dest, string from, object msg) 78 | { 79 | _parent.AddToQueue(this, dest, msg); 80 | } 81 | 82 | public void Send(NodeConnectionInfo dest, TimeoutNowRequest req) 83 | { 84 | _parent.AddToQueue(this, dest.Name, req); 85 | } 86 | 87 | public void Send(NodeConnectionInfo dest, DisconnectedFromCluster req) 88 | { 89 | _parent.AddToQueue(this, dest.Name, req); 90 | } 91 | 92 | public void Send(NodeConnectionInfo dest, AppendEntriesRequest req) 93 | { 94 | _parent.AddToQueue(this, dest.Name, req); 95 | } 96 | 97 | public void Send(NodeConnectionInfo dest, RequestVoteRequest req) 98 | { 99 | _parent.AddToQueue(this, dest.Name, req); 100 | } 101 | 102 | public void SendToSelf(AppendEntriesResponse resp) 103 | { 104 | _parent.AddToQueue(this, From, resp); 105 | } 106 | 107 | public void ForceTimeout() 108 | { 109 | _parent.AddToQueue(this, From, new TimeoutException(), evenIfDisconnected: true); 110 | } 111 | } 112 | 113 | private void AddToQueue(InMemoryTransport src, string dest, T message, Stream stream = null, 114 | bool evenIfDisconnected = false) 115 | { 116 | //if destination is considered disconnected --> drop the message so it never arrives 117 | if (( 118 | _disconnectedNodes.Contains(dest) || 119 | _disconnectedNodesFromSending.Contains(src.From) 120 | ) && evenIfDisconnected == false) 121 | return; 122 | 123 | var newMessage = new InMemoryMessageContext(src) 124 | { 125 | Destination = dest, 126 | Message = message, 127 | Stream = stream 128 | }; 129 | 130 | _messageQueue.AddOrUpdate(dest, new BlockingCollection { newMessage }, 131 | (destination, envelopes) => 132 | { 133 | envelopes.Add(newMessage); 134 | return envelopes; 135 | }); 136 | } 137 | 138 | private class InMemoryMessageContext : MessageContext 139 | { 140 | private readonly InMemoryTransport _parent; 141 | public string Destination { get; set; } 142 | 143 | public InMemoryMessageContext(InMemoryTransport parent) 144 | { 145 | _parent = parent; 146 | } 147 | 148 | public override void Reply(CanInstallSnapshotResponse resp) 149 | { 150 | _parent.SendInternal(_parent.From, Destination, resp); 151 | } 152 | 153 | public override void Reply(InstallSnapshotResponse resp) 154 | { 155 | _parent.SendInternal(_parent.From, Destination, resp); 156 | } 157 | 158 | public override void Reply(AppendEntriesResponse resp) 159 | { 160 | _parent.SendInternal(_parent.From, Destination, resp); 161 | } 162 | 163 | public override void Reply(RequestVoteResponse resp) 164 | { 165 | _parent.SendInternal(_parent.From, Destination, resp); 166 | } 167 | 168 | public override void ExecuteInEventLoop(Action action) 169 | { 170 | _parent.SendInternal(_parent.From, _parent.From, action); 171 | } 172 | 173 | public override void Done() 174 | { 175 | // nothing to do here. 176 | } 177 | 178 | public override void Error(Exception exception) 179 | { 180 | _parent.Log.Warn("Error processing message", exception); 181 | } 182 | } 183 | 184 | public void DisconnectNodeSending(string node) 185 | { 186 | _disconnectedNodesFromSending.Add(node); 187 | } 188 | 189 | public void ReconnectNodeSending(string node) 190 | { 191 | _disconnectedNodesFromSending.RemoveWhere(n => n.Equals(node, StringComparison.InvariantCultureIgnoreCase)); 192 | } 193 | 194 | public void DisconnectNode(string node) 195 | { 196 | _disconnectedNodes.Add(node); 197 | } 198 | 199 | public void ReconnectNode(string node) 200 | { 201 | _disconnectedNodes.RemoveWhere(n => n.Equals(node, StringComparison.InvariantCultureIgnoreCase)); 202 | } 203 | 204 | public bool TryReceiveMessage(string dest, int timeout, CancellationToken cancellationToken, 205 | out MessageContext messageContext) 206 | { 207 | if (timeout < 0) 208 | timeout = 0; 209 | 210 | var messageQueue = _messageQueue.GetOrAdd(dest, s => new BlockingCollection()); 211 | var tryReceiveMessage = messageQueue.TryTake(out messageContext, timeout, cancellationToken); 212 | if (tryReceiveMessage) 213 | { 214 | if (_disconnectedNodes.Contains(dest) || 215 | messageContext.Message is TimeoutException) 216 | { 217 | messageContext = null; 218 | return false; 219 | } 220 | } 221 | 222 | return tryReceiveMessage; 223 | } 224 | } 225 | } 226 | -------------------------------------------------------------------------------- /Rachis/Transport/NodeConnectionInfo.cs: -------------------------------------------------------------------------------- 1 | // ----------------------------------------------------------------------- 2 | // 3 | // Copyright (c) Hibernating Rhinos LTD. All rights reserved. 4 | // 5 | // ----------------------------------------------------------------------- 6 | 7 | using System; 8 | 9 | namespace Rachis.Transport 10 | { 11 | public class NodeConnectionInfo 12 | { 13 | public Uri Uri { get; set; } 14 | 15 | public string Name { get; set; } 16 | 17 | public string Username { get; set; } 18 | 19 | public string Domain { get; set; } 20 | 21 | public string ApiKey { get; set; } 22 | 23 | public override string ToString() 24 | { 25 | return Name; 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /Rachis/Transport/RaftController.cs: -------------------------------------------------------------------------------- 1 | // ----------------------------------------------------------------------- 2 | // 3 | // Copyright (c) Hibernating Rhinos LTD. All rights reserved. 4 | // 5 | // ----------------------------------------------------------------------- 6 | 7 | using System.Diagnostics; 8 | using System.IO; 9 | using System.Net.Http; 10 | using System.Threading; 11 | using System.Threading.Tasks; 12 | using System.Web.Http; 13 | using System.Web.Http.Controllers; 14 | using Newtonsoft.Json; 15 | using Rachis.Messages; 16 | using Rachis.Storage; 17 | 18 | namespace Rachis.Transport 19 | { 20 | public class RaftController : ApiController 21 | { 22 | private HttpTransportBus _bus; 23 | 24 | public override async Task ExecuteAsync(HttpControllerContext controllerContext, CancellationToken cancellationToken) 25 | { 26 | _bus = (HttpTransportBus) controllerContext.Configuration.Properties[typeof (HttpTransportBus)]; 27 | var sp = Stopwatch.StartNew(); 28 | var msg = await base.ExecuteAsync(controllerContext, cancellationToken); 29 | if (_bus.Log.IsDebugEnabled) 30 | { 31 | _bus.Log.Debug("{0} {1} {2} in {3:#,#;;0} ms", msg.StatusCode, controllerContext.Request.Method, controllerContext.Request.RequestUri, 32 | sp.ElapsedMilliseconds); 33 | } 34 | return msg; 35 | } 36 | 37 | 38 | [HttpPost] 39 | [Route("raft/installSnapshot")] 40 | public async Task InstallSnapshot([FromUri]InstallSnapshotRequest request, [FromUri]string topology) 41 | { 42 | request.Topology = JsonConvert.DeserializeObject(topology); 43 | var stream = await Request.Content.ReadAsStreamAsync(); 44 | var taskCompletionSource = new TaskCompletionSource(); 45 | _bus.Publish(request, taskCompletionSource, stream); 46 | return await taskCompletionSource.Task; 47 | } 48 | 49 | [HttpPost] 50 | [Route("raft/appendEntries")] 51 | public async Task AppendEntries([FromUri]AppendEntriesRequest request, [FromUri]int entriesCount) 52 | { 53 | var stream = await Request.Content.ReadAsStreamAsync(); 54 | request.Entries = new LogEntry[entriesCount]; 55 | for (int i = 0; i < entriesCount; i++) 56 | { 57 | var index = Read7BitEncodedInt(stream); 58 | var term = Read7BitEncodedInt(stream); 59 | var isTopologyChange = stream.ReadByte() == 1; 60 | var lengthOfData = (int)Read7BitEncodedInt(stream); 61 | request.Entries[i] = new LogEntry 62 | { 63 | Index = index, 64 | Term = term, 65 | IsTopologyChange = isTopologyChange, 66 | Data = new byte[lengthOfData] 67 | }; 68 | 69 | var start = 0; 70 | while (start < lengthOfData) 71 | { 72 | var read = stream.Read(request.Entries[i].Data, start, lengthOfData - start); 73 | start += read; 74 | } 75 | } 76 | 77 | var taskCompletionSource = new TaskCompletionSource(); 78 | _bus.Publish(request, taskCompletionSource); 79 | return await taskCompletionSource.Task; 80 | } 81 | 82 | internal protected long Read7BitEncodedInt(Stream stream) 83 | { 84 | long count = 0; 85 | int shift = 0; 86 | byte b; 87 | do 88 | { 89 | if (shift == 9 * 7) 90 | throw new InvalidDataException("Invalid 7bit shifted value, used more than 9 bytes"); 91 | 92 | var maybeEof = stream.ReadByte(); 93 | if (maybeEof == -1) 94 | throw new EndOfStreamException(); 95 | 96 | b = (byte)maybeEof; 97 | count |= (uint)(b & 0x7F) << shift; 98 | shift += 7; 99 | } while ((b & 0x80) != 0); 100 | return count; 101 | } 102 | 103 | [HttpGet] 104 | [Route("raft/requestVote")] 105 | public Task RequestVote([FromUri]RequestVoteRequest request) 106 | { 107 | var taskCompletionSource = new TaskCompletionSource(); 108 | _bus.Publish(request, taskCompletionSource); 109 | return taskCompletionSource.Task; 110 | } 111 | 112 | [HttpGet] 113 | [Route("raft/timeoutNow")] 114 | public Task TimeoutNow([FromUri]TimeoutNowRequest request) 115 | { 116 | var taskCompletionSource = new TaskCompletionSource(); 117 | _bus.Publish(request, taskCompletionSource); 118 | return taskCompletionSource.Task; 119 | } 120 | 121 | [HttpGet] 122 | [Route("raft/disconnectFromCluster")] 123 | public Task DisconnectFromCluster([FromUri]DisconnectedFromCluster request) 124 | { 125 | var taskCompletionSource = new TaskCompletionSource(); 126 | _bus.Publish(request, taskCompletionSource); 127 | return taskCompletionSource.Task; 128 | } 129 | 130 | [HttpGet] 131 | [Route("raft/canInstallSnapshot")] 132 | public Task CanInstallSnapshot([FromUri]CanInstallSnapshotRequest request) 133 | { 134 | var taskCompletionSource = new TaskCompletionSource(); 135 | _bus.Publish(request, taskCompletionSource); 136 | return taskCompletionSource.Task; 137 | } 138 | } 139 | } -------------------------------------------------------------------------------- /Rachis/Transport/RaftWebApiConfig.cs: -------------------------------------------------------------------------------- 1 | // ----------------------------------------------------------------------- 2 | // 3 | // Copyright (c) Hibernating Rhinos LTD. All rights reserved. 4 | // 5 | // ----------------------------------------------------------------------- 6 | 7 | using System.Web.Http; 8 | 9 | namespace Rachis.Transport 10 | { 11 | public static class RaftWebApiConfig 12 | { 13 | public static void Load() 14 | { 15 | // calling this force .NET to load this assembly, so then you can call MapHttpAttributeRoutes 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /Rachis/Utils/NotLeadingException.cs: -------------------------------------------------------------------------------- 1 | // ----------------------------------------------------------------------- 2 | // 3 | // Copyright (c) Hibernating Rhinos LTD. All rights reserved. 4 | // 5 | // ----------------------------------------------------------------------- 6 | 7 | using System; 8 | using System.Runtime.Serialization; 9 | 10 | namespace Rachis.Utils 11 | { 12 | [Serializable] 13 | public class NotLeadingException : Exception 14 | { 15 | public string CurrentLeader { get; set; } 16 | 17 | public NotLeadingException() 18 | { 19 | } 20 | 21 | public NotLeadingException(string message) : base(message) 22 | { 23 | } 24 | 25 | public NotLeadingException(string message, Exception inner) : base(message, inner) 26 | { 27 | } 28 | 29 | protected NotLeadingException( 30 | SerializationInfo info, 31 | StreamingContext context) : base(info, context) 32 | { 33 | } 34 | } 35 | } -------------------------------------------------------------------------------- /Rachis/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /TailFeather.Client/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("TailFeather.Client")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("TailFeather.Client")] 13 | [assembly: AssemblyCopyright("Copyright © 2014")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("b6c71761-2ce5-44cc-82a7-691b0920d75e")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /TailFeather.Client/TailFeather.Client.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {1E267CF9-EAE8-4F87-BE37-BB1E7C9E86D6} 8 | Library 9 | Properties 10 | TailFeather.Client 11 | TailFeather.Client 12 | 512 13 | ..\ 14 | true 15 | 16 | v4.5 17 | 18 | 19 | true 20 | full 21 | false 22 | bin\Debug\ 23 | DEBUG;TRACE 24 | prompt 25 | 4 26 | false 27 | 28 | 29 | pdbonly 30 | true 31 | bin\Release\ 32 | TRACE 33 | prompt 34 | 4 35 | false 36 | 37 | 38 | 39 | ..\packages\Newtonsoft.Json.6.0.4\lib\net40\Newtonsoft.Json.dll 40 | monodevelop 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | {F796F69F-D17B-4260-92D6-65CB94C0E05C} 62 | Rachis 63 | 64 | 65 | 66 | 67 | 68 | 69 | This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. 70 | 71 | 72 | 73 | 80 | -------------------------------------------------------------------------------- /TailFeather.Client/TailFeatherClient.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.IO; 5 | using System.Linq; 6 | using System.Net; 7 | using System.Net.Http; 8 | using System.Threading.Tasks; 9 | using Newtonsoft.Json; 10 | using Newtonsoft.Json.Linq; 11 | 12 | namespace TailFeather.Client 13 | { 14 | public class TailFeatherClient : IDisposable 15 | { 16 | private readonly ConcurrentDictionary _cache = new ConcurrentDictionary(); 17 | private Task _topologyTask; 18 | 19 | public TailFeatherClient(params Uri[] nodes) 20 | { 21 | _topologyTask = FindLatestTopology(nodes); 22 | } 23 | 24 | private HttpClient GetHttpClient(Uri node) 25 | { 26 | return _cache.GetOrAdd(node, uri => new HttpClient { BaseAddress = uri }); 27 | } 28 | 29 | private async Task FindLatestTopology(IEnumerable nodes) 30 | { 31 | var tasks = nodes.Select(node => GetHttpClient(node).GetAsync("tailfeather/admin/flock")).ToArray(); 32 | 33 | await Task.WhenAny(tasks); 34 | var topologies = new List(); 35 | foreach (var task in tasks) 36 | { 37 | var message = task.Result; 38 | if (message.IsSuccessStatusCode == false) 39 | continue; 40 | 41 | topologies.Add(new JsonSerializer().Deserialize( 42 | new JsonTextReader(new StreamReader(await message.Content.ReadAsStreamAsync())))); 43 | } 44 | 45 | return topologies.OrderByDescending(x => x.CommitIndex).FirstOrDefault(); 46 | } 47 | 48 | private async Task ContactServer(Func> operation, int retries = 3) 49 | { 50 | if (retries < 0) 51 | throw new InvalidOperationException("Cluster is not reachable, or no leader was selected. Out of retries, aborting."); 52 | 53 | var topology = (await _topologyTask ?? new TailFeatherTopology()); 54 | 55 | var leader = topology.AllVotingNodes.FirstOrDefault(x => x.Name == topology.CurrentLeader); 56 | if (leader == null) 57 | { 58 | _topologyTask = FindLatestTopology(topology.AllVotingNodes.Select(x => x.Uri)); 59 | return await ContactServer(operation, retries - 1); 60 | } 61 | 62 | // now we have a leader, we need to try calling it... 63 | var httpResponseMessage = await operation(GetHttpClient(leader.Uri)); 64 | if (httpResponseMessage.IsSuccessStatusCode == false) 65 | { 66 | // we were sent to a different server, let try that... 67 | if (httpResponseMessage.StatusCode == HttpStatusCode.Redirect) 68 | { 69 | var redirectUri = httpResponseMessage.Headers.Location; 70 | httpResponseMessage = await operation(GetHttpClient(redirectUri)); 71 | if (httpResponseMessage.IsSuccessStatusCode) 72 | { 73 | // we successfully contacted the redirected server, this is probably the leader, let us ask it for the topology, 74 | // it will be there for next time we access it 75 | _topologyTask = FindLatestTopology(new[] { redirectUri }.Union(topology.AllVotingNodes.Select(x => x.Uri))); 76 | 77 | return httpResponseMessage; 78 | } 79 | } 80 | 81 | // we couldn't get to the server, and we didn't get redirected, we'll check in the cluster in general 82 | _topologyTask = FindLatestTopology(topology.AllVotingNodes.Select(x => x.Uri)); 83 | return await ContactServer(operation, retries - 1); 84 | } 85 | 86 | // happy path, we are done 87 | return httpResponseMessage; 88 | } 89 | 90 | public async Task Cas(string key, JToken value, JToken previousValue) 91 | { 92 | var reply = await ContactServer(client => client.GetAsync(string.Format("tailfeather/key-val/cas?key={0}&val={1}&prevValue={2}", 93 | Uri.EscapeDataString(key), 94 | Uri.EscapeDataString(value.ToString(Formatting.None)), 95 | Uri.EscapeDataString(previousValue.ToString(Formatting.None))))); 96 | var result = JObject.Load(new JsonTextReader(new StreamReader(await reply.Content.ReadAsStreamAsync()))); 97 | 98 | return result.Value("ValueChanged"); 99 | } 100 | 101 | public Task Set(string key, JToken value) 102 | { 103 | return ContactServer(client => client.GetAsync(string.Format("tailfeather/key-val/set?key={0}&val={1}", 104 | Uri.EscapeDataString(key), Uri.EscapeDataString(value.ToString(Formatting.None))))); 105 | } 106 | 107 | public async Task Get(string key) 108 | { 109 | var reply = await ContactServer(client => client.GetAsync(string.Format("tailfeather/key-val/read?key={0}", 110 | Uri.EscapeDataString(key)))); 111 | var result = JObject.Load(new JsonTextReader(new StreamReader(await reply.Content.ReadAsStreamAsync()))); 112 | 113 | if (result.Value("Missing")) 114 | return null; 115 | 116 | return result["Value"]; 117 | } 118 | 119 | public Task Remove(string key) 120 | { 121 | return ContactServer(client => client.GetAsync(string.Format("tailfeather/key-val/del?key={0}", 122 | Uri.EscapeDataString(key)))); 123 | } 124 | 125 | public void Dispose() 126 | { 127 | foreach (var httpClient in _cache) 128 | { 129 | httpClient.Value.Dispose(); 130 | } 131 | _cache.Clear(); 132 | } 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /TailFeather.Client/TailFeatherTopology.cs: -------------------------------------------------------------------------------- 1 | using Rachis.Transport; 2 | 3 | namespace TailFeather.Client 4 | { 5 | public class TailFeatherTopology 6 | { 7 | public string CurrentLeader { get; set; } 8 | 9 | public long CurrentTerm { get; set; } 10 | 11 | public long CommitIndex { get; set; } 12 | 13 | public NodeConnectionInfo[] AllVotingNodes { get; set; } 14 | 15 | public NodeConnectionInfo[] PromotableNodes { get; set; } 16 | 17 | public NodeConnectionInfo[] NonVotingNodes { get; set; } 18 | 19 | } 20 | } -------------------------------------------------------------------------------- /TailFeather.Client/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | -------------------------------------------------------------------------------- /TailFeather/Controllers/AdminController.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Net; 3 | using System.Net.Http; 4 | using System.Threading.Tasks; 5 | using System.Web.Http; 6 | using Rachis.Transport; 7 | 8 | namespace TailFeather.Controllers 9 | { 10 | public class AdminController : TailFeatherController 11 | { 12 | [HttpGet] 13 | [Route("tailfeather/admin/flock")] 14 | public HttpResponseMessage Topology() 15 | { 16 | return Request.CreateResponse(HttpStatusCode.OK, new 17 | { 18 | RaftEngine.CurrentLeader, 19 | RaftEngine.PersistentState.CurrentTerm, 20 | RaftEngine.State, 21 | RaftEngine.CommitIndex, 22 | RaftEngine.CurrentTopology.AllVotingNodes, 23 | RaftEngine.CurrentTopology.PromotableNodes, 24 | RaftEngine.CurrentTopology.NonVotingNodes 25 | }); 26 | } 27 | 28 | [HttpGet] 29 | [Route("tailfeather/admin/fly-with-us")] 30 | public async Task Join([FromUri] string url, [FromUri] string name) 31 | { 32 | var uri = new Uri(url); 33 | name = name ?? uri.Host + (uri.IsDefaultPort ? "" : ":" + uri.Port); 34 | 35 | await RaftEngine.AddToClusterAsync(new NodeConnectionInfo 36 | { 37 | Name = name, 38 | Uri = uri 39 | }); 40 | return new HttpResponseMessage(HttpStatusCode.Accepted); 41 | } 42 | 43 | [HttpGet] 44 | [Route("tailfeather/admin/fly-away")] 45 | public async Task Leave([FromUri] string name) 46 | { 47 | await RaftEngine.RemoveFromClusterAsync(new NodeConnectionInfo 48 | { 49 | Name = name 50 | }); 51 | return new HttpResponseMessage(HttpStatusCode.Accepted); 52 | } 53 | } 54 | } -------------------------------------------------------------------------------- /TailFeather/Controllers/KeyValueController.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Linq; 4 | using System.Net; 5 | using System.Net.Http; 6 | using System.Threading.Tasks; 7 | using System.Web.Http; 8 | using Newtonsoft.Json; 9 | using Newtonsoft.Json.Linq; 10 | using Rachis; 11 | using Rachis.Utils; 12 | using TailFeather.Storage; 13 | 14 | namespace TailFeather.Controllers 15 | { 16 | public class KeyValueController : TailFeatherController 17 | { 18 | [HttpGet] 19 | [Route("tailfeather/key-val/read")] 20 | public async Task Read([FromUri] string key, [FromUri] string mode = null) 21 | { 22 | switch (mode) 23 | { 24 | case "quorum": 25 | var taskCompletionSource = new TaskCompletionSource(); 26 | try 27 | { 28 | RaftEngine.AppendCommand(new GetCommand 29 | { 30 | Key = key, 31 | Completion = taskCompletionSource 32 | }); 33 | } 34 | catch (NotLeadingException e) 35 | { 36 | return RedirectToLeader(e.CurrentLeader, Request.RequestUri); 37 | } 38 | var consistentRead = await taskCompletionSource.Task; 39 | return Request.CreateResponse(HttpStatusCode.OK, new 40 | { 41 | RaftEngine.State, 42 | Key = key, 43 | Value = consistentRead, 44 | Missing = consistentRead == null 45 | }); 46 | case "leader": 47 | if (RaftEngine.State != RaftEngineState.Leader) 48 | { 49 | return RedirectToLeader(RaftEngine.CurrentLeader, Request.RequestUri); 50 | } 51 | goto case null; 52 | case "any": 53 | case null: 54 | var read = StateMachine.Read(key); 55 | return Request.CreateResponse(HttpStatusCode.OK, new 56 | { 57 | RaftEngine.State, 58 | Key = key, 59 | Value = read, 60 | Missing = read == null 61 | }); 62 | default: 63 | return Request.CreateResponse(HttpStatusCode.BadRequest, new 64 | { 65 | Error = "Unknown read mode" 66 | }); 67 | } 68 | 69 | } 70 | 71 | private HttpResponseMessage RedirectToLeader(string currentLeader, Uri baseUrl) 72 | { 73 | var leaderNode = RaftEngine.CurrentTopology.AllNodes.FirstOrDefault(x => { return x.Name == currentLeader; }); 74 | if (leaderNode == null) 75 | { 76 | return Request.CreateResponse(HttpStatusCode.BadRequest, new 77 | { 78 | Error = "There is no current leader, try again later" 79 | }); 80 | } 81 | var httpResponseMessage = Request.CreateResponse(HttpStatusCode.Redirect); 82 | httpResponseMessage.Headers.Location = new UriBuilder(leaderNode.Uri) 83 | { 84 | Path = baseUrl.LocalPath, 85 | Query = baseUrl.Query.TrimStart('?'), 86 | Fragment = baseUrl.Fragment 87 | }.Uri; 88 | return httpResponseMessage; 89 | } 90 | 91 | [HttpGet] 92 | [Route("tailfeather/key-val/set")] 93 | public Task Set([FromUri] string key, [FromUri] string val) 94 | { 95 | JToken jVal; 96 | try 97 | { 98 | jVal = JToken.Parse(val); 99 | } 100 | catch (JsonReaderException) 101 | { 102 | jVal = val; 103 | } 104 | 105 | var op = new KeyValueOperation 106 | { 107 | Key = key, 108 | Type = KeyValueOperationTypes.Add, 109 | Value = jVal 110 | }; 111 | 112 | return Batch(new[] { op }); 113 | } 114 | 115 | [HttpGet] 116 | [Route("tailfeather/key-val/cas")] 117 | public async Task Cas([FromUri] string key, [FromUri] string val, [FromUri] string prevValue) 118 | { 119 | JToken jVal; 120 | try 121 | { 122 | jVal = JToken.Parse(val); 123 | } 124 | catch (JsonReaderException) 125 | { 126 | jVal = val; 127 | } 128 | 129 | JToken jPrevVal; 130 | try 131 | { 132 | jPrevVal = JToken.Parse(prevValue); 133 | } 134 | catch (JsonReaderException) 135 | { 136 | jPrevVal = prevValue; 137 | } 138 | 139 | 140 | var taskCompletionSource = new TaskCompletionSource(); 141 | var op = new CasCommand 142 | { 143 | Key = key, 144 | Value = jVal, 145 | PrevValue = jPrevVal, 146 | Completion = taskCompletionSource 147 | }; 148 | try 149 | { 150 | RaftEngine.AppendCommand(op); 151 | } 152 | catch (NotLeadingException e) 153 | { 154 | return RedirectToLeader(e.CurrentLeader, Request.RequestUri); 155 | } 156 | var newValueSet = await taskCompletionSource.Task; 157 | return Request.CreateResponse(HttpStatusCode.OK, new 158 | { 159 | RaftEngine.State, 160 | Key = key, 161 | ValueChanged = newValueSet 162 | }); 163 | } 164 | 165 | [HttpGet] 166 | [Route("tailfeather/key-val/del")] 167 | public Task Del([FromUri] string key) 168 | { 169 | var op = new KeyValueOperation 170 | { 171 | Key = key, 172 | Type = KeyValueOperationTypes.Del, 173 | }; 174 | 175 | return Batch(new[] { op }); 176 | } 177 | 178 | [HttpPost] 179 | [Route("tailfeather/key-val/batch")] 180 | public async Task Batch() 181 | { 182 | var stream = await Request.Content.ReadAsStreamAsync(); 183 | var operations = new JsonSerializer().Deserialize(new JsonTextReader(new StreamReader(stream))); 184 | 185 | return await Batch(operations); 186 | } 187 | 188 | private async Task Batch(KeyValueOperation[] operations) 189 | { 190 | var taskCompletionSource = new TaskCompletionSource(); 191 | try 192 | { 193 | RaftEngine.AppendCommand(new OperationBatchCommand 194 | { 195 | Batch = operations, 196 | Completion = taskCompletionSource 197 | }); 198 | } 199 | catch (NotLeadingException e) 200 | { 201 | return RedirectToLeader(e.CurrentLeader, Request.RequestUri); 202 | } 203 | await taskCompletionSource.Task; 204 | 205 | return Request.CreateResponse(HttpStatusCode.Accepted); 206 | } 207 | } 208 | } -------------------------------------------------------------------------------- /TailFeather/Controllers/TailFeatherController.cs: -------------------------------------------------------------------------------- 1 | using System.Net; 2 | using System.Net.Http; 3 | using System.Threading; 4 | using System.Threading.Tasks; 5 | using System.Web.Http; 6 | using System.Web.Http.Controllers; 7 | using Rachis; 8 | using Rachis.Utils; 9 | using TailFeather.Storage; 10 | 11 | namespace TailFeather.Controllers 12 | { 13 | public abstract class TailFeatherController : ApiController 14 | { 15 | public KeyValueStateMachine StateMachine { get; private set; } 16 | public RaftEngine RaftEngine { get; private set; } 17 | 18 | public override async Task ExecuteAsync(HttpControllerContext controllerContext, CancellationToken cancellationToken) 19 | { 20 | RaftEngine = (RaftEngine)controllerContext.Configuration.Properties[typeof(RaftEngine)]; 21 | StateMachine = (KeyValueStateMachine)RaftEngine.StateMachine; 22 | try 23 | { 24 | return await base.ExecuteAsync(controllerContext, cancellationToken); 25 | } 26 | catch (NotLeadingException) 27 | { 28 | var currentLeader = RaftEngine.CurrentLeader; 29 | if (currentLeader == null) 30 | { 31 | return Request.CreateErrorResponse(HttpStatusCode.PreconditionFailed, "No current leader, try again later"); 32 | } 33 | var leaderNode = RaftEngine.CurrentTopology.GetNodeByName(currentLeader); 34 | if (leaderNode == null) 35 | { 36 | return Request.CreateErrorResponse(HttpStatusCode.PreconditionFailed, "Current leader " + currentLeader + " is not found in the topology. This should not happen."); 37 | } 38 | return new HttpResponseMessage(HttpStatusCode.Redirect) 39 | { 40 | Headers = 41 | { 42 | Location = leaderNode.Uri 43 | } 44 | }; 45 | } 46 | } 47 | } 48 | } -------------------------------------------------------------------------------- /TailFeather/NLog.config: -------------------------------------------------------------------------------- 1 |  2 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /TailFeather/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Web.Http; 4 | using CommandLine; 5 | using CommandLine.Text; 6 | using Microsoft.Owin.Hosting; 7 | using Owin; 8 | using Rachis; 9 | using Rachis.Storage; 10 | using Rachis.Transport; 11 | using TailFeather.Storage; 12 | using Voron; 13 | 14 | namespace TailFeather 15 | { 16 | internal class Program 17 | { 18 | private static void Main(string[] args) 19 | { 20 | var options = new TailFeatherCommandLineOptions(); 21 | if (Parser.Default.ParseArguments(args, options) == false) 22 | { 23 | var autoBuild = HelpText.AutoBuild(options); 24 | HelpText.DefaultParsingErrorsHandler(options, autoBuild); 25 | Console.WriteLine(autoBuild.ToString()); 26 | return; 27 | } 28 | 29 | var nodeName = options.NodeName ?? (Environment.MachineName + ":" + options.Port); 30 | Console.Title = string.Format("Node name: {0}, port: {1}", nodeName, options.Port); 31 | 32 | var kvso = StorageEnvironmentOptions.ForPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, options.DataPath, "KeyValue")); 33 | using (var statemachine = new KeyValueStateMachine(kvso)) 34 | { 35 | var storageEnvironmentOptions = StorageEnvironmentOptions.ForPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, options.DataPath, "Raft")); 36 | var httpTransport = new HttpTransport(nodeName); 37 | var raftEngineOptions = new RaftEngineOptions( 38 | new NodeConnectionInfo 39 | { 40 | Name = nodeName, 41 | Uri = new Uri("http://" + Environment.MachineName + ":" + options.Port), 42 | }, 43 | storageEnvironmentOptions, 44 | httpTransport, 45 | statemachine 46 | ) 47 | { 48 | ElectionTimeout = 5 * 1000, 49 | HeartbeatTimeout = 1000, 50 | MaxLogLengthBeforeCompaction = 25 51 | }; 52 | 53 | if (options.Boostrap) 54 | { 55 | PersistentState.ClusterBootstrap(raftEngineOptions); 56 | Console.WriteLine("Setup node as the cluster seed, exiting..."); 57 | return; 58 | } 59 | 60 | using (var raftEngine = new RaftEngine(raftEngineOptions)) 61 | { 62 | using (WebApp.Start(new StartOptions 63 | { 64 | Urls = { "http://+:" + options.Port + "/" } 65 | }, builder => 66 | { 67 | var httpConfiguration = new HttpConfiguration(); 68 | httpConfiguration.Formatters.Remove(httpConfiguration.Formatters.XmlFormatter); 69 | httpConfiguration.Formatters.JsonFormatter.SerializerSettings.Formatting = Newtonsoft.Json.Formatting.Indented; 70 | httpConfiguration.Formatters.JsonFormatter.SerializerSettings.Converters.Add(new Newtonsoft.Json.Converters.StringEnumConverter()); 71 | RaftWebApiConfig.Load(); 72 | httpConfiguration.MapHttpAttributeRoutes(); 73 | httpConfiguration.Properties[typeof(HttpTransportBus)] = httpTransport.Bus; 74 | httpConfiguration.Properties[typeof(RaftEngine)] = raftEngine; 75 | builder.UseWebApi(httpConfiguration); 76 | })) 77 | { 78 | Console.WriteLine("Ready @ http://" + Environment.MachineName + ":" + options.Port + "/, press ENTER to stop"); 79 | 80 | Console.ReadLine(); 81 | } 82 | } 83 | } 84 | } 85 | } 86 | } -------------------------------------------------------------------------------- /TailFeather/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("TailFeather")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("TailFeather")] 13 | [assembly: AssemblyCopyright("Copyright © 2014")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("4637acc2-5c2e-4fa2-8f7e-094bf13bdd78")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /TailFeather/Storage/KeyValueOperation.cs: -------------------------------------------------------------------------------- 1 | using Newtonsoft.Json.Linq; 2 | 3 | namespace TailFeather.Storage 4 | { 5 | public class KeyValueOperation 6 | { 7 | public KeyValueOperationTypes Type; 8 | public string Key; 9 | public JToken Value; 10 | } 11 | } -------------------------------------------------------------------------------- /TailFeather/Storage/KeyValueOperationTypes.cs: -------------------------------------------------------------------------------- 1 | namespace TailFeather.Storage 2 | { 3 | public enum KeyValueOperationTypes 4 | { 5 | Add, 6 | Del 7 | } 8 | } -------------------------------------------------------------------------------- /TailFeather/Storage/OperationBatchCommand.cs: -------------------------------------------------------------------------------- 1 | using Newtonsoft.Json.Linq; 2 | using Rachis.Commands; 3 | 4 | namespace TailFeather.Storage 5 | { 6 | public class OperationBatchCommand : Command 7 | { 8 | public KeyValueOperation[] Batch { get; set; } 9 | } 10 | 11 | public class GetCommand : Command 12 | { 13 | public string Key { get; set; } 14 | } 15 | 16 | public class CasCommand : Command 17 | { 18 | public string Key { get; set; } 19 | public JToken Value; 20 | public JToken PrevValue; 21 | } 22 | } -------------------------------------------------------------------------------- /TailFeather/TailFeather.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {832A4C02-72AE-4F30-9691-E286457DD39D} 8 | Exe 9 | Properties 10 | TailFeather 11 | TailFeather 12 | 512 13 | ..\ 14 | true 15 | 16 | v4.5 17 | 18 | 19 | AnyCPU 20 | true 21 | full 22 | false 23 | bin\Debug\ 24 | DEBUG;TRACE 25 | prompt 26 | 4 27 | false 28 | 29 | 30 | AnyCPU 31 | pdbonly 32 | true 33 | bin\Release\ 34 | TRACE 35 | prompt 36 | 4 37 | false 38 | 39 | 40 | 41 | ..\packages\CommandLineParser.1.9.71\lib\net45\CommandLine.dll 42 | 43 | 44 | ..\packages\Microsoft.Owin.2.0.2\lib\net45\Microsoft.Owin.dll 45 | 46 | 47 | ..\packages\Microsoft.Owin.Host.HttpListener.2.0.2\lib\net45\Microsoft.Owin.Host.HttpListener.dll 48 | 49 | 50 | ..\packages\Microsoft.Owin.Hosting.2.0.2\lib\net45\Microsoft.Owin.Hosting.dll 51 | 52 | 53 | ..\packages\Newtonsoft.Json.6.0.4\lib\net45\Newtonsoft.Json.dll 54 | monodevelop 55 | 56 | 57 | ..\packages\Owin.1.0\lib\net40\Owin.dll 58 | 59 | 60 | 61 | 62 | 63 | False 64 | ..\packages\Microsoft.AspNet.WebApi.Client.5.2.2\lib\net45\System.Net.Http.Formatting.dll 65 | 66 | 67 | ..\packages\Microsoft.AspNet.WebApi.Core.5.2.2\lib\net45\System.Web.Http.dll 68 | 69 | 70 | ..\packages\Microsoft.AspNet.WebApi.Owin.5.2.2\lib\net45\System.Web.Http.Owin.dll 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | PreserveNewest 94 | Designer 95 | 96 | 97 | 98 | 99 | 100 | {FF83C7C2-BC7B-4DCC-A782-49EF9BBD9390} 101 | Voron 102 | 103 | 104 | {F796F69F-D17B-4260-92D6-65CB94C0E05C} 105 | Rachis 106 | 107 | 108 | 109 | 110 | 111 | 112 | This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. 113 | 114 | 115 | 116 | 123 | -------------------------------------------------------------------------------- /TailFeather/TailFeatherCommandLineOptions.cs: -------------------------------------------------------------------------------- 1 | using CommandLine; 2 | 3 | namespace TailFeather 4 | { 5 | public class TailFeatherCommandLineOptions 6 | { 7 | [Option('p', "port", Required = true, HelpText = "The http port to use")] 8 | public int Port { get; set; } 9 | 10 | [Option("bootstrap", HelpText = "Setup this node as the seed for first time cluster bootstrap")] 11 | public bool Boostrap { get; set; } 12 | 13 | [Option('d',"DataPath", HelpText = "Path for the node to use for persistent data", Required = true)] 14 | public string DataPath { get; set; } 15 | 16 | [Option('n', "Name", HelpText = "The friendly name of the node")] 17 | public string NodeName { get; set; } 18 | } 19 | } -------------------------------------------------------------------------------- /TailFeather/app.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /TailFeather/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /Tryouts/App.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /Tryouts/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Threading.Tasks; 5 | using Newtonsoft.Json; 6 | using Newtonsoft.Json.Linq; 7 | using TailFeather.Client; 8 | using Rachis.Tests; 9 | 10 | namespace Tryouts 11 | { 12 | class Program 13 | { 14 | static void Main() 15 | { 16 | for (int i = 0; i < 1000; i++) { 17 | using (var test = new TopologyChangesTests ()) { 18 | Console.WriteLine (i); 19 | test.New_node_can_be_added_even_if_it_is_down (); 20 | } 21 | } 22 | //var tailFeatherClient = new TailFeatherClient(new Uri("http://localhost:9078")); 23 | //int i =0; 24 | //while (true) 25 | //{ 26 | //tailFeatherClient.Set("now-"+i, DateTime.Now); 27 | //Console.WriteLine(i++); 28 | //Console.ReadKey(); 29 | //} 30 | } 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /Tryouts/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("Tryouts")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("Tryouts")] 13 | [assembly: AssemblyCopyright("Copyright © 2014")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("68abbe59-22ee-450f-9659-dfd04bd83ea6")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /Tryouts/Tryouts.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {54412F82-A711-4AF6-931B-D8AF808E57C5} 8 | Exe 9 | Properties 10 | Tryouts 11 | Tryouts 12 | 512 13 | 14 | ..\ 15 | true 16 | v4.5.1 17 | 18 | 19 | AnyCPU 20 | true 21 | full 22 | false 23 | bin\Debug\ 24 | DEBUG;TRACE 25 | prompt 26 | 4 27 | false 28 | 29 | 30 | AnyCPU 31 | pdbonly 32 | true 33 | bin\Release\ 34 | TRACE 35 | prompt 36 | 4 37 | 38 | 39 | 40 | ..\packages\Microsoft.Owin.2.0.2\lib\net45\Microsoft.Owin.dll 41 | 42 | 43 | ..\packages\Microsoft.Owin.Host.HttpListener.2.0.2\lib\net45\Microsoft.Owin.Host.HttpListener.dll 44 | 45 | 46 | False 47 | ..\packages\Microsoft.Owin.Hosting.2.0.2\lib\net45\Microsoft.Owin.Hosting.dll 48 | 49 | 50 | False 51 | ..\packages\Newtonsoft.Json.6.0.4\lib\net45\Newtonsoft.Json.dll 52 | 53 | 54 | ..\packages\Owin.1.0\lib\net40\Owin.dll 55 | 56 | 57 | 58 | 59 | 60 | False 61 | ..\packages\Microsoft.AspNet.WebApi.Client.5.2.2\lib\net45\System.Net.Http.Formatting.dll 62 | 63 | 64 | False 65 | ..\packages\Microsoft.AspNet.WebApi.Core.5.2.2\lib\net45\System.Web.Http.dll 66 | 67 | 68 | ..\packages\Microsoft.AspNet.WebApi.Owin.5.2.2\lib\net45\System.Web.Http.Owin.dll 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | ..\packages\xunit.1.9.2\lib\net20\xunit.dll 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | {FF83C7C2-BC7B-4DCC-A782-49EF9BBD9390} 90 | Voron 91 | 92 | 93 | {61B3D01A-F286-42B1-AA6B-83D9D6DF0873} 94 | Rachis.Tests 95 | 96 | 97 | {F796F69F-D17B-4260-92D6-65CB94C0E05C} 98 | Rachis 99 | 100 | 101 | {1E267CF9-EAE8-4F87-BE37-BB1E7C9E86D6} 102 | TailFeather.Client 103 | 104 | 105 | 106 | 107 | 108 | 109 | This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. 110 | 111 | 112 | 113 | 120 | -------------------------------------------------------------------------------- /Tryouts/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /license.txt: -------------------------------------------------------------------------------- 1 | Raven DB - A Document Database for the .NET platfrom 2 | Copyright (C) 2010 Hibernating rhinos 3 | =================================================================== 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU Affero General Public License as 7 | published by the Free Software Foundation, either version 3 of the 8 | License. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU Affero General Public License for more details. 14 | 15 | You should have received a copy of the GNU Affero General Public License 16 | along with this program. If not, see http://www.gnu.org/licenses/agpl-3.0.html. 17 | 18 | =================================================================== 19 | 20 | FOSS License Exception 21 | =================================================================== 22 | 23 | What is the FOSS License Exception? 24 | 25 | This Free and Open Source Software ("FOSS") License Exception allows 26 | developers of FOSS applications to include RavenDB with their FOSS 27 | applications. RavenDB is typically licensed pursuant to version 3 28 | of the Affero General Public License ("AGPL"), but this exception permits 29 | distribution of RavenDB with a developer's FOSS applications licensed 30 | under the terms of another FOSS license listed below, even though such 31 | other FOSS license may be incompatible with the AGPL. 32 | 33 | The following terms and conditions describe the circumstances under which 34 | this FOSS License Exception applies. 35 | 36 | FOSS License Exception Terms and Conditions 37 | 38 | Definitions. 39 | * "Derivative Work" means a derivative work, as defined under applicable 40 | copyright law, formed entirely from the Program and one or more FOSS Applications. 41 | * "FOSS Application" means a free and open source software application distributed 42 | subject to a license approved by the Open Source Initiative (OSI) board. A list of 43 | applicable licenses appears at: http://www.opensource.org/licenses/category. 44 | * "FOSS Notice" means a notice placed by Hibernating Rhinos or the RavenDB author 45 | in a copy of the RavenDB library stating that such copy of the RavenDB library 46 | may be distributed under Hibernating Rhinos's or RavenDB's FOSS License Exception. 47 | * "Independent Work" means portions of the Derivative Work that are not derived 48 | from the Program and can reasonably be considered independent and separate works. 49 | * "Program" means a copy of Hibernating Rhinos's RavenDB library that contains a FOSS 50 | Notice. 51 | 52 | A FOSS application developer ("you" or "your") may distribute a Derivative Work provided 53 | that you and the Derivative Work meet all of the following conditions: 54 | 55 | * You obey the AGPL in all respects for the Program and all portions (including 56 | modifications) of the Program included in the Derivative Work (provided that this 57 | condition does not apply to Independent Works); 58 | * The Derivative Work does not include any work licensed under the AGPL other than 59 | the Program; 60 | * You distribute Independent Works subject to a license approved by the OSI which is 61 | listed in http://www.opensource.org/licenses/category; 62 | * You distribute Independent Works in object code or executable form with the complete 63 | corresponding machine-readable source code on the same medium and under the same 64 | FOSS license applying to the object code or executable forms; 65 | * All works that are aggregated with the Program or the Derivative Work on a medium or 66 | volume of storage are not derivative works of the Program, Derivative Work or FOSS 67 | Application, and must reasonably be considered independent and separate works. 68 | 69 | Hibernating Rhinos reserves all rights not expressly granted in these terms and conditions. 70 | If all of the above conditions are not met, then this FOSS License Exception does not 71 | apply to you or your Derivative Work. 72 | 73 | =================================================================== 74 | 75 | Commercial Licensing 76 | =================================================================== 77 | In addition to this license, RavenDB is offered under a commerical license. 78 | You can learn more about this option by contacting us using: 79 | http://hibernatingrhinos.com/contact 80 | --------------------------------------------------------------------------------