├── .gitignore ├── LICENSE.md ├── NSQCore.sln ├── NSQCore ├── Addresses.cs ├── BackoffStrategies.cs ├── Commands │ ├── ByteArrays.cs │ ├── Finish.cs │ ├── ICommand.cs │ ├── Identify.cs │ ├── Nop.cs │ ├── Publish.cs │ ├── Ready.cs │ ├── Requeue.cs │ └── Subscribe.cs ├── CommunicationException.cs ├── ConsumerOptions.cs ├── DiscoveryEventArgs.cs ├── Frame.cs ├── FrameReader.cs ├── HttpClientWrapper.cs ├── InternalMessageEventArgs.cs ├── Message.cs ├── MessageBody.cs ├── NSQCore.csproj ├── NsqConsumer.cs ├── NsqLookup.cs ├── NsqLookupConsumer.cs ├── NsqProducer.cs ├── NsqStatistics.cs ├── NsqTcpConnection.cs ├── Properties │ └── PublishProfiles │ │ └── FolderProfile.pubxml ├── System │ └── Disposable.cs └── TopicAndChannel.cs ├── NSQCoreClient ├── NSQCoreClient.csproj └── Program.cs └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | bin 3 | obj 4 | csx 5 | .vs 6 | edge 7 | Publish 8 | .vscode 9 | 10 | *.user 11 | *.suo 12 | *.cscfg 13 | *.Cache 14 | project.lock.json 15 | 16 | /packages 17 | /TestResults 18 | 19 | /tools/NuGet.exe 20 | /App_Data 21 | /secrets 22 | /data 23 | .secrets 24 | Binaries 25 | Output 26 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Webjet/NSQCore/77954242e3df6fc504d68e8ce445cf0eea3fd5e2/LICENSE.md -------------------------------------------------------------------------------- /NSQCore.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.26606.0 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NSQCore", "NSQCore\NSQCore.csproj", "{A555696C-42D4-42C2-9B96-B73EBB7C61B7}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NSQCoreClient", "NSQCoreClient\NSQCoreClient.csproj", "{9E219E99-B620-4552-8E8E-0244C1861A41}" 9 | EndProject 10 | Global 11 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 12 | Debug|Any CPU = Debug|Any CPU 13 | Release|Any CPU = Release|Any CPU 14 | EndGlobalSection 15 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 16 | {A555696C-42D4-42C2-9B96-B73EBB7C61B7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 17 | {A555696C-42D4-42C2-9B96-B73EBB7C61B7}.Debug|Any CPU.Build.0 = Debug|Any CPU 18 | {A555696C-42D4-42C2-9B96-B73EBB7C61B7}.Release|Any CPU.ActiveCfg = Release|Any CPU 19 | {A555696C-42D4-42C2-9B96-B73EBB7C61B7}.Release|Any CPU.Build.0 = Release|Any CPU 20 | {9E219E99-B620-4552-8E8E-0244C1861A41}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 21 | {9E219E99-B620-4552-8E8E-0244C1861A41}.Debug|Any CPU.Build.0 = Debug|Any CPU 22 | {9E219E99-B620-4552-8E8E-0244C1861A41}.Release|Any CPU.ActiveCfg = Release|Any CPU 23 | {9E219E99-B620-4552-8E8E-0244C1861A41}.Release|Any CPU.Build.0 = Release|Any CPU 24 | EndGlobalSection 25 | GlobalSection(SolutionProperties) = preSolution 26 | HideSolutionNode = FALSE 27 | EndGlobalSection 28 | EndGlobal 29 | -------------------------------------------------------------------------------- /NSQCore/Addresses.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace NSQCore 4 | { 5 | /// 6 | /// The address of an nsqlookupd instance. 7 | /// 8 | public struct LookupAddress : IEquatable 9 | { 10 | public string HostName { get; } 11 | public int HttpPort { get; } 12 | 13 | public LookupAddress(string hostName, int httpPort) 14 | : this() 15 | { 16 | HostName = hostName; 17 | HttpPort = httpPort; 18 | } 19 | 20 | public override string ToString() 21 | { 22 | return "HostName = " + HostName + ", HttpPort = " + HttpPort; 23 | } 24 | 25 | public override bool Equals(object obj) 26 | { 27 | if (obj is LookupAddress) 28 | return Equals((LookupAddress)obj); 29 | 30 | return false; 31 | } 32 | 33 | public override int GetHashCode() 34 | { 35 | // See: http://stackoverflow.com/a/263416/19818 36 | const int BASE = 151; 37 | const int mixer = 2011; 38 | unchecked // Overflow is fine 39 | { 40 | int hash = BASE; 41 | hash = hash * mixer + HttpPort.GetHashCode(); 42 | if (HostName != null) 43 | hash = hash * mixer + HostName.GetHashCode(); 44 | 45 | return hash; 46 | } 47 | } 48 | 49 | /// 50 | /// The address of an nsqd instance. 51 | /// 52 | public bool Equals(LookupAddress other) 53 | { 54 | return HttpPort == other.HttpPort 55 | && string.Equals(HostName, other.HostName, StringComparison.CurrentCultureIgnoreCase); 56 | } 57 | } 58 | 59 | /// 60 | /// The address of an nsqd instance. 61 | /// 62 | public struct NsqAddress : IEquatable 63 | { 64 | public string BroadcastAddress { get; } 65 | public string HostName { get; } 66 | public int HttpPort { get; } 67 | public int TcpPort { get; } 68 | 69 | public NsqAddress(string broadcastAddress, string hostName, int tcpPort, int httpPort) 70 | : this() 71 | { 72 | BroadcastAddress = broadcastAddress; 73 | HostName = hostName; 74 | TcpPort = tcpPort; 75 | HttpPort = httpPort; 76 | } 77 | 78 | public override string ToString() 79 | { 80 | return "BroadcastAddress = " + BroadcastAddress + ", TcpPort = " + TcpPort; 81 | } 82 | 83 | public override int GetHashCode() 84 | { 85 | // See: http://stackoverflow.com/a/263416/19818 86 | const int BASE = 151; 87 | const int mixer = 2011; 88 | unchecked // Overflow is fine 89 | { 90 | int hash = BASE; 91 | 92 | hash = hash * mixer + HttpPort.GetHashCode(); 93 | hash = hash * mixer + TcpPort.GetHashCode(); 94 | 95 | if (BroadcastAddress != null) 96 | hash = hash * mixer + BroadcastAddress.GetHashCode(); 97 | 98 | if (HostName != null) 99 | hash = hash * mixer + HostName.GetHashCode(); 100 | 101 | return hash; 102 | } 103 | } 104 | 105 | public override bool Equals(object obj) 106 | { 107 | if (obj is NsqAddress) 108 | return Equals((NsqAddress)obj); 109 | 110 | return false; 111 | } 112 | 113 | public bool Equals(NsqAddress other) 114 | { 115 | return HttpPort == other.HttpPort 116 | && TcpPort == other.TcpPort 117 | && string.Equals(BroadcastAddress, other.BroadcastAddress, StringComparison.CurrentCultureIgnoreCase) 118 | && string.Equals(HostName, other.HostName, StringComparison.CurrentCultureIgnoreCase); 119 | } 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /NSQCore/BackoffStrategies.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace NSQCore 4 | { 5 | public interface IBackoffStrategy 6 | { 7 | IBackoffLimiter Create(); 8 | } 9 | 10 | public interface IBackoffLimiter 11 | { 12 | bool ShouldReconnect(out TimeSpan delay); 13 | } 14 | 15 | /// 16 | /// Delays a constant amount of time between reconnections. 17 | /// 18 | public class FixedDelayBackoffStrategy : IBackoffStrategy 19 | { 20 | private readonly TimeSpan _delay; 21 | public FixedDelayBackoffStrategy(TimeSpan delay) 22 | { 23 | _delay = delay; 24 | } 25 | 26 | public IBackoffLimiter Create() 27 | { 28 | return new FixedDelayBackoffLimiter(_delay); 29 | } 30 | 31 | private class FixedDelayBackoffLimiter : IBackoffLimiter 32 | { 33 | private readonly TimeSpan _delay; 34 | public FixedDelayBackoffLimiter(TimeSpan delay) 35 | { 36 | _delay = delay; 37 | } 38 | 39 | public bool ShouldReconnect(out TimeSpan delay) 40 | { 41 | delay = _delay; 42 | return true; 43 | } 44 | } 45 | } 46 | 47 | /// 48 | /// Delays reconnection with an expontential back-off. For example, 49 | /// repeated attempts will delay 1 second, 2 seconds, 4 seconds, etc. 50 | /// 51 | public class ExponentialBackoffStrategy : IBackoffStrategy 52 | { 53 | private readonly TimeSpan _initialDelay; 54 | private readonly TimeSpan _maxDelay; 55 | public ExponentialBackoffStrategy(TimeSpan initialDelay, TimeSpan maxDelay) 56 | { 57 | _initialDelay = initialDelay; 58 | _maxDelay = maxDelay; 59 | } 60 | public IBackoffLimiter Create() 61 | { 62 | return new ExponentialBackoffLimiter(_initialDelay, _maxDelay); 63 | } 64 | 65 | private class ExponentialBackoffLimiter : IBackoffLimiter 66 | { 67 | private TimeSpan _currentDelay; 68 | private readonly TimeSpan _maxDelay; 69 | public ExponentialBackoffLimiter(TimeSpan initialDelay, TimeSpan maxDelay) 70 | { 71 | _currentDelay = initialDelay; 72 | _maxDelay = maxDelay; 73 | } 74 | 75 | public bool ShouldReconnect(out TimeSpan delay) 76 | { 77 | delay = _currentDelay; 78 | var nextDelay = _currentDelay.Add(_currentDelay); 79 | _currentDelay = nextDelay < _maxDelay ? nextDelay : _maxDelay; 80 | return true; 81 | } 82 | } 83 | } 84 | 85 | /// 86 | /// Never retries reconnecting. A connection with this back-off strategy 87 | /// will simply die when disconnected. 88 | /// 89 | public class NoRetryBackoffStrategy : IBackoffStrategy 90 | { 91 | public IBackoffLimiter Create() 92 | { 93 | return new NoRetryBackoffLimiter(); 94 | } 95 | 96 | private class NoRetryBackoffLimiter : IBackoffLimiter 97 | { 98 | public bool ShouldReconnect(out TimeSpan delay) 99 | { 100 | delay = TimeSpan.Zero; 101 | return false; 102 | } 103 | } 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /NSQCore/Commands/ByteArrays.cs: -------------------------------------------------------------------------------- 1 | using System.Text; 2 | 3 | namespace NSQCore.Commands 4 | { 5 | internal static class ByteArrays 6 | { 7 | public static readonly byte[] Lf = Encoding.ASCII.GetBytes("\n"); 8 | public static readonly byte[] Space = Encoding.ASCII.GetBytes(" "); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /NSQCore/Commands/Finish.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using System.Text; 3 | 4 | namespace NSQCore.Commands 5 | { 6 | internal class Finish : ICommand 7 | { 8 | private static readonly byte[] FinSpace = Encoding.ASCII.GetBytes("FIN "); 9 | 10 | private readonly Message _message; 11 | 12 | public Finish(Message message) 13 | { 14 | _message = message; 15 | } 16 | 17 | public byte[] ToByteArray() 18 | { 19 | return FinSpace 20 | .Concat(Encoding.UTF8.GetBytes(_message.Id)) 21 | .Concat(ByteArrays.Lf) 22 | .ToArray(); 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /NSQCore/Commands/ICommand.cs: -------------------------------------------------------------------------------- 1 | namespace NSQCore.Commands 2 | { 3 | internal interface ICommand 4 | { 5 | byte[] ToByteArray(); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /NSQCore/Commands/Identify.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Linq; 4 | using System.Text; 5 | using Newtonsoft.Json; 6 | 7 | namespace NSQCore.Commands 8 | { 9 | internal class Identify : ICommand 10 | { 11 | private readonly JsonSerializer _serializer = new JsonSerializer(); 12 | private readonly IdentifyRequest _identifyRequest; 13 | 14 | public Identify(ConsumerOptions options) 15 | { 16 | _identifyRequest = new IdentifyRequest 17 | { 18 | client_id = options.ClientId, 19 | hostname = options.HostName, 20 | feature_negotiation = true, 21 | tls_v1 = false, 22 | snappy = false, 23 | deflate = false, 24 | msg_timeout = options.MessageTimeout 25 | }; 26 | } 27 | 28 | private static readonly byte[] IdentifyLf = Encoding.ASCII.GetBytes("IDENTIFY\n"); 29 | 30 | public byte[] ToByteArray() 31 | { 32 | byte[] body; 33 | using (var stream = new MemoryStream(1024)) 34 | using (var writer = new StreamWriter(stream)) 35 | using (var jsonWriter = new JsonTextWriter(writer)) 36 | { 37 | _serializer.Serialize(jsonWriter, _identifyRequest); 38 | jsonWriter.Flush(); 39 | writer.Flush(); 40 | body = stream.ToArray(); 41 | } 42 | 43 | byte[] length = BitConverter.GetBytes(body.Length); 44 | if (BitConverter.IsLittleEndian) 45 | Array.Reverse(length); 46 | 47 | return IdentifyLf 48 | .Concat(length) 49 | .Concat(body) 50 | .ToArray(); 51 | } 52 | 53 | public IdentifyResponse ParseIdentifyResponse(byte[] data) 54 | { 55 | using (var stream = new MemoryStream(data)) 56 | using (var reader = new StreamReader(stream, Encoding.UTF8)) 57 | using (var jsonReader = new JsonTextReader(reader)) 58 | { 59 | return _serializer.Deserialize(jsonReader); 60 | } 61 | } 62 | 63 | private class IdentifyRequest 64 | { 65 | public string client_id { get; set; } 66 | public string hostname { get; set; } 67 | public bool feature_negotiation { get; set; } 68 | 69 | [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] 70 | public int? heartbeat_interval { get; set; } 71 | 72 | [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] 73 | public int? output_buffer_size { get; set; } 74 | 75 | [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] 76 | public int? output_buffer_timeout { get; set; } 77 | 78 | public bool tls_v1 { get; set; } 79 | public bool snappy { get; set; } 80 | public bool deflate { get; set; } 81 | public int deflate_level { get; set; } 82 | 83 | public int sample_rate { get; set; } 84 | [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] 85 | public int? msg_timeout { get; set; } 86 | public string user_agent { get { return "NSQCore/1.0"; } } 87 | } 88 | } 89 | 90 | internal class IdentifyResponse 91 | { 92 | [JsonProperty("max_rdy_count")] 93 | public long MaxReadyCount { get; set; } 94 | 95 | [JsonProperty("version")] 96 | public string Version { get; set; } 97 | 98 | [JsonProperty("max_msg_timeout")] 99 | public long MaxMessageTimeoutMilliseconds { get; set; } 100 | 101 | [JsonProperty("msg_timeout")] 102 | public long MessageTimeoutMilliseconds { get; set; } 103 | 104 | [JsonProperty("tls_v1")] 105 | public bool Tls { get; set; } 106 | 107 | [JsonProperty("deflate")] 108 | public bool Deflate { get; set; } 109 | 110 | [JsonProperty("deflate_level")] 111 | public int DeflateLevel { get; set; } 112 | 113 | [JsonProperty("max_deflate_level")] 114 | public int MaxDeflateLevel { get; set; } 115 | 116 | [JsonProperty("snappy")] 117 | public bool Snappy { get; set; } 118 | 119 | [JsonProperty("sample_rate")] 120 | public int SampleRate { get; set; } 121 | 122 | [JsonProperty("auth_required")] 123 | public bool AuthRequired { get; set; } 124 | 125 | [JsonProperty("output_buffer_size")] 126 | public int OutputBufferSize { get; set; } 127 | 128 | [JsonProperty("output_buffer_timeout")] 129 | public long OutputBufferTimeout { get; set; } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /NSQCore/Commands/Nop.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using System.Text; 3 | 4 | namespace NSQCore.Commands 5 | { 6 | internal class Nop : ICommand 7 | { 8 | private static readonly byte[] NopLf = Encoding.ASCII.GetBytes("NOP\n"); 9 | 10 | public byte[] ToByteArray() 11 | { 12 | return NopLf.ToArray(); // Make a new one 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /NSQCore/Commands/Publish.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Text; 4 | 5 | namespace NSQCore.Commands 6 | { 7 | internal class Publish : ICommand 8 | { 9 | private static readonly byte[] Prefix = Encoding.ASCII.GetBytes("PUB "); 10 | 11 | private readonly Topic _topic; 12 | private readonly MessageBody _message; 13 | 14 | 15 | public Publish(Topic topic, MessageBody message) 16 | { 17 | _message = message; 18 | _topic = topic; 19 | } 20 | 21 | public byte[] ToByteArray() 22 | { 23 | byte[] messageBody = _message; 24 | byte[] size = BitConverter.GetBytes(messageBody.Length); 25 | if (BitConverter.IsLittleEndian) 26 | Array.Reverse(size); 27 | 28 | return Prefix 29 | .Concat(_topic.ToUtf8()) 30 | .Concat(ByteArrays.Lf) 31 | .Concat(size) 32 | .Concat(messageBody) 33 | .ToArray(); 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /NSQCore/Commands/Ready.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using System.Text; 3 | 4 | namespace NSQCore.Commands 5 | { 6 | internal class Ready : ICommand 7 | { 8 | private readonly int _count; 9 | 10 | public Ready(int count) 11 | { 12 | _count = count; 13 | } 14 | 15 | private static readonly byte[] RdySpace = Encoding.ASCII.GetBytes("RDY "); 16 | 17 | public byte[] ToByteArray() 18 | { 19 | return RdySpace 20 | .Concat(Encoding.UTF8.GetBytes(_count.ToString())) 21 | .Concat(ByteArrays.Lf) 22 | .ToArray(); 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /NSQCore/Commands/Requeue.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | 6 | namespace NSQCore.Commands 7 | { 8 | internal class Requeue : ICommand 9 | { 10 | private static readonly byte[] Prefix = Encoding.ASCII.GetBytes("REQ "); 11 | 12 | private readonly string _messageId; 13 | private readonly int _maxTimeout; 14 | 15 | public Requeue(string id, int maxTimeout = 0) 16 | { 17 | _messageId = string.Format("{0} ", id); 18 | _maxTimeout = maxTimeout; 19 | } 20 | 21 | public byte[] ToByteArray() 22 | { 23 | return Prefix 24 | .Concat(Encoding.ASCII.GetBytes(_messageId)) 25 | .Concat(Encoding.ASCII.GetBytes(_maxTimeout.ToString())) 26 | .Concat(ByteArrays.Lf) 27 | .ToArray(); 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /NSQCore/Commands/Subscribe.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using System.Text; 3 | 4 | namespace NSQCore.Commands 5 | { 6 | internal class Subscribe : ICommand 7 | { 8 | private readonly Topic _topic; 9 | private readonly Channel _channel; 10 | 11 | public Subscribe(Topic topic, Channel channel) 12 | { 13 | _topic = topic; 14 | _channel = channel; 15 | } 16 | 17 | private static readonly byte[] SubSpace = Encoding.ASCII.GetBytes("SUB "); 18 | 19 | public byte[] ToByteArray() 20 | { 21 | return SubSpace 22 | .Concat(_topic.ToUtf8()) 23 | .Concat(ByteArrays.Space) 24 | .Concat(_channel.ToUtf8()) 25 | .Concat(ByteArrays.Lf) 26 | .ToArray(); 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /NSQCore/CommunicationException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace NSQCore 4 | { 5 | public class CommunicationException : Exception 6 | { 7 | public CommunicationException(string message) 8 | : base(message) 9 | { 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /NSQCore/ConsumerOptions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Net; 5 | using System.Text; 6 | 7 | namespace NSQCore 8 | { 9 | /// 10 | /// Configures the behavior of an NSQ consumer connection. 11 | /// 12 | public class ConsumerOptions 13 | { 14 | /// 15 | /// EndPoints for nsqlookupd instances to use. If any are present, 16 | /// this overrides the NsqEndPoint property. 17 | /// 18 | public HashSet LookupEndPoints { get; } 19 | 20 | /// 21 | /// The EndPoint to a single nsqd service to use. If any Lookup endpoints 22 | /// are present, this setting is ignored. 23 | /// 24 | public DnsEndPoint NsqEndPoint { get; set; } 25 | 26 | /// 27 | /// The topic to which to subscribe. 28 | /// 29 | public Topic Topic { get; set; } 30 | 31 | /// 32 | /// The channel to which to subscribe. 33 | /// 34 | public Channel Channel { get; set; } 35 | 36 | 37 | /// 38 | /// An identifier for this particular consumer. 39 | /// 40 | public string ClientId { get; set; } 41 | 42 | /// 43 | /// The hostname of the computer connected to NSQ. 44 | /// 45 | public string HostName { get; set; } 46 | 47 | public TimeSpan LookupPeriod { get; set; } 48 | 49 | /// 50 | /// The server-side message timeout in milliseconds for messages delivered 51 | /// to this client. 52 | /// 53 | public int? MessageTimeout { get; set; } 54 | 55 | /// 56 | /// The initial delay before attempting reconnection if the connection to NSQ fails. 57 | /// By default, the delay will be doubled on each attempt until reconnection, up to 58 | /// a maximum of ReconnectionMaxDelay. 59 | /// 60 | public TimeSpan ReconnectionDelay { get; set; } 61 | 62 | 63 | /// 64 | /// The maximum delay between reconnection attempts. 65 | /// 66 | public TimeSpan ReconnectionMaxDelay { get; set; } 67 | 68 | private const string LookupdKey = "lookupd"; 69 | private const string NsqdKey = "nsqd"; 70 | private const string TopicKey = "topic"; 71 | private const string ChannelKey = "channel"; 72 | private const string ClientidKey = "clientid"; 73 | private const string HostnameKey = "hostname"; 74 | private const string LookupperiodKey = "lookupperiod"; 75 | private const string ReconnectiondelayKey = "reconnectiondelay"; 76 | private const string ReconnectionmaxdelayKey = "reconnectionmaxdelay"; 77 | private const string MessagetimeoutKey = "messagetimeout"; 78 | 79 | private const int DefaultLookupdHttpPort = 4061; 80 | private const int DefaultNsqdTcpPort = 4050; 81 | 82 | /// 83 | /// Creates a default set of options. 84 | /// 85 | public ConsumerOptions() 86 | { 87 | LookupEndPoints = new HashSet(); 88 | 89 | ClientId = "NSQCore"; 90 | HostName = "localhost";// Dns.GetHostName(); 91 | LookupPeriod = TimeSpan.FromSeconds(15); 92 | ReconnectionDelay = TimeSpan.FromSeconds(1); 93 | ReconnectionMaxDelay = TimeSpan.FromSeconds(30); 94 | } 95 | 96 | /// 97 | /// Parses a connection string into a ConsumerOptions instance. 98 | /// 99 | /// A semicolon-delimited list of key=value pairs of connection options. 100 | /// 101 | public static ConsumerOptions Parse(string connectionString) 102 | { 103 | var options = new ConsumerOptions(); 104 | var parts = ParseIntoSegments(connectionString); 105 | 106 | if (parts.Contains(LookupdKey)) 107 | { 108 | foreach (var endPoint in ParseEndPoints(parts[LookupdKey], DefaultLookupdHttpPort)) 109 | { 110 | options.LookupEndPoints.Add(endPoint); 111 | } 112 | 113 | } 114 | else if (parts.Contains(NsqdKey)) 115 | { 116 | options.NsqEndPoint = ParseEndPoints(parts[NsqdKey], DefaultNsqdTcpPort).Last(); 117 | } 118 | else 119 | { 120 | throw new ArgumentException("Must provide either nsqlookupd or nsqd endpoints"); 121 | } 122 | 123 | if (parts.Contains(ClientidKey)) 124 | { 125 | options.ClientId = parts[ClientidKey].Last(); 126 | } 127 | 128 | if (parts.Contains(HostnameKey)) 129 | { 130 | options.HostName = parts[HostnameKey].Last(); 131 | } 132 | 133 | if (parts.Contains(LookupperiodKey)) 134 | { 135 | options.LookupPeriod = TimeSpan.FromSeconds(int.Parse(parts[LookupperiodKey].Last())); 136 | } 137 | 138 | if (parts.Contains(TopicKey)) 139 | { 140 | options.Topic = parts[TopicKey].Last(); 141 | } 142 | 143 | if (parts.Contains(ChannelKey)) 144 | { 145 | options.Channel = parts[ChannelKey].Last(); 146 | } 147 | 148 | if (parts.Contains(ReconnectiondelayKey)) 149 | { 150 | options.ReconnectionDelay = TimeSpan.FromSeconds(int.Parse(parts[ReconnectiondelayKey].Last())); 151 | } 152 | 153 | if (parts.Contains(ReconnectionmaxdelayKey)) 154 | { 155 | options.ReconnectionMaxDelay = TimeSpan.FromSeconds(int.Parse(parts[ReconnectionmaxdelayKey].Last())); 156 | } 157 | 158 | if (parts.Contains(MessagetimeoutKey)) 159 | { 160 | options.MessageTimeout = int.Parse(parts[MessagetimeoutKey].Last()); 161 | } 162 | 163 | return options; 164 | } 165 | 166 | private static ILookup ParseIntoSegments(string connectionString) 167 | { 168 | return 169 | connectionString.Split(new[] { ";" }, StringSplitOptions.RemoveEmptyEntries) 170 | .Select(part => part.Split(new[] { "=" }, StringSplitOptions.RemoveEmptyEntries)) 171 | .Where(part => part.Length == 1 || part.Length == 2) 172 | .Select(part => 173 | { 174 | if (part.Length == 2) 175 | return part; 176 | return new[] { LookupdKey, part[0] }; 177 | }) 178 | .ToLookup( 179 | part => part[0].ToLowerInvariant().Trim(), 180 | part => part[1].Trim()); 181 | 182 | } 183 | 184 | private static IEnumerable ParseEndPoints(IEnumerable list, int defaultPort) 185 | { 186 | return list 187 | .Select(endpoint => endpoint.Trim()) 188 | .Select(endpoint => endpoint.Split(new[] { ':' }, 2)) 189 | .Select(endpointParts => new DnsEndPoint(endpointParts[0], endpointParts.Length == 2 ? int.Parse(endpointParts[1]) : defaultPort)); 190 | } 191 | } 192 | } 193 | -------------------------------------------------------------------------------- /NSQCore/DiscoveryEventArgs.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace NSQCore 5 | { 6 | public class DiscoveryEventArgs : EventArgs 7 | { 8 | public List NsqAddresses { get; } 9 | 10 | public DiscoveryEventArgs(List addresses) 11 | { 12 | NsqAddresses = addresses; 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /NSQCore/Frame.cs: -------------------------------------------------------------------------------- 1 | using System.Text; 2 | 3 | namespace NSQCore 4 | { 5 | internal enum FrameType 6 | { 7 | Result, 8 | Error, 9 | Message 10 | } 11 | 12 | /// 13 | /// A frame of data received from nsqd. 14 | /// 15 | internal class Frame 16 | { 17 | public FrameType Type { get; set; } 18 | public int MessageSize { get; set; } 19 | 20 | public byte[] Data { get; set; } 21 | 22 | public string GetReadableData() 23 | { 24 | if (Data == null) return "(null)"; 25 | return Encoding.ASCII.GetString(Data, 0, Data.Length); 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /NSQCore/FrameReader.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Net.Sockets; 3 | 4 | namespace NSQCore 5 | { 6 | internal class FrameReader 7 | { 8 | private const int FrameSizeLength = 4; 9 | private const int FrameTypeLength = 4; 10 | 11 | private readonly NetworkStream _stream; 12 | 13 | private readonly object _lock = new object(); 14 | private readonly byte[] _frameSizeBuffer = new byte[FrameSizeLength]; 15 | private readonly byte[] _frameTypeBuffer = new byte[FrameTypeLength]; 16 | 17 | public FrameReader(NetworkStream stream) 18 | { 19 | _stream = stream; 20 | } 21 | 22 | public Frame ReadFrame() 23 | { 24 | lock (_lock) 25 | { 26 | // MESSAGE FRAME FORMAT: 27 | // 4 bytes - Int32, size of the frame, excluding this field 28 | // 4 bytes - Int32, frame type 29 | // N bytes - data 30 | // 8 bytes - Int64, timestamp 31 | // 2 bytes - UInt16, attempts 32 | // 16 bytes - Hex-string encoded message ID 33 | // N bytes - message body 34 | 35 | // Get the size of the incoming frame 36 | ReadBytes(_frameSizeBuffer, 0, FrameSizeLength); 37 | if (BitConverter.IsLittleEndian) 38 | Array.Reverse(_frameSizeBuffer); 39 | var frameLength = BitConverter.ToInt32(_frameSizeBuffer, 0); 40 | 41 | // Read the rest of the frame 42 | var frame = ReadBytesWithAllocation(frameLength); 43 | 44 | // Get the frame type 45 | Array.ConstrainedCopy(frame, 0, _frameTypeBuffer, 0, FrameTypeLength); 46 | if (BitConverter.IsLittleEndian) 47 | Array.Reverse(_frameTypeBuffer); 48 | var frameType = (FrameType)BitConverter.ToInt32(_frameTypeBuffer, 0); 49 | 50 | // Get the data portion of the frame 51 | var dataLength = frameLength - FrameTypeLength; 52 | byte[] dataBuffer = new byte[dataLength]; 53 | Array.ConstrainedCopy(frame, FrameTypeLength, dataBuffer, 0, dataLength); 54 | 55 | return new Frame 56 | { 57 | MessageSize = frameLength, 58 | Type = frameType, 59 | Data = dataBuffer 60 | }; 61 | } 62 | } 63 | 64 | private void ReadBytes(byte[] buffer, int offset, int count) 65 | { 66 | int bytesRead; 67 | int bytesLeft = count; 68 | 69 | while ((bytesRead = _stream.Read(buffer, offset, bytesLeft)) > 0) 70 | { 71 | offset += bytesRead; 72 | bytesLeft -= bytesRead; 73 | if (offset > count) throw new InvalidOperationException("Read too many bytes"); 74 | if (offset == count) break; 75 | } 76 | 77 | if (bytesLeft > 0) 78 | throw new SocketException((int)SocketError.SocketError); 79 | } 80 | 81 | private byte[] ReadBytesWithAllocation(int count) 82 | { 83 | byte[] buffer = new byte[count]; 84 | int offset = 0; 85 | ReadBytes(buffer, offset, count); 86 | return buffer; 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /NSQCore/HttpClientWrapper.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Net.Http; 4 | using System.Text; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | using Newtonsoft.Json.Linq; 8 | 9 | namespace NSQCore 10 | { 11 | public static class HttpClientWrapper 12 | { 13 | public static async Task PostAsync(HttpClient httpClient, SemaphoreSlim webClientLock, string url, byte[] data, Func handler) 14 | { 15 | await webClientLock.WaitAsync().ConfigureAwait(false); 16 | 17 | try 18 | { 19 | var byteArrayContent = new ByteArrayContent(data); 20 | var response = await httpClient.PostAsync(url, byteArrayContent).ConfigureAwait(false); 21 | 22 | if (response.Content == null) 23 | throw new Exception("null response"); 24 | 25 | var byteArray = await response.Content.ReadAsByteArrayAsync(); 26 | 27 | return handler(byteArray); 28 | } 29 | catch (Exception ex) 30 | { 31 | throw new Exception(ex.Message); 32 | } 33 | finally 34 | { 35 | webClientLock.Release(); 36 | } 37 | } 38 | 39 | public static async Task GetAsync(HttpClient httpClient, SemaphoreSlim webClientLock, string url, Func handler) 40 | { 41 | await webClientLock.WaitAsync().ConfigureAwait(false); 42 | try 43 | { 44 | string data = await httpClient.GetStringAsync(url).ConfigureAwait(false); 45 | 46 | var response = JObject.Parse(data); 47 | return response == null ? throw new Exception("invalid response") : handler(response); 48 | } 49 | catch (Exception ex) 50 | { 51 | throw new Exception(ex.Message); 52 | } 53 | finally 54 | { 55 | webClientLock.Release(); 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /NSQCore/InternalMessageEventArgs.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace NSQCore 4 | { 5 | /// 6 | /// EventArgs for a message raised by the internals of NSQCore. 7 | /// 8 | public class InternalMessageEventArgs : EventArgs 9 | { 10 | internal InternalMessageEventArgs(string message) 11 | { 12 | Message = message; 13 | } 14 | 15 | /// 16 | /// The message raised. 17 | /// 18 | public string Message { get; } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /NSQCore/Message.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Text; 3 | using System.Threading.Tasks; 4 | using NSQCore.Commands; 5 | 6 | namespace NSQCore 7 | { 8 | /// 9 | /// A message delivered from NSQ. 10 | /// 11 | public class Message 12 | { 13 | /// 14 | /// The ID of the message, which is generated by nsqd. 15 | /// 16 | public string Id { get; } 17 | 18 | /// 19 | /// The number of times this message has been delivered to consumers. 20 | /// 21 | public short Attempts { get; } 22 | 23 | /// 24 | /// The nanosecond time the mssage was created in NSQ. 25 | /// 26 | public long Timestamp { get; } 27 | 28 | /// 29 | /// The body of the message. 30 | /// 31 | public MessageBody Body { get; } 32 | 33 | private readonly NsqTcpConnection _connection; 34 | 35 | private const int TimestampStart = 0; 36 | private const int TimestampCount = 8; 37 | private const int AttemptsStart = 8; 38 | private const int AttemptsCount = 2; 39 | private const int IdStart = 10; 40 | private const int IdCount = 16; 41 | private const int DataStart = TimestampCount + AttemptsCount + IdCount; 42 | 43 | internal Message(Frame frame, NsqTcpConnection connection) 44 | { 45 | _connection = connection; 46 | 47 | if (frame.Type != FrameType.Message) 48 | throw new ArgumentException("Frame must have FrameType 'Message'", "frame"); 49 | 50 | if (BitConverter.IsLittleEndian) 51 | { 52 | Array.Reverse(frame.Data, TimestampStart, TimestampCount); 53 | Array.Reverse(frame.Data, AttemptsStart, AttemptsCount); 54 | } 55 | 56 | Timestamp = BitConverter.ToInt64(frame.Data, TimestampStart); 57 | Attempts = BitConverter.ToInt16(frame.Data, AttemptsStart); 58 | Id = Encoding.ASCII.GetString(frame.Data, IdStart, IdCount); 59 | 60 | // Data 61 | var dataLength = frame.Data.Length - DataStart; 62 | Body = new byte[dataLength]; 63 | Array.ConstrainedCopy(frame.Data, DataStart, Body, 0, dataLength); 64 | } 65 | 66 | /// 67 | /// Finishes the message, which tells the nsqd instance the message has been processed. 68 | /// 69 | public Task FinishAsync() 70 | { 71 | return _connection.SendCommandAsync(new Finish(this)); 72 | } 73 | 74 | /// 75 | /// Re-queues the message in NSQ so it will be delivered again to a consumer. 76 | /// 77 | public Task RequeueAsync() 78 | { 79 | return _connection.SendCommandAsync(new Requeue(this.Id)); 80 | } 81 | 82 | /// 83 | /// Notifies NSQ that the message is still being processed. This prevents 84 | /// NSQ from re-queueing the message automatically. 85 | /// 86 | public Task TouchAsync() 87 | { 88 | throw new NotImplementedException(); 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /NSQCore/MessageBody.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Text; 3 | 4 | namespace NSQCore 5 | { 6 | /// 7 | /// Represends the body of an NSQ message. NSQ does not interpret the body 8 | /// of a message, so this is equivalent to a byte array. The library will 9 | /// automatically convert a MessageBody to and from a string in UTF-8 encoding. 10 | /// 11 | public struct MessageBody 12 | { 13 | private static readonly byte[] Empty = new byte[0]; 14 | 15 | private readonly byte[] _data; 16 | 17 | private MessageBody(byte[] data) 18 | { 19 | _data = data; 20 | } 21 | 22 | public override string ToString() 23 | { 24 | return this; 25 | } 26 | 27 | public bool IsNull => _data == null || _data.Length == 0; 28 | 29 | /// 30 | /// Converts a byte array to a MessageBody. 31 | /// 32 | public static implicit operator MessageBody(byte[] msg) 33 | { 34 | if (msg == null) return default(MessageBody); 35 | return new MessageBody(msg); 36 | } 37 | 38 | /// 39 | /// Encodes a string as UTF-8 and converts it to a MessageBody. 40 | /// 41 | public static implicit operator MessageBody(string msg) 42 | { 43 | if (msg == null) return default(MessageBody); 44 | var data = Encoding.UTF8.GetBytes(msg); 45 | return new MessageBody(data); 46 | } 47 | 48 | /// 49 | /// Converts a message body to the underlying byte array. 50 | /// 51 | public static implicit operator byte[] (MessageBody msg) 52 | { 53 | return msg._data; 54 | } 55 | 56 | /// 57 | /// Converts a message from a UTF-8 encoded byte array to a string. 58 | /// 59 | public static implicit operator string(MessageBody msg) 60 | { 61 | if (msg._data == null) 62 | return null; 63 | 64 | try 65 | { 66 | return Encoding.UTF8.GetString(msg._data); 67 | } 68 | catch 69 | { 70 | return BitConverter.ToString(msg._data); 71 | } 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /NSQCore/NSQCore.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard1.4 5 | NSQCore - .NET Core Client for NSQ 6 | NSQCore 7 | 1.0.0 8 | Kailesh 9 | Webjet 10 | .Net Core client for NSQ 11 | false 12 | First release 13 | Copyright 2017 (c) Webjet Ltd. All rights reserved. 14 | NSQ NSQCore .NetCore 15 | https://github.com/Webjet/NSQCore 16 | https://github.com/Webjet/NSQCore/blob/master/LICENSE.md 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /NSQCore/NsqConsumer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Threading.Tasks; 4 | 5 | namespace NSQCore 6 | { 7 | public interface INsqConsumer : IDisposable 8 | { 9 | void Connect(MessageHandler handler); 10 | Task ConnectAndWaitAsync(MessageHandler handler); 11 | Task PublishAsync(Topic topic, MessageBody message); 12 | Task SetMaxInFlightAsync(int maxInFlight); 13 | } 14 | 15 | public delegate Task MessageHandler(Message message); 16 | 17 | public static class NsqConsumer 18 | { 19 | public static INsqConsumer Create(string connectionString) 20 | { 21 | return Create(ConsumerOptions.Parse(connectionString)); 22 | } 23 | 24 | public static INsqConsumer Create(ConsumerOptions options) 25 | { 26 | if (options.LookupEndPoints.Any()) 27 | { 28 | return new NsqLookupConsumer(options); 29 | } 30 | return new NsqTcpConnection(options.NsqEndPoint, options); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /NSQCore/NsqLookup.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Net.Http; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | using Newtonsoft.Json.Linq; 8 | 9 | namespace NSQCore 10 | { 11 | /// 12 | /// A client for querying an instance of nsqlookupd. 13 | /// 14 | public class NsqLookup 15 | { 16 | private readonly HttpClient _httpClient = new HttpClient(); 17 | private readonly SemaphoreSlim _webClientLock = new SemaphoreSlim(1, 1); 18 | private static readonly byte[] Empty = new byte[0]; 19 | /// 20 | /// Creates a new instance of NsqLookup. 21 | /// 22 | /// The host name or IP address of the nsqlookupd instance. 23 | /// The HTTP port of the nsqlookupd instance. 24 | public NsqLookup(string host, int port) 25 | { 26 | _httpClient.BaseAddress = new Uri("http://" + host + ":" + port); 27 | } 28 | 29 | /// 30 | /// Looks up the nsqd instances which are producing a topic. 31 | /// 32 | public Task> LookupAsync(Topic topic) 33 | { 34 | return RequestListAsync("/lookup?topic=" + topic, response => 35 | { 36 | return 37 | ((JArray)response["producers"]) 38 | .Select(producer => new NsqAddress( 39 | (string)producer["broadcast_address"], 40 | (string)producer["hostname"], 41 | (int)producer["tcp_port"], 42 | (int)producer["http_port"])) 43 | .ToList(); 44 | }); 45 | } 46 | 47 | /// 48 | /// Queries the list of topics known to this nsqlookupd instance. 49 | /// 50 | public Task> TopicsAsync() 51 | { 52 | return RequestListAsync("/topics", response => 53 | { 54 | return response["topics"] 55 | .Select(t => new Topic((string)t)) 56 | .ToList(); 57 | }); 58 | } 59 | 60 | /// 61 | /// Queries the channels known to this nsqlookupd instance. 62 | /// 63 | /// The topic to query. 64 | public Task> ChannelsAsync(Topic topic) 65 | { 66 | return RequestListAsync("/channels?topic=" + topic, response => 67 | { 68 | return response["channels"] 69 | .Select(t => new Channel((string)t)) 70 | .ToList(); 71 | }); 72 | } 73 | 74 | /// 75 | /// Queries the nsqd nodes known to this nsqlookupd instance. 76 | /// 77 | public Task> NodesAsync() 78 | { 79 | return RequestListAsync("/nodes", response => 80 | { 81 | return 82 | ((JArray)response["producers"]) 83 | .Select(producer => new NsqAddress( 84 | (string)producer["broadcast_address"], 85 | (string)producer["hostname"], 86 | (int)producer["tcp_port"], 87 | (int)producer["http_port"])) 88 | .ToList(); 89 | }); 90 | } 91 | 92 | /// 93 | /// Deletes a topic. 94 | /// 95 | public Task DeleteTopicAsync(Topic topic) 96 | { 97 | return PostAsync("/topic/delete?topic=" + topic, _ => true); 98 | } 99 | 100 | /// 101 | /// Deletes a channel. 102 | /// 103 | public Task DeleteChannelAsync(Topic topic, Channel channel) 104 | { 105 | var url = "/channel/delete?topic=" + topic + "&channel=" + channel; 106 | return PostAsync(url, _ => true); 107 | } 108 | 109 | /// 110 | /// Tombstones a topic for an nsqd instance. 111 | /// 112 | public Task TombstoneTopicAsync(Topic topic, NsqAddress producer) 113 | { 114 | var url = string.Format("/topic/tombstone?topic={0}&node={1}:{2}", topic, producer.BroadcastAddress, producer.HttpPort); 115 | return PostAsync(url, _ => true); 116 | } 117 | 118 | /// 119 | /// Queries the version of the nsqlookupd instance. 120 | /// 121 | public Task VersionAsync() 122 | { 123 | return GetAsync("/info", response => (string)response["version"]); 124 | } 125 | 126 | /// 127 | /// Queries the nsqlookupd instance for liveliness. 128 | /// 129 | /// True if nsqlookupd returns "OK". 130 | public Task PingAsync() 131 | { 132 | var response = _httpClient.GetAsync("ping").ConfigureAwait(false).GetAwaiter().GetResult(); 133 | return Task.FromResult(response.IsSuccessStatusCode); 134 | } 135 | 136 | private async Task> RequestListAsync(string url, Func> handler) 137 | { 138 | var result = await GetAsync(url, handler).ConfigureAwait(false); 139 | return result ?? new List(); 140 | } 141 | 142 | 143 | private Task PostAsync(string url, Func handler) 144 | { 145 | return PostAsync(url, Empty, handler); 146 | } 147 | 148 | private Task PostAsync(string url, byte[] data, Func handler) 149 | { 150 | return HttpClientWrapper.PostAsync(_httpClient, _webClientLock, url, data, handler); 151 | } 152 | 153 | private Task GetAsync(string url, Func handler) 154 | { 155 | return HttpClientWrapper.GetAsync(_httpClient, _webClientLock, url, handler); 156 | } 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /NSQCore/NsqLookupConsumer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Net; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | 8 | namespace NSQCore 9 | { 10 | public class NsqLookupConsumer : INsqConsumer 11 | { 12 | private readonly Dictionary _connections = new Dictionary(); 13 | private readonly List _lookupServers = new List(); 14 | private readonly ConsumerOptions _options; 15 | private readonly Task _firstConnectionTask; 16 | private readonly TaskCompletionSource _firstConnectionTaskCompletionSource = new TaskCompletionSource(); 17 | 18 | private Timer _lookupTimer; 19 | private bool _firstDiscoveryCycle = true; 20 | private int _maxInFlight; 21 | private int _started; 22 | private bool _disposed; 23 | 24 | // No need to ever reconnect, we'll reconnect on the next lookup cycle 25 | private static readonly NoRetryBackoffStrategy NoRetryBackoff = new NoRetryBackoffStrategy(); 26 | 27 | public EventHandler InternalMessages; 28 | public EventHandler DiscoveryCompleted; 29 | 30 | internal void OnDiscoveryCompleted(List addresses) 31 | { 32 | var handler = DiscoveryCompleted; 33 | handler?.Invoke(this, new DiscoveryEventArgs(addresses)); 34 | } 35 | 36 | internal void OnInternalMessage(string format, object arg0) 37 | { 38 | var handler = InternalMessages; 39 | handler?.Invoke(this, new InternalMessageEventArgs(string.Format(format, arg0))); 40 | } 41 | 42 | internal void OnInternalMessage(string format, params object[] args) 43 | { 44 | var handler = InternalMessages; 45 | handler?.Invoke(this, new InternalMessageEventArgs(string.Format(format, args))); 46 | } 47 | 48 | public NsqLookupConsumer(ConsumerOptions options) 49 | { 50 | _options = options; 51 | 52 | foreach (var lookupEndPoint in options.LookupEndPoints) 53 | { 54 | _lookupServers.Add(new NsqLookup(lookupEndPoint.Host, lookupEndPoint.Port)); 55 | } 56 | 57 | _firstConnectionTask = _firstConnectionTaskCompletionSource.Task; 58 | } 59 | 60 | public NsqLookupConsumer(string connectionString) 61 | : this(ConsumerOptions.Parse(connectionString)) 62 | { 63 | } 64 | 65 | public async Task ConnectAndWaitAsync(MessageHandler handler) 66 | { 67 | ThrowIfDisposed(); 68 | Connect(handler); 69 | await _firstConnectionTask.ConfigureAwait(false); 70 | } 71 | 72 | public void Connect(MessageHandler handler) 73 | { 74 | ThrowIfDisposed(); 75 | var wasStarted = Interlocked.CompareExchange(ref _started, 1, 0); 76 | if (wasStarted != 0) return; 77 | 78 | _lookupTimer = new Timer(LookupTask, handler, TimeSpan.Zero, _options.LookupPeriod); 79 | } 80 | 81 | private void LookupTask(object messageHandler) 82 | { 83 | MessageHandler handler = (MessageHandler)messageHandler; 84 | 85 | OnInternalMessage("Begin lookup cycle"); 86 | int beginningCount, endingCount, 87 | added = 0, removed = 0; 88 | 89 | List nsqAddresses; 90 | lock (_connections) 91 | { 92 | var tasks = _lookupServers.Select(server => server.LookupAsync(_options.Topic)).ToList(); 93 | var delay = Task.Delay(5000); 94 | Task.WhenAny(Task.WhenAll(tasks), delay).Wait(); 95 | 96 | nsqAddresses = 97 | tasks.Where(t => t.Status == TaskStatus.RanToCompletion) 98 | .SelectMany(t => t.Result) 99 | .Distinct() 100 | .ToList(); 101 | 102 | var servers = 103 | nsqAddresses 104 | .Select(add => new DnsEndPoint(add.BroadcastAddress, add.TcpPort)) 105 | .ToList(); 106 | 107 | var currentEndPoints = _connections.Keys.ToList(); 108 | var newEndPoints = servers.Except(currentEndPoints).ToList(); 109 | var removedEndPoints = currentEndPoints.Except(servers).ToList(); 110 | 111 | foreach (var endPoint in removedEndPoints) 112 | { 113 | var connection = _connections[endPoint]; 114 | _connections.Remove(endPoint); 115 | connection.Dispose(); 116 | removed++; 117 | } 118 | 119 | foreach (var endPoint in newEndPoints) 120 | { 121 | if (!_connections.ContainsKey(endPoint)) 122 | { 123 | var connection = new NsqTcpConnection(endPoint, _options, NoRetryBackoff); 124 | connection.InternalMessages += 125 | (sender, e) => OnInternalMessage("{0}: {1}", endPoint, e.Message); 126 | try 127 | { 128 | connection.Connect(handler); 129 | _connections[endPoint] = connection; 130 | added++; 131 | } 132 | catch (Exception ex) 133 | { 134 | OnInternalMessage("Connection to endpoint {0} failed: {1}", endPoint, ex.Message); 135 | } 136 | } 137 | } 138 | 139 | beginningCount = currentEndPoints.Count; 140 | endingCount = _connections.Count; 141 | 142 | SetMaxInFlightWithoutWaitingForInitialConnectionAsync(_maxInFlight).Wait(); 143 | } 144 | 145 | if (_firstDiscoveryCycle) 146 | { 147 | _firstConnectionTaskCompletionSource.TrySetResult(true); 148 | _firstDiscoveryCycle = false; 149 | } 150 | 151 | OnDiscoveryCompleted(nsqAddresses); 152 | OnInternalMessage("End lookup cycle. BeginningCount = {0}, EndingCount = {1}, Added = {2}, Removed = {3}", beginningCount, endingCount, added, removed); 153 | } 154 | 155 | public async Task PublishAsync(Topic topic, MessageBody message) 156 | { 157 | ThrowIfDisposed(); 158 | await _firstConnectionTask.ConfigureAwait(false); 159 | 160 | List connections; 161 | lock (_connections) 162 | { 163 | connections = _connections.Values.ToList(); 164 | } 165 | 166 | if (connections.Count == 0) 167 | throw new CommunicationException("No NSQ connections are available"); 168 | 169 | foreach (var thing in connections) 170 | { 171 | try 172 | { 173 | await thing.PublishAsync(topic, message).ConfigureAwait(false); 174 | return; 175 | } 176 | catch 177 | { 178 | } 179 | } 180 | 181 | throw new CommunicationException("Write failed against all NSQ connections"); 182 | } 183 | 184 | public async Task SetMaxInFlightAsync(int maxInFlight) 185 | { 186 | ThrowIfDisposed(); 187 | _maxInFlight = maxInFlight; 188 | await _firstConnectionTask.ConfigureAwait(false); 189 | await SetMaxInFlightWithoutWaitingForInitialConnectionAsync(maxInFlight).ConfigureAwait(false); 190 | } 191 | 192 | // I need a better name for this 193 | private async Task SetMaxInFlightWithoutWaitingForInitialConnectionAsync(int maxInFlight) 194 | { 195 | if (maxInFlight < 0) 196 | throw new ArgumentOutOfRangeException("maxInFlight", "MaxInFlight must be non-negative."); 197 | 198 | List connections; 199 | lock (_connections) 200 | { 201 | connections = _connections.Values.ToList(); 202 | } 203 | 204 | if (connections.Count == 0) return; 205 | 206 | int maxInFlightPerServer = maxInFlight / connections.Count; 207 | int remainder = maxInFlight % connections.Count; 208 | 209 | var tasks = new List(connections.Count); 210 | foreach (var connection in connections) 211 | { 212 | int max = maxInFlightPerServer; 213 | if (remainder > 0) 214 | { 215 | remainder -= 1; 216 | if (max < int.MaxValue) 217 | max += 1; 218 | } 219 | 220 | var setMaxTask = connection.SetMaxInFlightAsync(max) 221 | .ContinueWith(t => 222 | { 223 | if (t.Status == TaskStatus.Faulted) 224 | { 225 | connection.Dispose(); 226 | OnInternalMessage("Setting MaxInFlight on {0} threw: {1}", connection.EndPoint, t.Exception.GetBaseException().Message); 227 | } 228 | }); 229 | tasks.Add(setMaxTask); 230 | } 231 | 232 | await Task.WhenAll(tasks).ConfigureAwait(false); 233 | } 234 | 235 | public void Dispose() 236 | { 237 | lock (_connections) 238 | { 239 | _disposed = true; 240 | 241 | _lookupTimer.Dispose(); 242 | 243 | foreach (var connection in _connections.Values) 244 | connection.Dispose(); 245 | } 246 | } 247 | 248 | private void ThrowIfDisposed() 249 | { 250 | if (_disposed) throw new ObjectDisposedException("NsqLookupConnection"); 251 | } 252 | } 253 | } 254 | -------------------------------------------------------------------------------- /NSQCore/NsqProducer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Net.Http; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | using Newtonsoft.Json.Linq; 7 | 8 | namespace NSQCore 9 | { 10 | /// 11 | /// A client for nsqd which delivers messages using the HTTP protocol. 12 | /// 13 | public sealed class NsqProducer 14 | { 15 | private readonly HttpClient _httpClient = new HttpClient(); 16 | private readonly SemaphoreSlim _webClientLock = new SemaphoreSlim(1, 1); 17 | 18 | private static readonly byte[] Empty = new byte[0]; 19 | 20 | /// 21 | /// Creates a new client. 22 | /// 23 | /// The host name or IP address of the nsqd instance. 24 | /// The HTTP port of the nsqd instance. 25 | public NsqProducer(string host, int port) 26 | { 27 | _httpClient.BaseAddress = new Uri("http://" + host + ":" + port); 28 | } 29 | 30 | /// 31 | /// Creates a new topic on the nsqd instance. 32 | /// 33 | public Task CreateTopicAsync(Topic topic) 34 | { 35 | return PostAsync("/topic/create?topic=" + topic, _ => true); 36 | } 37 | 38 | /// 39 | /// Deletes a topic from the nsqd instance. 40 | /// 41 | public Task DeleteTopicAsync(Topic topic) 42 | { 43 | return PostAsync("/topic/delete?topic=" + topic, _ => true); 44 | } 45 | 46 | /// 47 | /// Clears all messages from the topic on the nsqd instance. 48 | /// 49 | public Task EmptyTopicAsync(Topic topic) 50 | { 51 | return PostAsync("/topic/empty?topic=" + topic, _ => true); 52 | } 53 | 54 | /// 55 | /// Pauses a topic on the nsqd instance. 56 | /// 57 | public Task PauseTopicAsync(Topic topic) 58 | { 59 | return PostAsync("/topic/pause?topic=" + topic, _ => true); 60 | } 61 | 62 | /// 63 | /// Unpauses a topic on the nsqd instance. 64 | /// 65 | public Task UnpauseTopicAsync(Topic topic) 66 | { 67 | return PostAsync("/topic/unpause?topic=" + topic, _ => true); 68 | } 69 | 70 | /// 71 | /// Creates a channel on the nsqd instance. 72 | /// 73 | public Task CreateChannelAsync(Topic topic, Channel channel) 74 | { 75 | return PostAsync("/channel/create?topic=" + topic + "&channel=" + channel, _ => true); 76 | } 77 | 78 | /// 79 | /// Deletes a channel from a topic on the nsqd instance. 80 | /// 81 | public Task DeleteChannelAsync(Topic topic, Channel channel) 82 | { 83 | return PostAsync("/channel/delete?topic=" + topic + "&channel=" + channel, _ => true); 84 | } 85 | 86 | /// 87 | /// Clears all messages from a channel on the nsqd instance. 88 | /// 89 | public Task EmptyChannelAsync(Topic topic, Channel channel) 90 | { 91 | return PostAsync("/channel/empty?topic=" + topic + "&channel=" + channel, _ => true); 92 | } 93 | 94 | /// 95 | /// Pauses a channel on the nsqd instance. 96 | /// 97 | public Task PauseChannelAsync(Topic topic, Channel channel) 98 | { 99 | return PostAsync("/channel/pause?topic=" + topic + "&channel=" + channel, _ => true); 100 | } 101 | 102 | /// 103 | /// Unpauses a channel on the nsqd instance. 104 | /// 105 | public Task UnpauseChannelAsync(Topic topic, Channel channel) 106 | { 107 | return PostAsync("/channel/unpause?topic=" + topic + "&channel=" + channel, _ => true); 108 | } 109 | 110 | /// 111 | /// Determines the liveliness of the nsqd instance. 112 | /// 113 | public Task PingAsync() 114 | { 115 | var response= _httpClient.GetAsync("ping").ConfigureAwait(false).GetAwaiter().GetResult(); 116 | return Task.FromResult(response.IsSuccessStatusCode); 117 | } 118 | 119 | /// 120 | /// Queries for runtime statistics of the nsqd instance. 121 | /// 122 | public Task StatisticsAsync() 123 | { 124 | return GetAsync("/stats?format=json", response => response.ToObject()); 125 | } 126 | 127 | 128 | 129 | /// 130 | /// Publishes a message to the nsqd instance. 131 | /// 132 | public Task PublishAsync(Topic topic, MessageBody data) 133 | { 134 | if (data.IsNull) 135 | throw new ArgumentOutOfRangeException("data", "Must provide data to publish"); 136 | 137 | return PostAsync("/pub?topic=" + topic, data, _ => true); 138 | } 139 | 140 | /// 141 | /// Publishes multiple messages to the nsqd instance in a single HTTP request. 142 | /// 143 | public Task PublishAsync(Topic topic, MessageBody[] messages) 144 | { 145 | byte[] totalArray; 146 | checked 147 | { 148 | var dataLength = messages.Sum(msg => ((byte[])msg).Length); 149 | var totalLength = dataLength + (4 * messages.Length) + 4; 150 | totalArray = new byte[totalLength]; 151 | } 152 | 153 | Array.Copy(BitConverter.GetBytes(messages.Length), totalArray, 4); 154 | if (BitConverter.IsLittleEndian) 155 | Array.Reverse(totalArray, 0, 4); 156 | 157 | var offsetIntoTotalArray = 4; 158 | foreach (MessageBody messageBody in messages) 159 | { 160 | var messageLength = ((byte[])messageBody).Length; 161 | Array.Copy(BitConverter.GetBytes(messageLength), 0, totalArray, offsetIntoTotalArray, 4); 162 | if (BitConverter.IsLittleEndian) 163 | Array.Reverse(totalArray, offsetIntoTotalArray, 4); 164 | offsetIntoTotalArray += 4; 165 | Array.Copy(messageBody, 0, totalArray, offsetIntoTotalArray, messageLength); 166 | offsetIntoTotalArray += messageLength; 167 | } 168 | 169 | return PostAsync("/mpub?binary=true&topic=" + topic, totalArray, _ => true); 170 | } 171 | 172 | private Task PostAsync(string url, Func handler) 173 | { 174 | return PostAsync(url, Empty, handler); 175 | } 176 | 177 | private Task PostAsync(string url, byte[] data, Func handler) 178 | { 179 | return HttpClientWrapper.PostAsync(_httpClient, _webClientLock, url, data, handler); 180 | } 181 | 182 | private Task GetAsync(string url, Func handler) 183 | { 184 | return HttpClientWrapper.GetAsync(_httpClient, _webClientLock, url, handler); 185 | } 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /NSQCore/NsqStatistics.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using Newtonsoft.Json; 3 | 4 | namespace NSQCore 5 | { 6 | /// 7 | /// Statistics from an nsqd instance. 8 | /// 9 | public class NsqStatistics 10 | { 11 | /// 12 | /// Describes the health of an nsqd instance. Will be "OK" for a healthy instance. 13 | /// 14 | [JsonProperty("health")] 15 | public string Health { get; internal set; } 16 | 17 | /// 18 | /// The version of nsqd running. 19 | /// 20 | [JsonProperty("version")] 21 | public string Version { get; internal set; } 22 | 23 | /// 24 | /// When the nsqd instance was started. 25 | /// 26 | [JsonProperty("start_time")] 27 | public int StartTime { get; internal set; } 28 | 29 | /// 30 | /// A list of statistics of the topics produced by this nsqd instance. 31 | /// 32 | [JsonProperty("topics")] 33 | public List Topics { get; internal set; } 34 | 35 | internal NsqStatistics() { } 36 | } 37 | 38 | /// 39 | /// Statistics for a particular topic in an nsqd instance. 40 | /// 41 | public class TopicStatistics 42 | { 43 | /// 44 | /// The name of the topic. 45 | /// 46 | [JsonProperty("topic_name")] 47 | public Topic Name { get; internal set; } 48 | 49 | /// 50 | /// The number of messages yet to be handled in the topic. 51 | /// 52 | [JsonProperty("depth")] 53 | public long Depth { get; internal set; } 54 | 55 | /// 56 | /// The number of messages which have been persisted to backend storage. 57 | /// 58 | [JsonProperty("backend_depth")] 59 | public long BackendDepth { get; internal set; } 60 | 61 | /// 62 | /// The number of messages which have been delivered to this topic. 63 | /// 64 | [JsonProperty("message_count")] 65 | public long MessageCount { get; internal set; } 66 | 67 | /// 68 | /// Indicates whether the topic has been paused. 69 | /// A paused topic will not be delivered to any channels. 70 | /// 71 | [JsonProperty("paused")] 72 | public bool Paused { get; internal set; } 73 | 74 | /// 75 | /// A list of statistics of the channels for this topic. 76 | /// 77 | [JsonProperty("channels")] 78 | public List Channels { get; internal set; } 79 | 80 | internal TopicStatistics() { } 81 | } 82 | 83 | /// 84 | /// Statistics for a particular topic in an nsqd instance. 85 | /// 86 | public class ChannelStatistics 87 | { 88 | /// 89 | /// The name of the channel. 90 | /// 91 | [JsonProperty("channel_name")] 92 | public Channel Name { get; set; } 93 | 94 | /// 95 | /// The number of messages yet to be handled in the channel. 96 | /// 97 | [JsonProperty("depth")] 98 | public long Depth { get; internal set; } 99 | 100 | /// 101 | /// The number of messages in the channel which have been persisted to backend storage. 102 | /// 103 | [JsonProperty("backend_depth")] 104 | public long BackendDepth { get; internal set; } 105 | 106 | /// 107 | /// The number of messages in the channel which are currently in-flight (delivered to a consumer but 108 | /// not yet marked finished). 109 | /// 110 | [JsonProperty("in_flight_count")] 111 | public long InFlightCount { get; internal set; } 112 | 113 | /// 114 | /// the number of messages in the channel which have been re-queued in the channel with a timeout. 115 | /// 116 | [JsonProperty("deferred_count")] 117 | public long DeferredCount { get; internal set; } 118 | 119 | /// 120 | /// The number of messages which have been delivered to the channel. 121 | /// 122 | [JsonProperty("message_count")] 123 | public long MessageCount { get; internal set; } 124 | 125 | /// 126 | /// The number of times a message has been delivered to a consumer but not been finished before timing out. 127 | /// The timeout duration is determined by the configuration of the nsqd instance. 128 | /// 129 | [JsonProperty("timeout_count")] 130 | public long TimeoutCount { get; internal set; } 131 | 132 | /// 133 | /// Indicates whether the channel has been paused. 134 | /// A paused topic will not be delivered to any channels. 135 | /// 136 | [JsonProperty("paused")] 137 | public bool Paused { get; internal set; } 138 | 139 | internal ChannelStatistics() { } 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /NSQCore/NsqTcpConnection.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Linq; 4 | using System.Linq.Expressions; 5 | using System.Net; 6 | using System.Net.Sockets; 7 | using System.Text; 8 | using System.Threading; 9 | using System.Threading.Tasks; 10 | using NSQCore.Commands; 11 | using NSQCore.System; 12 | 13 | namespace NSQCore 14 | { 15 | /// 16 | /// Maintains a TCP connection to a single nsqd instance and allows consuming messages. 17 | /// 18 | public sealed class NsqTcpConnection : INsqConsumer 19 | { 20 | private static readonly byte[] Heartbeat = Encoding.ASCII.GetBytes("_heartbeat_"); 21 | private static readonly byte[] MagicV2 = Encoding.ASCII.GetBytes(" V2"); 22 | 23 | public EventHandler InternalMessages; 24 | 25 | public bool Connected { get; private set; } 26 | 27 | private readonly CancellationTokenSource _connectionClosedSource; 28 | private readonly ConsumerOptions _options; 29 | internal readonly DnsEndPoint EndPoint; 30 | private readonly IBackoffStrategy _backoffStrategy; 31 | private readonly Thread _workerThread; 32 | private readonly TaskCompletionSource _firstConnection = new TaskCompletionSource(); 33 | 34 | private readonly object _connectionSwapLock = new object(); 35 | private readonly object _connectionSwapInProgressLock = new object(); 36 | 37 | private IdentifyResponse _identifyResponse; 38 | private NetworkStream _stream; 39 | private TaskCompletionSource _nextReconnectionTaskSource = new TaskCompletionSource(); 40 | private int _started; 41 | private readonly object _disposeLock = new object(); 42 | private bool _disposed; 43 | 44 | internal void OnInternalMessage(string format, object arg0) 45 | { 46 | var handler = InternalMessages; 47 | handler?.Invoke(this, new InternalMessageEventArgs(string.Format(format, arg0))); 48 | } 49 | 50 | internal void OnInternalMessage(string format, params object[] args) 51 | { 52 | var handler = InternalMessages; 53 | handler?.Invoke(this, new InternalMessageEventArgs(string.Format(format, args))); 54 | } 55 | 56 | public NsqTcpConnection(DnsEndPoint endPoint, ConsumerOptions options) 57 | : this(endPoint, options, new ExponentialBackoffStrategy(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(30))) 58 | { 59 | } 60 | 61 | public NsqTcpConnection(DnsEndPoint endPoint, ConsumerOptions options, IBackoffStrategy backoffStrategy) 62 | { 63 | EndPoint = endPoint; 64 | _options = options; 65 | _backoffStrategy = backoffStrategy; 66 | 67 | _connectionClosedSource = new CancellationTokenSource(); 68 | 69 | _workerThread = new Thread(WorkerLoop); 70 | _workerThread.Name = "NSQCore Worker"; 71 | } 72 | 73 | public Task ConnectAndWaitAsync(MessageHandler handler) 74 | { 75 | Connect(handler); 76 | return _firstConnection.Task; 77 | } 78 | 79 | /// 80 | /// Creates a new instance and connects to an nsqd instance. 81 | /// 82 | /// The delegate used to handle delivered messages. 83 | /// A connected NSQ connection. 84 | public void Connect(MessageHandler handler) 85 | { 86 | // Only start if we're the first 87 | var wasStarted = Interlocked.CompareExchange(ref _started, 1, 0); 88 | if (wasStarted != 0) return; 89 | 90 | OnInternalMessage("Worker thread starting"); 91 | _workerThread.Start(handler); 92 | OnInternalMessage("Worker thread started"); 93 | } 94 | 95 | public void Dispose() 96 | { 97 | lock (_disposeLock) 98 | { 99 | if (_disposed) return; 100 | _disposed = true; 101 | _connectionClosedSource.Cancel(); 102 | _connectionClosedSource.Dispose(); 103 | } 104 | } 105 | 106 | /// 107 | /// Publishes a message to NSQ. 108 | /// 109 | public Task PublishAsync(Topic topic, MessageBody message) 110 | { 111 | return SendCommandAsync(new Publish(topic, message)); 112 | } 113 | 114 | internal async Task SendCommandAsync(ICommand command) 115 | { 116 | var buffer = command.ToByteArray(); 117 | while (true) 118 | { 119 | NetworkStream stream; 120 | Task reconnectionTask; 121 | lock (_connectionSwapInProgressLock) 122 | { 123 | stream = _stream; 124 | reconnectionTask = _nextReconnectionTaskSource.Task; 125 | } 126 | 127 | try 128 | { 129 | if (stream != null) 130 | { 131 | await stream.WriteAsync(buffer, 0, buffer.Length).ConfigureAwait(false); 132 | return; 133 | } 134 | } 135 | catch (IOException) 136 | { 137 | continue; 138 | } 139 | catch (SocketException) 140 | { 141 | continue; 142 | } 143 | 144 | await reconnectionTask.ConfigureAwait(false); 145 | } 146 | } 147 | 148 | /// 149 | /// Sets the maximum number of messages which may be in-flight at once. 150 | /// 151 | public Task SetMaxInFlightAsync(int maxInFlight) 152 | { 153 | return SendCommandAsync(new Ready(maxInFlight)); 154 | } 155 | 156 | private void WorkerLoop(object messageHandler) 157 | { 158 | MessageHandler handler = (MessageHandler)messageHandler; 159 | bool firstConnectionAttempt = true; 160 | TcpClient client = null; 161 | FrameReader reader = null; 162 | IBackoffLimiter backoffLimiter = null; 163 | IDisposable cancellationRegistration = Disposable.Empty; 164 | 165 | while (true) 166 | { 167 | try 168 | { 169 | if (_connectionClosedSource.IsCancellationRequested) 170 | { 171 | return; 172 | } 173 | 174 | if (!Connected) 175 | { 176 | lock (_connectionSwapLock) 177 | { 178 | if (firstConnectionAttempt) 179 | { 180 | firstConnectionAttempt = false; 181 | } 182 | else 183 | { 184 | if (backoffLimiter == null) 185 | backoffLimiter = _backoffStrategy.Create(); 186 | 187 | TimeSpan delay; 188 | if (backoffLimiter.ShouldReconnect(out delay)) 189 | { 190 | OnInternalMessage("Delaying {0} ms before reconnecting", (int)delay.TotalMilliseconds); 191 | Thread.Sleep(delay); 192 | } 193 | else 194 | { 195 | // We give up 196 | OnInternalMessage("Abandoning connection"); 197 | Dispose(); 198 | return; 199 | } 200 | } 201 | 202 | lock (_connectionSwapInProgressLock) 203 | { 204 | CancellationToken cancellationToken; 205 | lock (_disposeLock) 206 | { 207 | if (_disposed) return; 208 | 209 | if (client != null) 210 | { 211 | cancellationRegistration.Dispose(); 212 | ((IDisposable)client).Dispose(); 213 | } 214 | 215 | cancellationToken = _connectionClosedSource.Token; 216 | } 217 | 218 | OnInternalMessage("TCP client starting"); 219 | 220 | client = new TcpClient(); 221 | client.ConnectAsync(EndPoint.Host, EndPoint.Port).Wait(cancellationToken); 222 | //client.Client. 223 | //((_endPoint.Host, _endPoint.Port); 224 | //cancellationRegistration = cancellationToken.Register(() => ((IDisposable)client).Dispose(), false); 225 | Connected = true; 226 | OnInternalMessage("TCP client started"); 227 | 228 | _stream = client.GetStream(); 229 | reader = new FrameReader(_stream); 230 | 231 | Handshake(_stream, reader); 232 | 233 | _firstConnection.TrySetResult(true); 234 | 235 | // Start a new backoff cycle next time we disconnect 236 | backoffLimiter = null; 237 | 238 | _nextReconnectionTaskSource.SetResult(true); 239 | _nextReconnectionTaskSource = new TaskCompletionSource(); 240 | } 241 | } 242 | } 243 | 244 | Frame frame; 245 | while ((frame = reader.ReadFrame()) != null) 246 | { 247 | if (frame.Type == FrameType.Result) 248 | { 249 | if (Heartbeat.SequenceEqual(frame.Data)) 250 | { 251 | OnInternalMessage("Heartbeat"); 252 | SendCommandAsync(new Nop()) 253 | .ContinueWith(t => Dispose(), TaskContinuationOptions.OnlyOnFaulted); 254 | } 255 | else 256 | { 257 | OnInternalMessage("Received result. Length = {0}", frame.MessageSize); 258 | } 259 | } 260 | else if (frame.Type == FrameType.Message) 261 | { 262 | OnInternalMessage("Received message. Length = {0}", frame.MessageSize); 263 | var message = new Message(frame, this); 264 | Task.Run(() => 265 | { 266 | try 267 | { 268 | handler(message); 269 | } 270 | catch (Exception) 271 | { 272 | // ignored 273 | } 274 | } 275 | ); 276 | } 277 | else if (frame.Type == FrameType.Error) 278 | { 279 | string errorString; 280 | try 281 | { 282 | errorString = Encoding.ASCII.GetString(frame.Data); 283 | } 284 | catch 285 | { 286 | errorString = BitConverter.ToString(frame.Data); 287 | } 288 | OnInternalMessage("Received error. Message = {0}", errorString); 289 | } 290 | else 291 | { 292 | OnInternalMessage("Unknown message type: {0}", frame.Type); 293 | throw new InvalidOperationException("Unknown message type " + frame.Type); 294 | } 295 | } 296 | } 297 | catch (ObjectDisposedException ex) 298 | { 299 | OnInternalMessage("Exiting worker loop due to disposal. Message = {0}", ex.Message); 300 | Connected = false; 301 | return; 302 | } 303 | catch (IOException ex) 304 | { 305 | if (!_disposed) OnInternalMessage("EXCEPTION: {0}", ex.Message); 306 | Connected = false; 307 | } 308 | catch (SocketException ex) 309 | { 310 | if (!_disposed) OnInternalMessage("EXCEPTION: {0}", ex.Message); 311 | Connected = false; 312 | } 313 | } 314 | } 315 | 316 | private void Handshake(NetworkStream stream, FrameReader reader) 317 | { 318 | // Initiate the V2 protocol 319 | stream.Write(MagicV2, 0, MagicV2.Length); 320 | _identifyResponse = Identify(stream, reader); 321 | if (_identifyResponse.AuthRequired) 322 | { 323 | Dispose(); 324 | throw new NotSupportedException("Authorization is not supported"); 325 | } 326 | 327 | SendCommandToStream(stream, new Subscribe(_options.Topic, _options.Channel)); 328 | } 329 | 330 | private IdentifyResponse Identify(NetworkStream stream, FrameReader reader) 331 | { 332 | var identify = new Identify(_options); 333 | SendCommandToStream(stream, identify); 334 | var frame = reader.ReadFrame(); 335 | if (frame.Type != FrameType.Result) 336 | { 337 | throw new InvalidOperationException("Unexpected frame type after IDENTIFY"); 338 | } 339 | return identify.ParseIdentifyResponse(frame.Data); 340 | } 341 | 342 | private static void SendCommandToStream(NetworkStream stream, ICommand command) 343 | { 344 | var msg = command.ToByteArray(); 345 | stream.Write(msg, 0, msg.Length); 346 | } 347 | } 348 | } 349 | -------------------------------------------------------------------------------- /NSQCore/Properties/PublishProfiles/FolderProfile.pubxml: -------------------------------------------------------------------------------- 1 |  2 | 6 | 7 | 8 | FileSystem 9 | Release 10 | netstandard1.4 11 | bin\Release\PublishOutput 12 | 13 | -------------------------------------------------------------------------------- /NSQCore/System/Disposable.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace NSQCore.System 4 | { 5 | internal class Disposable 6 | { 7 | public static readonly IDisposable Empty = new EmptyDisposable(); 8 | 9 | private class EmptyDisposable : IDisposable 10 | { 11 | public void Dispose() 12 | { 13 | } 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /NSQCore/TopicAndChannel.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Text; 3 | 4 | namespace NSQCore 5 | { 6 | /// 7 | /// A topic name in NSQ. Topics should represent a type of message, 8 | /// for example, "new-users" or "order-updated". 9 | /// 10 | public struct Topic 11 | { 12 | public string Name { get; } 13 | 14 | public Topic(string topic) 15 | : this() 16 | { 17 | Name = topic ?? throw new ArgumentNullException("topic"); 18 | } 19 | 20 | public override string ToString() 21 | { 22 | return Name; 23 | } 24 | 25 | internal byte[] ToUtf8() 26 | { 27 | return Encoding.UTF8.GetBytes(Name); 28 | } 29 | 30 | public static implicit operator string(Topic topic) 31 | { 32 | return topic.Name; 33 | } 34 | 35 | public static implicit operator Topic(string topic) 36 | { 37 | return new Topic(topic); 38 | } 39 | } 40 | 41 | /// 42 | /// A channel name in NSQ. Channels should represent the action of a consumer, 43 | /// for example, "send_email" or "create_database_record". 44 | /// 45 | public struct Channel 46 | { 47 | public string Name { get; } 48 | 49 | public Channel(string channel) 50 | : this() 51 | { 52 | Name = channel ?? throw new ArgumentNullException("channel"); 53 | } 54 | 55 | public override string ToString() 56 | { 57 | return Name; 58 | } 59 | 60 | internal byte[] ToUtf8() 61 | { 62 | return Encoding.UTF8.GetBytes(Name); 63 | } 64 | 65 | public static implicit operator string(Channel channel) 66 | { 67 | return channel.Name; 68 | } 69 | 70 | public static implicit operator Channel(string channel) 71 | { 72 | return new Channel(channel); 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /NSQCoreClient/NSQCoreClient.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp1.1 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /NSQCoreClient/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Text; 4 | using System.Threading.Tasks; 5 | using NSQCore; 6 | 7 | namespace NSQCoreClient 8 | { 9 | class Program 10 | { 11 | private static void Main(string[] args) 12 | { 13 | 14 | //Task.Run( PublishMessage).GetAwaiter().GetResult(); 15 | 16 | Task.Run(LookupConsumeMessages).GetAwaiter().GetResult(); 17 | 18 | Console.ReadKey(); 19 | } 20 | 21 | private static async Task PublishMessage() 22 | { 23 | 24 | Console.WriteLine("Publishing Message"); 25 | 26 | var prod = new NsqProducer("localhost", 4151); 27 | await prod.PublishAsync("topic1", "hello world" ); 28 | Console.WriteLine("Message Published"); 29 | } 30 | 31 | 32 | private static async Task LookupConsumeMessages() 33 | { 34 | 35 | var cons = NsqConsumer.Create("lookupd=localhost:4161; topic=topic1; channel=abc"); 36 | await cons.ConnectAndWaitAsync(Handler); 37 | await cons.SetMaxInFlightAsync(5); 38 | } 39 | 40 | 41 | static async Task Handler(Message message) 42 | { 43 | Console.WriteLine("Received: Message={0}", message.Body); 44 | await message.FinishAsync(); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | NSQCore 2 | ================ 3 | 4 | [![NuGet](https://img.shields.io/nuget/v/NSQCore.svg)](https://www.nuget.org/packages/NSQCore) 5 | 6 | An [NSQ][nsq] .NET Core 1.0 client library which targets netstandard 1.4 7 | 8 | It is only compatible with nsqio version 1.0.0-compat 9 | 10 | This project was originally branched from [Turbocharged.NSQ](https://github.com/jennings/Turbocharged.NSQ) and then converted to .Net Standard 1.4 11 | 12 | 13 | Usage 14 | ----- 15 | 16 | Nuget source: 17 | 18 | PM> Install-Package NSQCore 19 | 20 | ### Producing Messages 21 | 22 | var prod = new NsqProducer("localhost", 4151); 23 | await prod.PublishAsync("topic1", "hello world" ); 24 | await prod.SetMaxInFlightAsync(1); 25 | 26 | A connection string looks like this: 27 | 28 | nsqd=localhost:4151; 29 | 30 | ### Consuming Messages 31 | 32 | var cons = NsqConsumer.Create("lookupd=localhost:4161; topic=topic1; channel=abc"); 33 | await cons.ConnectAndWaitAsync(Handler); 34 | await cons.SetMaxInFlightAsync(1); 35 | 36 | 37 | static async Task Handler(Message message) 38 | { 39 | Console.WriteLine("Received: Message={0}", message.Body); 40 | await message.FinishAsync(); 41 | } 42 | 43 | 44 | 45 | Connection string values 46 | ------------------------ 47 | 48 | A connection string looks like this: 49 | 50 | nsqd=localhost:4150; 51 | 52 | Or, to use nsqlookupd: 53 | 54 | lookup1:4161; lookup2:4161; 55 | 56 | A connection string must specify _either_ an `nsqd` endpoint _or_ `nsqlookupd` endpoints, but not both. 57 | 58 | | Setting | Description | 59 | | --------------------- | ----------------------------------------------------------------------------------------------------- | 60 | | lookupd={endpoints} | List of `nsqlookupd` servers in the form `hostname:httpport`, e.g., `lookup1:4161;lookup2:4161` | 61 | | nsqd={endpoints} | A _single_ `nsqd` servers in the form `hostname:tcpport`, e.g., `nsqd=server1:4150;nsqd=server2:4150` | 62 | | clientId={string} | A string identifying this client to the `nsqd` server | 63 | | hostname={string} | The hostname to identify as (defaults to Environment.MachineName) | 64 | | maxInFlight={int} | The maximum number of messages this client wants to receive without completion | 65 | 66 | 67 | License 68 | ------- 69 | 70 | The MIT License. See `LICENSE.md`. 71 | 72 | 73 | [nsq]: http://nsq.io/ 74 | --------------------------------------------------------------------------------