├── .github └── workflows │ └── publish.yml ├── .gitignore ├── LICENSE ├── README.txt ├── StreamNet.sln └── StreamNet ├── Consumers ├── Consumer.cs ├── ConsumerGroupAttribute.cs └── TopicNameAttribute.cs ├── Extensions └── StringExtensions.cs ├── Producers ├── IPublisher.cs └── Publisher.cs ├── Serializers ├── Deserializer.cs └── Serializer.cs ├── Settings.cs ├── Startup.cs ├── StreamNet.csproj ├── Topic ├── ITopicManagement.cs ├── TopicData.cs └── TopicManagement.cs └── UnitTestingHelpers └── UnitTestDetector.cs /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: publish to nuget 2 | on: 3 | push: 4 | branches: 5 | - main # Default release branch, may also be named 'master' or 'develop' 6 | jobs: 7 | publish: 8 | name: build, pack & publish 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v2 12 | 13 | - name: Setup dotnet 14 | uses: actions/setup-dotnet@v1 15 | with: 16 | dotnet-version: 6.0.300 17 | 18 | # Publish 19 | - name: publish on version change 20 | id: publish_nuget 21 | uses: brandedoutcast/publish-nuget@v2.5.2 22 | 23 | with: 24 | # Filepath of the project to be packaged, relative to root of repository 25 | PROJECT_FILE_PATH: StreamNet/StreamNet.csproj 26 | 27 | # Configuration to build and package 28 | BUILD_CONFIGURATION: Release 29 | 30 | # Platform target to compile (default is empty/AnyCPU) 31 | # BUILD_PLATFORM: x64 32 | 33 | # NuGet package id, used for version detection & defaults to project name 34 | PACKAGE_NAME: StreamNet 35 | 36 | # Filepath with version info, relative to root of repository & defaults to PROJECT_FILE_PATH 37 | # VERSION_FILE_PATH: Directory.Build.props 38 | 39 | # Regex pattern to extract version info in a capturing group 40 | #VERSION_REGEX: ^\s*(.*)<\/Version>\s*$ 41 | 42 | # Useful with external providers like Nerdbank.GitVersioning, ignores VERSION_FILE_PATH & VERSION_REGEX 43 | VERSION_STATIC: 1.6.3 44 | 45 | # Flag to toggle git tagging, enabled by default 46 | TAG_COMMIT: true 47 | 48 | # Format of the git tag, [*] gets replaced with actual version 49 | # TAG_FORMAT: v* 50 | 51 | # API key to authenticate with NuGet server 52 | NUGET_KEY: ${{secrets.NUGET_KEY}} 53 | 54 | # NuGet server uri hosting the packages, defaults to https://api.nuget.org 55 | NUGET_SOURCE: https://api.nuget.org 56 | 57 | # Flag to toggle pushing symbols along with nuget package to the server, disabled by default 58 | # INCLUDE_SYMBOLS: false 59 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.rsuser 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Mono auto generated files 17 | mono_crash.* 18 | 19 | # Build results 20 | [Dd]ebug/ 21 | [Dd]ebugPublic/ 22 | [Rr]elease/ 23 | [Rr]eleases/ 24 | x64/ 25 | x86/ 26 | [Aa][Rr][Mm]/ 27 | [Aa][Rr][Mm]64/ 28 | bld/ 29 | [Bb]in/ 30 | [Oo]bj/ 31 | [Ll]og/ 32 | [Ll]ogs/ 33 | 34 | # Visual Studio 2015/2017 cache/options directory 35 | .vs/ 36 | .idea/ 37 | WebApplication1/ 38 | UnitTest/ 39 | Application/ 40 | IntegratedTest/ 41 | 42 | # Uncomment if you have tasks that create the project's static files in wwwroot 43 | #wwwroot/ 44 | 45 | # Visual Studio 2017 auto generated files 46 | Generated\ Files/ 47 | 48 | # MSTest test Results 49 | [Tt]est[Rr]esult*/ 50 | [Bb]uild[Ll]og.* 51 | 52 | # NUnit 53 | *.VisualState.xml 54 | TestResult.xml 55 | nunit-*.xml 56 | 57 | # Build Results of an ATL Project 58 | [Dd]ebugPS/ 59 | [Rr]eleasePS/ 60 | dlldata.c 61 | 62 | # Benchmark Results 63 | BenchmarkDotNet.Artifacts/ 64 | 65 | # .NET Core 66 | project.lock.json 67 | project.fragment.lock.json 68 | artifacts/ 69 | 70 | # StyleCop 71 | StyleCopReport.xml 72 | 73 | # Files built by Visual Studio 74 | *_i.c 75 | *_p.c 76 | *_h.h 77 | *.ilk 78 | *.meta 79 | *.obj 80 | *.iobj 81 | *.pch 82 | *.pdb 83 | *.ipdb 84 | *.pgc 85 | *.pgd 86 | *.rsp 87 | *.sbr 88 | *.tlb 89 | *.tli 90 | *.tlh 91 | *.tmp 92 | *.tmp_proj 93 | *_wpftmp.csproj 94 | *.log 95 | *.vspscc 96 | *.vssscc 97 | .builds 98 | *.pidb 99 | *.svclog 100 | *.scc 101 | 102 | # Chutzpah Test files 103 | _Chutzpah* 104 | 105 | # Visual C++ cache files 106 | ipch/ 107 | *.aps 108 | *.ncb 109 | *.opendb 110 | *.opensdf 111 | *.sdf 112 | *.cachefile 113 | *.VC.db 114 | *.VC.VC.opendb 115 | 116 | # Visual Studio profiler 117 | *.psess 118 | *.vsp 119 | *.vspx 120 | *.sap 121 | 122 | # Visual Studio Trace Files 123 | *.e2e 124 | 125 | # TFS 2012 Local Workspace 126 | $tf/ 127 | 128 | # Guidance Automation Toolkit 129 | *.gpState 130 | 131 | # ReSharper is a .NET coding add-in 132 | _ReSharper*/ 133 | *.[Rr]e[Ss]harper 134 | *.DotSettings.user 135 | 136 | # TeamCity is a build add-in 137 | _TeamCity* 138 | 139 | # DotCover is a Code Coverage Tool 140 | *.dotCover 141 | 142 | # AxoCover is a Code Coverage Tool 143 | .axoCover/* 144 | !.axoCover/settings.json 145 | 146 | # Visual Studio code coverage results 147 | *.coverage 148 | *.coveragexml 149 | 150 | # NCrunch 151 | _NCrunch_* 152 | .*crunch*.local.xml 153 | nCrunchTemp_* 154 | 155 | # MightyMoose 156 | *.mm.* 157 | AutoTest.Net/ 158 | 159 | # Web workbench (sass) 160 | .sass-cache/ 161 | 162 | # Installshield output folder 163 | [Ee]xpress/ 164 | 165 | # DocProject is a documentation generator add-in 166 | DocProject/buildhelp/ 167 | DocProject/Help/*.HxT 168 | DocProject/Help/*.HxC 169 | DocProject/Help/*.hhc 170 | DocProject/Help/*.hhk 171 | DocProject/Help/*.hhp 172 | DocProject/Help/Html2 173 | DocProject/Help/html 174 | 175 | # Click-Once directory 176 | publish/ 177 | 178 | # Publish Web Output 179 | *.[Pp]ublish.xml 180 | *.azurePubxml 181 | # Note: Comment the next line if you want to checkin your web deploy settings, 182 | # but database connection strings (with potential passwords) will be unencrypted 183 | *.pubxml 184 | *.publishproj 185 | 186 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 187 | # checkin your Azure Web App publish settings, but sensitive information contained 188 | # in these scripts will be unencrypted 189 | PublishScripts/ 190 | 191 | # NuGet Packages 192 | *.nupkg 193 | # NuGet Symbol Packages 194 | *.snupkg 195 | # The packages folder can be ignored because of Package Restore 196 | **/[Pp]ackages/* 197 | # except build/, which is used as an MSBuild target. 198 | !**/[Pp]ackages/build/ 199 | # Uncomment if necessary however generally it will be regenerated when needed 200 | #!**/[Pp]ackages/repositories.config 201 | # NuGet v3's project.json files produces more ignorable files 202 | *.nuget.props 203 | *.nuget.targets 204 | 205 | # Microsoft Azure Build Output 206 | csx/ 207 | *.build.csdef 208 | 209 | # Microsoft Azure Emulator 210 | ecf/ 211 | rcf/ 212 | 213 | # Windows Store app package directories and files 214 | AppPackages/ 215 | BundleArtifacts/ 216 | Package.StoreAssociation.xml 217 | _pkginfo.txt 218 | *.appx 219 | *.appxbundle 220 | *.appxupload 221 | 222 | # Visual Studio cache files 223 | # files ending in .cache can be ignored 224 | *.[Cc]ache 225 | # but keep track of directories ending in .cache 226 | !?*.[Cc]ache/ 227 | 228 | # Others 229 | ClientBin/ 230 | ~$* 231 | *~ 232 | *.dbmdl 233 | *.dbproj.schemaview 234 | *.jfm 235 | *.pfx 236 | *.publishsettings 237 | orleans.codegen.cs 238 | 239 | # Including strong name files can present a security risk 240 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 241 | #*.snk 242 | 243 | # Since there are multiple workflows, uncomment next line to ignore bower_components 244 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 245 | #bower_components/ 246 | 247 | # RIA/Silverlight projects 248 | Generated_Code/ 249 | 250 | # Backup & report files from converting an old project file 251 | # to a newer Visual Studio version. Backup files are not needed, 252 | # because we have git ;-) 253 | _UpgradeReport_Files/ 254 | Backup*/ 255 | UpgradeLog*.XML 256 | UpgradeLog*.htm 257 | ServiceFabricBackup/ 258 | *.rptproj.bak 259 | 260 | # SQL Server files 261 | *.mdf 262 | *.ldf 263 | *.ndf 264 | 265 | # Business Intelligence projects 266 | *.rdl.data 267 | *.bim.layout 268 | *.bim_*.settings 269 | *.rptproj.rsuser 270 | *- [Bb]ackup.rdl 271 | *- [Bb]ackup ([0-9]).rdl 272 | *- [Bb]ackup ([0-9][0-9]).rdl 273 | 274 | # Microsoft Fakes 275 | FakesAssemblies/ 276 | 277 | # GhostDoc plugin setting file 278 | *.GhostDoc.xml 279 | 280 | # Node.js Tools for Visual Studio 281 | .ntvs_analysis.dat 282 | node_modules/ 283 | 284 | # Visual Studio 6 build log 285 | *.plg 286 | 287 | # Visual Studio 6 workspace options file 288 | *.opt 289 | 290 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 291 | *.vbw 292 | 293 | # Visual Studio LightSwitch build output 294 | **/*.HTMLClient/GeneratedArtifacts 295 | **/*.DesktopClient/GeneratedArtifacts 296 | **/*.DesktopClient/ModelManifest.xml 297 | **/*.Server/GeneratedArtifacts 298 | **/*.Server/ModelManifest.xml 299 | _Pvt_Extensions 300 | 301 | # Paket dependency manager 302 | .paket/paket.exe 303 | paket-files/ 304 | 305 | # FAKE - F# Make 306 | .fake/ 307 | 308 | # CodeRush personal settings 309 | .cr/personal 310 | 311 | # Python Tools for Visual Studio (PTVS) 312 | __pycache__/ 313 | *.pyc 314 | 315 | # Cake - Uncomment if you are using it 316 | # tools/** 317 | # !tools/packages.config 318 | 319 | # Tabs Studio 320 | *.tss 321 | 322 | # Telerik's JustMock configuration file 323 | *.jmconfig 324 | 325 | # BizTalk build output 326 | *.btp.cs 327 | *.btm.cs 328 | *.odx.cs 329 | *.xsd.cs 330 | 331 | # OpenCover UI analysis results 332 | OpenCover/ 333 | 334 | # Azure Stream Analytics local run output 335 | ASALocalRun/ 336 | 337 | # MSBuild Binary and Structured Log 338 | *.binlog 339 | 340 | # NVidia Nsight GPU debugger configuration file 341 | *.nvuser 342 | 343 | # MFractors (Xamarin productivity tool) working folder 344 | .mfractor/ 345 | 346 | # Local History for Visual Studio 347 | .localhistory/ 348 | 349 | # BeatPulse healthcheck temp database 350 | healthchecksdb 351 | 352 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 353 | MigrationBackup/ 354 | 355 | # Ionide (cross platform F# VS Code tools) working folder 356 | .ionide/ 357 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Otávio Larrosa 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.txt: -------------------------------------------------------------------------------- 1 | StreamNet 2 | ================ 3 | 4 | The official documentation can be found here: 5 | https://stream-net.github.io/docs/ 6 | -------------------------------------------------------------------------------- /StreamNet.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SolutionItems", "SolutionItems", "{CD4DCA5B-D48F-4E0D-8D6B-F2C8C5B5881B}" 4 | ProjectSection(SolutionItems) = preProject 5 | .gitignore = .gitignore 6 | .github\workflows\publish.yml = .github\workflows\publish.yml 7 | README.md = README.md 8 | EndProjectSection 9 | EndProject 10 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StreamNet", "StreamNet\StreamNet.csproj", "{82C19D39-F43B-46B7-90FF-86F4511837B6}" 11 | EndProject 12 | Global 13 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 14 | Debug|Any CPU = Debug|Any CPU 15 | Release|Any CPU = Release|Any CPU 16 | EndGlobalSection 17 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 18 | {82C19D39-F43B-46B7-90FF-86F4511837B6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 19 | {82C19D39-F43B-46B7-90FF-86F4511837B6}.Debug|Any CPU.Build.0 = Debug|Any CPU 20 | {82C19D39-F43B-46B7-90FF-86F4511837B6}.Release|Any CPU.ActiveCfg = Release|Any CPU 21 | {82C19D39-F43B-46B7-90FF-86F4511837B6}.Release|Any CPU.Build.0 = Release|Any CPU 22 | EndGlobalSection 23 | EndGlobal 24 | -------------------------------------------------------------------------------- /StreamNet/Consumers/Consumer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading; 3 | using System.Threading.Tasks; 4 | using Confluent.Kafka; 5 | using StreamNet.Serializers; 6 | using Microsoft.Extensions.Hosting; 7 | using Microsoft.Extensions.Logging; 8 | using Polly; 9 | using Polly.Retry; 10 | using StreamNet.Producers; 11 | using StreamNet.UnitTestingHelpers; 12 | 13 | namespace StreamNet.Consumers 14 | { 15 | public abstract class Consumer : IHostedService 16 | { 17 | private ConsumerConfig _config; 18 | private AsyncRetryPolicy _retryPolicy; 19 | protected IConsumer _consumer; 20 | protected ConsumeResult _consumeResult; 21 | private string ConsumerGroupId { get; set; } 22 | private string TopicName { get; set; } 23 | protected TEvent Message { get; set; } 24 | protected readonly ILogger> _logger; 25 | protected Guid CorrelationId; 26 | 27 | protected Consumer(ILogger> logger) 28 | { 29 | Settings.GetInstance(); 30 | 31 | _logger = logger; 32 | SetConsumerId(); 33 | SetTopicName(); 34 | _retryPolicy = Policy.Handle() 35 | .WaitAndRetryAsync(retryCount: Settings.RetryCount, 36 | sleepDurationProvider: _ => TimeSpan.FromSeconds(Settings.TimeToRetryInSeconds)); 37 | } 38 | 39 | private void SetTopicName() 40 | { 41 | var topicNameFromAttribute = 42 | ((TopicNameAttribute)Attribute.GetCustomAttribute(GetType(), typeof(TopicNameAttribute))!)?.TopicName; 43 | TopicName = string.IsNullOrEmpty(topicNameFromAttribute) ? typeof(TEvent).FullName : topicNameFromAttribute; 44 | } 45 | 46 | private void SetConsumerId() 47 | { 48 | var consumerGroup = 49 | ((ConsumerGroupAttribute)Attribute.GetCustomAttribute(GetType(), typeof(ConsumerGroupAttribute))!) 50 | .ConsumerGroup; 51 | if (string.IsNullOrEmpty(consumerGroup) && !UnitTestDetector.IsRunningFromUnitTesting()) 52 | throw new ArgumentNullException($"ConsumerGroup attribute is required!"); 53 | ConsumerGroupId = consumerGroup; 54 | } 55 | 56 | 57 | public abstract Task HandleAsync(); 58 | 59 | protected async Task Consume() 60 | { 61 | try 62 | { 63 | _consumer.Subscribe(TopicName); 64 | var cancelToken = new CancellationTokenSource(); 65 | try 66 | { 67 | while (true) 68 | { 69 | _consumeResult = _consumer.Consume(cancelToken.Token); 70 | CorrelationId = Guid.NewGuid(); 71 | Message = _consumeResult.Message.Value; 72 | _logger.BeginScope("{@CorrelationId}", CorrelationId); 73 | _logger.LogInformation("Processing message: {@Message}", Message); 74 | 75 | await _retryPolicy.ExecuteAsync(async () => { await HandleAsync(); }); 76 | } 77 | } 78 | catch (OperationCanceledException) 79 | { 80 | _consumer.Close(); 81 | throw; 82 | } 83 | } 84 | catch (Exception ex) 85 | { 86 | _logger.LogInformation("Sending message to dead-letter {@Message}", Message); 87 | await new Publisher().ProduceAsyncDeadLetter(Message, TopicName); 88 | Consume(); 89 | } 90 | } 91 | 92 | public async Task StartAsync(CancellationToken cancellationToken) 93 | { 94 | Settings.GetInstance(); 95 | var consumerConfig = Settings.ConsumerConfig; 96 | consumerConfig.GroupId = ConsumerGroupId; 97 | consumerConfig.EnableAutoCommit = true; 98 | consumerConfig.EnableAutoOffsetStore = true; 99 | _config = consumerConfig; 100 | _consumer = new ConsumerBuilder(_config) 101 | .SetValueDeserializer(new Deserializer()) 102 | .Build(); 103 | 104 | Task.Factory.StartNew(async () => { return Consume(); }); 105 | } 106 | 107 | public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; 108 | } 109 | } -------------------------------------------------------------------------------- /StreamNet/Consumers/ConsumerGroupAttribute.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace StreamNet.Consumers 4 | { 5 | [AttributeUsage(AttributeTargets.Class)] 6 | public class ConsumerGroupAttribute : Attribute 7 | { 8 | public string ConsumerGroup { get; } 9 | public ConsumerGroupAttribute(string consumerGroup) => ConsumerGroup = consumerGroup; 10 | } 11 | } -------------------------------------------------------------------------------- /StreamNet/Consumers/TopicNameAttribute.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace StreamNet.Consumers 4 | { 5 | [AttributeUsage(AttributeTargets.Class)] 6 | public class TopicNameAttribute : Attribute 7 | { 8 | public string TopicName { get; } 9 | public TopicNameAttribute(string topicName) => TopicName = topicName; 10 | } 11 | } -------------------------------------------------------------------------------- /StreamNet/Extensions/StringExtensions.cs: -------------------------------------------------------------------------------- 1 | namespace StreamNet.Extensions 2 | { 3 | public static class StringExtensions 4 | { 5 | public static bool IsNullOrEmpty(this string str) => string.IsNullOrEmpty(str); 6 | } 7 | } -------------------------------------------------------------------------------- /StreamNet/Producers/IPublisher.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace StreamNet.Producers 4 | { 5 | public interface IPublisher 6 | { 7 | Task ProduceAsync(T message, string? topicName = null); 8 | } 9 | } -------------------------------------------------------------------------------- /StreamNet/Producers/Publisher.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using Confluent.Kafka; 4 | using StreamNet.Serializers; 5 | 6 | namespace StreamNet.Producers 7 | { 8 | public class Publisher : IPublisher 9 | { 10 | public Publisher() => Settings.GetInstance(); 11 | 12 | public async Task ProduceAsync(T message, string? topicName = null) 13 | { 14 | try 15 | { 16 | Task.Run(async () => 17 | { 18 | using var producerBuilder = new ProducerBuilder(Settings.ProducerConfig) 19 | .SetValueSerializer(new Serializer()).Build(); 20 | topicName ??= message?.GetType().FullName; 21 | await producerBuilder.ProduceAsync(topicName, new Message { Value = message }); 22 | }); 23 | } 24 | catch (Exception e) 25 | { 26 | Console.WriteLine(e); 27 | throw; 28 | } 29 | } 30 | 31 | internal async Task ProduceAsyncDeadLetter(T message, string? topicName = null) 32 | { 33 | try 34 | { 35 | Settings.GetInstance(); 36 | using var producerBuilder = new ProducerBuilder(Settings.ProducerConfig) 37 | .SetValueSerializer(new Serializer()).Build(); 38 | topicName ??= message?.GetType().FullName; 39 | 40 | if (string.IsNullOrEmpty(topicName)) 41 | return; 42 | 43 | await producerBuilder.ProduceAsync($"{topicName}_dead_letter", 44 | new Message { Value = message }); 45 | } 46 | catch (Exception e) 47 | { 48 | Console.WriteLine(e); 49 | throw; 50 | } 51 | } 52 | } 53 | } -------------------------------------------------------------------------------- /StreamNet/Serializers/Deserializer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Confluent.Kafka; 3 | 4 | namespace StreamNet.Serializers 5 | { 6 | public class Deserializer : IDeserializer 7 | { 8 | public TEvent Deserialize(ReadOnlySpan data, bool isNull, SerializationContext context) 9 | { 10 | var deserializedWeatherForecast = System.Text.Json.JsonSerializer.Deserialize(data)!; 11 | return deserializedWeatherForecast; 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /StreamNet/Serializers/Serializer.cs: -------------------------------------------------------------------------------- 1 | using System.Text; 2 | using Confluent.Kafka; 3 | 4 | namespace StreamNet.Serializers 5 | { 6 | public class Serializer : ISerializer 7 | { 8 | public byte[] Serialize(TEvent data, SerializationContext context) 9 | { 10 | var deserializedWeatherForecast = System.Text.Json.JsonSerializer.Serialize(data); 11 | return Encoding.ASCII.GetBytes(deserializedWeatherForecast); 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /StreamNet/Settings.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Net; 4 | using Confluent.Kafka; 5 | using StreamNet.Extensions; 6 | using Microsoft.Extensions.Configuration; 7 | using StreamNet.UnitTestingHelpers; 8 | 9 | namespace StreamNet 10 | { 11 | public class Settings 12 | { 13 | private Settings() 14 | { 15 | var jsonFiles = Directory.EnumerateFiles(".", "*.json", SearchOption.AllDirectories); 16 | var builder = new ConfigurationBuilder(); 17 | builder.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true); 18 | builder.AddJsonFile($"appsettings.{Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT")}.json", 19 | optional: true); 20 | builder.SetBasePath(Environment.CurrentDirectory); 21 | builder.AddEnvironmentVariables(); 22 | var configuration = builder.Build(); 23 | 24 | var bootstrapServers = configuration.GetSection("Kafka:BootstrapServers").Value; 25 | var saslMechanism = configuration.GetSection("Kafka:SaslMechanism").Value; 26 | var securityProtocol = configuration.GetSection("Kafka:SecurityProtocol").Value; 27 | var username = configuration.GetSection("Kafka:Username").Value; 28 | var saslPassword = configuration.GetSection("Kafka:Password").Value; 29 | 30 | _bootstrapServers = bootstrapServers; 31 | _saslMechanism = saslMechanism; 32 | _securityProtocol = securityProtocol; 33 | _username = username; 34 | _saslPassword = saslPassword; 35 | 36 | int.TryParse(configuration.GetSection("Kafka:RetryCount").Value, out var retryCount); 37 | RetryCount = retryCount; 38 | int.TryParse(configuration.GetSection("Kafka:TimeToRetryInSeconds").Value, out var timeToRetryInSeconds); 39 | TimeToRetryInSeconds = timeToRetryInSeconds; 40 | int.TryParse(configuration.GetSection("Kafka:ConsumerInstances").Value, out var consumerInstances); 41 | TimeToRetryInSeconds = timeToRetryInSeconds; 42 | 43 | if (_bootstrapServers.IsNullOrEmpty()) 44 | if (!UnitTestDetector.IsRunningFromUnitTesting()) 45 | throw new ArgumentNullException("BootstrapServers is required!"); 46 | 47 | var config = new AdminClientConfig 48 | { 49 | BootstrapServers = bootstrapServers, 50 | SecurityProtocol = GetSecurityProtocol(securityProtocol), 51 | SaslMechanism = GetSaslMechanism(saslMechanism), 52 | SaslUsername = username, 53 | SaslPassword = saslPassword, 54 | }; 55 | 56 | AdminClient = new AdminClientBuilder(config).Build(); 57 | ProducerConfig = new ProducerConfig 58 | { 59 | ClientId = Dns.GetHostName(), 60 | BootstrapServers = bootstrapServers, 61 | SecurityProtocol = GetSecurityProtocol(securityProtocol), 62 | SaslMechanism = GetSaslMechanism(saslMechanism), 63 | SaslUsername = username, 64 | SaslPassword = saslPassword, 65 | }; 66 | 67 | ConsumerConfig = new ConsumerConfig 68 | { 69 | ClientId = Dns.GetHostName(), 70 | BootstrapServers = _bootstrapServers, 71 | SecurityProtocol = GetSecurityProtocol(_securityProtocol), 72 | SaslMechanism = GetSaslMechanism(_saslMechanism), 73 | SaslUsername = _username, 74 | SaslPassword = _saslPassword, 75 | AutoOffsetReset = AutoOffsetReset.Earliest 76 | }; 77 | } 78 | 79 | private static Settings _instance; 80 | private static readonly object _lock = new object(); 81 | 82 | public static Settings GetInstance() 83 | { 84 | lock (_lock) 85 | if (_instance == null) 86 | _instance = new Settings(); 87 | return _instance; 88 | } 89 | 90 | internal SecurityProtocol GetSecurityProtocol(string securityProtocol) 91 | { 92 | switch (securityProtocol) 93 | { 94 | case "PlainText": 95 | return SecurityProtocol.Plaintext; 96 | case "Ssl": 97 | return SecurityProtocol.Ssl; 98 | case "SaslPlaintext": 99 | return SecurityProtocol.SaslPlaintext; 100 | case "SaslSsl": 101 | return SecurityProtocol.SaslSsl; 102 | default: 103 | return default; 104 | } 105 | } 106 | 107 | internal SaslMechanism GetSaslMechanism(string saslMechanism) 108 | { 109 | switch (saslMechanism) 110 | { 111 | case "GssApi": 112 | return SaslMechanism.Gssapi; 113 | case "Plain": 114 | return SaslMechanism.Plain; 115 | case "ScramSha256": 116 | return SaslMechanism.ScramSha256; 117 | case "ScramSha512": 118 | return SaslMechanism.ScramSha512; 119 | case "OAuthBearer": 120 | return SaslMechanism.OAuthBearer; 121 | default: 122 | return default; 123 | } 124 | } 125 | 126 | internal IAdminClient GetNewAdminClient() 127 | { 128 | var config = new AdminClientConfig 129 | { 130 | BootstrapServers = _bootstrapServers, 131 | SecurityProtocol = GetSecurityProtocol(_securityProtocol), 132 | SaslMechanism = GetSaslMechanism(_saslMechanism), 133 | SaslUsername = _username, 134 | SaslPassword = _saslPassword 135 | }; 136 | return new AdminClientBuilder(config).Build(); 137 | } 138 | 139 | internal void SetNewAdminClient() 140 | { 141 | var config = new AdminClientConfig 142 | { 143 | BootstrapServers = _bootstrapServers, 144 | SecurityProtocol = GetSecurityProtocol(_securityProtocol), 145 | SaslMechanism = GetSaslMechanism(_saslMechanism), 146 | SaslUsername = _username, 147 | SaslPassword = _saslPassword 148 | }; 149 | AdminClient = new AdminClientBuilder(config).Build(); 150 | } 151 | 152 | 153 | internal readonly string _bootstrapServers; 154 | internal readonly string _securityProtocol; 155 | internal readonly string _saslMechanism; 156 | internal readonly string _username; 157 | internal readonly string _saslPassword; 158 | 159 | public static IAdminClient AdminClient { get; private set; } 160 | public static ProducerConfig ProducerConfig { get; private set; } 161 | public static ConsumerConfig ConsumerConfig { get; private set; } 162 | public static int RetryCount { get; private set; } 163 | public static int TimeToRetryInSeconds { get; private set; } 164 | } 165 | } -------------------------------------------------------------------------------- /StreamNet/Startup.cs: -------------------------------------------------------------------------------- 1 | using StreamNet.Consumers; 2 | using StreamNet.Producers; 3 | using Microsoft.Extensions.DependencyInjection; 4 | using StreamNet.Topic; 5 | 6 | namespace StreamNet 7 | { 8 | public static class Startup 9 | { 10 | public static IServiceCollection AddConsumer(this IServiceCollection services) 11 | where TConsumer : Consumer 12 | { 13 | services.AddHostedService(); 14 | return services; 15 | } 16 | 17 | public static IServiceCollection AddProducer(this IServiceCollection services) 18 | { 19 | services.AddTransient(); 20 | return services; 21 | } 22 | 23 | public static IServiceCollection AddTopicManagement(this IServiceCollection services) 24 | { 25 | services.AddTransient(); 26 | return services; 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /StreamNet/StreamNet.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netstandard2.1 5 | 1.6.3 6 | enable 7 | StreamNet 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /StreamNet/Topic/ITopicManagement.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Threading.Tasks; 3 | 4 | namespace StreamNet.Topic 5 | { 6 | public interface ITopicManagement 7 | { 8 | Task CreateTopicAsync(string topicName, short? replicationFactor = null, int? numberOfPartitions = null); 9 | 10 | void DeleteTopicAsync(string topicName); 11 | 12 | void DeleteTopicsAsync(IEnumerable topicNames); 13 | 14 | TopicData GetTopicData(string topicName); 15 | 16 | TopicData GetTopicsData(); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /StreamNet/Topic/TopicData.cs: -------------------------------------------------------------------------------- 1 | using Confluent.Kafka; 2 | using System.Collections.Generic; 3 | 4 | namespace StreamNet.Topic 5 | { 6 | public class TopicData 7 | { 8 | public TopicData(IEnumerable data) => Data = data; 9 | 10 | public IEnumerable Data { get; private set; } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /StreamNet/Topic/TopicManagement.cs: -------------------------------------------------------------------------------- 1 | using Confluent.Kafka; 2 | using Confluent.Kafka.Admin; 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Linq; 6 | using System.Threading.Tasks; 7 | 8 | namespace StreamNet.Topic 9 | { 10 | internal class TopicManagement : ITopicManagement 11 | { 12 | public TopicManagement() 13 | { 14 | Settings.GetInstance(); 15 | } 16 | 17 | public async Task CreateTopicAsync(string topicName, short? replicationFactor = null, int? numberOfPartitions = null) 18 | { 19 | var topicSpecification = new TopicSpecification(); 20 | topicSpecification.Name = topicName; 21 | if (replicationFactor.HasValue) 22 | topicSpecification.ReplicationFactor = replicationFactor.Value; 23 | 24 | if (numberOfPartitions.HasValue) 25 | topicSpecification.NumPartitions = numberOfPartitions.Value; 26 | 27 | await CreateTopicAsync(topicSpecification); 28 | } 29 | 30 | public void DeleteTopicAsync(string topicName) 31 | { 32 | var generalConfig = Settings.GetInstance(); 33 | 34 | var config = new AdminClientConfig 35 | { 36 | BootstrapServers = generalConfig._bootstrapServers, 37 | SecurityProtocol = generalConfig.GetSecurityProtocol(generalConfig._securityProtocol), 38 | SaslMechanism = generalConfig.GetSaslMechanism(generalConfig._saslMechanism), 39 | SaslUsername = generalConfig._username, 40 | SaslPassword = generalConfig._saslPassword 41 | }; 42 | 43 | using (var adminClient = new AdminClientBuilder(config).Build()) 44 | { 45 | adminClient.DeleteTopicsAsync(new[] { topicName }, new DeleteTopicsOptions 46 | { 47 | OperationTimeout = TimeSpan.FromMilliseconds(2000), 48 | RequestTimeout = TimeSpan.FromMilliseconds(2000) 49 | }).Wait(); 50 | } 51 | } 52 | 53 | public void DeleteTopicsAsync(IEnumerable topicNames) 54 | { 55 | var generalConfig = Settings.GetInstance(); 56 | 57 | var config = new AdminClientConfig 58 | { 59 | BootstrapServers = generalConfig._bootstrapServers, 60 | SecurityProtocol = generalConfig.GetSecurityProtocol(generalConfig._securityProtocol), 61 | SaslMechanism = generalConfig.GetSaslMechanism(generalConfig._saslMechanism), 62 | SaslUsername = generalConfig._username, 63 | SaslPassword = generalConfig._saslPassword 64 | }; 65 | 66 | using (var adminClient = new AdminClientBuilder(config).Build()) 67 | { 68 | adminClient.DeleteTopicsAsync(topicNames, new DeleteTopicsOptions 69 | { 70 | OperationTimeout = TimeSpan.FromMilliseconds(2000), 71 | RequestTimeout = TimeSpan.FromMilliseconds(2000) 72 | }).Wait(); 73 | } 74 | } 75 | 76 | public TopicData GetTopicData(string topicName) 77 | { 78 | try 79 | { 80 | var generalConfig = Settings.GetInstance(); 81 | 82 | var config = new AdminClientConfig 83 | { 84 | BootstrapServers = generalConfig._bootstrapServers, 85 | SecurityProtocol = generalConfig.GetSecurityProtocol(generalConfig._securityProtocol), 86 | SaslMechanism = generalConfig.GetSaslMechanism(generalConfig._saslMechanism), 87 | SaslUsername = generalConfig._username, 88 | SaslPassword = generalConfig._saslPassword 89 | }; 90 | 91 | using (var adminClient = new AdminClientBuilder(config).Build()) 92 | { 93 | var fullDataTopic = Settings.AdminClient.GetMetadata(topicName, TimeSpan.FromMilliseconds(100)); 94 | return new TopicData(fullDataTopic?.Topics?.Where(x => !x.Error.IsError)); 95 | } 96 | } 97 | catch (Exception) 98 | { 99 | return new TopicData(Enumerable.Empty()); 100 | } 101 | } 102 | 103 | public TopicData GetTopicsData() 104 | { 105 | var fullDataTopic = Settings.AdminClient.GetMetadata(TimeSpan.FromMilliseconds(100)); 106 | return new TopicData(fullDataTopic.Topics); 107 | } 108 | 109 | private async Task CreateTopicAsync(TopicSpecification topicSpecification) => await Settings.AdminClient.CreateTopicsAsync(new[] { topicSpecification }); 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /StreamNet/UnitTestingHelpers/UnitTestDetector.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | 4 | namespace StreamNet.UnitTestingHelpers 5 | { 6 | public static class UnitTestDetector 7 | { 8 | private static bool _isCalled = false; 9 | public static bool IsRunningFromUnitTesting() 10 | { 11 | if (_isCalled) 12 | return _isCalled; 13 | 14 | _isCalled = true; 15 | if (AppDomain.CurrentDomain.GetAssemblies().Any(x => x.FullName.ToLowerInvariant().Contains("test"))) 16 | { 17 | return true; 18 | } 19 | 20 | if (AppDomain.CurrentDomain.GetAssemblies().Any(x => x.FullName.ToLowerInvariant().Contains("xunit"))) 21 | { 22 | return true; 23 | } 24 | 25 | if (AppDomain.CurrentDomain.GetAssemblies().Any(x => x.FullName.ToLowerInvariant().Contains("nunit"))) 26 | { 27 | return true; 28 | } 29 | 30 | if (AppDomain.CurrentDomain.GetAssemblies().Any(x => x.FullName.ToLowerInvariant().Contains("mstest"))) 31 | { 32 | return true; 33 | } 34 | 35 | return false; 36 | } 37 | } 38 | } --------------------------------------------------------------------------------