├── website ├── static │ ├── .nojekyll │ └── img │ │ └── favicon.ico ├── versions.json ├── babel.config.js ├── docs │ ├── guides │ │ ├── _category_.json │ │ ├── middlewares │ │ │ └── _category_.json │ │ ├── admin │ │ │ ├── _category_.json │ │ │ └── admin.md │ │ ├── consumers │ │ │ ├── _category_.json │ │ │ └── built-in-workers-algorithms │ │ │ │ └── _category_.json │ │ ├── migration │ │ │ └── _category_.json │ │ └── dependency-injection.md │ ├── reference │ │ ├── _category_.json │ │ ├── KafkaFlow │ │ │ └── _category_.json │ │ ├── KafkaFlow.Admin │ │ │ └── _category_.json │ │ ├── KafkaFlow.Unity │ │ │ └── _category_.json │ │ ├── KafkaFlow.BatchConsume │ │ │ └── _category_.json │ │ ├── KafkaFlow.Compressor │ │ │ └── _category_.json │ │ ├── KafkaFlow.Serializer │ │ │ └── _category_.json │ │ ├── KafkaFlow.SchemaRegistry │ │ │ └── _category_.json │ │ ├── KafkaFlow.TypedHandler │ │ │ └── _category_.json │ │ ├── KafkaFlow.Extensions.Hosting │ │ │ └── _category_.json │ │ ├── KafkaFlow.LogHandler.Console │ │ │ └── _category_.json │ │ ├── KafkaFlow.Serializer.JsonCore │ │ │ └── _category_.json │ │ ├── KafkaFlow.LogHandler.Microsoft │ │ │ └── _category_.json │ │ ├── KafkaFlow.Serializer.ProtobufNet │ │ │ └── _category_.json │ │ ├── KafkaFlow.Serializer.NewtonsoftJson │ │ │ └── _category_.json │ │ ├── KafkaFlow.Microsoft.DependencyInjection │ │ │ └── _category_.json │ │ ├── KafkaFlow.Serializer.SchemaRegistry.ConfluentAvro │ │ │ └── _category_.json │ │ ├── KafkaFlow.Serializer.SchemaRegistry.ConfluentJson │ │ │ └── _category_.json │ │ └── KafkaFlow.Serializer.SchemaRegistry.ConfluentProtobuf │ │ │ └── _category_.json │ └── getting-started │ │ └── _category_.json ├── versioned_docs │ └── version-2.x │ │ ├── guides │ │ ├── _category_.json │ │ ├── consumers │ │ │ └── _category_.json │ │ ├── middlewares │ │ │ └── _category_.json │ │ ├── admin │ │ │ ├── _category_.json │ │ │ └── admin.md │ │ └── dependency-injection.md │ │ ├── reference │ │ ├── _category_.json │ │ ├── KafkaFlow │ │ │ └── _category_.json │ │ ├── KafkaFlow.Admin │ │ │ └── _category_.json │ │ ├── KafkaFlow.Unity │ │ │ └── _category_.json │ │ ├── KafkaFlow.Compressor │ │ │ └── _category_.json │ │ ├── KafkaFlow.Serializer │ │ │ └── _category_.json │ │ ├── KafkaFlow.BatchConsume │ │ │ └── _category_.json │ │ ├── KafkaFlow.TypedHandler │ │ │ └── _category_.json │ │ ├── KafkaFlow.SchemaRegistry │ │ │ └── _category_.json │ │ ├── KafkaFlow.Extensions.Hosting │ │ │ └── _category_.json │ │ ├── KafkaFlow.LogHandler.Console │ │ │ └── _category_.json │ │ ├── KafkaFlow.LogHandler.Microsoft │ │ │ └── _category_.json │ │ ├── KafkaFlow.Serializer.JsonCore │ │ │ └── _category_.json │ │ ├── KafkaFlow.Serializer.ProtobufNet │ │ │ └── _category_.json │ │ ├── KafkaFlow.Serializer.NewtonsoftJson │ │ │ └── _category_.json │ │ ├── KafkaFlow.Microsoft.DependencyInjection │ │ │ └── _category_.json │ │ ├── KafkaFlow.Serializer.SchemaRegistry.ConfluentAvro │ │ │ └── _category_.json │ │ ├── KafkaFlow.Serializer.SchemaRegistry.ConfluentJson │ │ │ └── _category_.json │ │ └── KafkaFlow.Serializer.SchemaRegistry.ConfluentProtobuf │ │ │ └── _category_.json │ │ └── getting-started │ │ └── _category_.json ├── src │ ├── pages │ │ ├── markdown-page.md │ │ └── index.module.css │ └── components │ │ └── HomepageFeatures │ │ └── styles.module.css ├── .gitignore ├── sidebars.js └── README.md ├── src ├── KafkaFlow.Admin.Dashboard │ ├── ClientApp │ │ ├── src │ │ │ ├── app │ │ │ │ ├── app.component.html │ │ │ │ ├── api │ │ │ │ │ ├── models │ │ │ │ │ │ ├── consumer-group.ts │ │ │ │ │ │ ├── telemetry-response.ts │ │ │ │ │ │ ├── consumer.ts │ │ │ │ │ │ └── topic-partition-assignment.ts │ │ │ │ │ └── api-configuration.ts │ │ │ │ ├── app.component.ts │ │ │ │ ├── home │ │ │ │ │ ├── home.component.ts │ │ │ │ │ └── home.component.html │ │ │ │ ├── app-routing.module.ts │ │ │ │ ├── consumer │ │ │ │ │ └── shared │ │ │ │ │ │ ├── stop-modal │ │ │ │ │ │ ├── stop-modal.component.ts │ │ │ │ │ │ └── stop-modal.component.html │ │ │ │ │ │ ├── start-modal │ │ │ │ │ │ ├── start-modal.component.ts │ │ │ │ │ │ └── start-modal.component.html │ │ │ │ │ │ ├── restart-modal │ │ │ │ │ │ ├── restart-modal.component.ts │ │ │ │ │ │ └── restart-modal.component.html │ │ │ │ │ │ ├── pause-modal │ │ │ │ │ │ ├── pause-modal.component.ts │ │ │ │ │ │ └── pause-modal.component.html │ │ │ │ │ │ ├── reset-modal │ │ │ │ │ │ ├── reset-modal.component.ts │ │ │ │ │ │ └── reset-modal.component.html │ │ │ │ │ │ ├── resume-modal │ │ │ │ │ │ ├── resume-modal.component.ts │ │ │ │ │ │ └── resume-modal.component.html │ │ │ │ │ │ ├── rewind-modal │ │ │ │ │ │ └── rewind-modal.component.ts │ │ │ │ │ │ └── workers-count-modal │ │ │ │ │ │ ├── workers-count-modal.component.ts │ │ │ │ │ │ └── workers-count-modal.component.html │ │ │ │ ├── sort.pipe.ts │ │ │ │ └── group-by.pipe.ts │ │ │ ├── styles.scss │ │ │ ├── environments │ │ │ │ ├── environment.prod.ts │ │ │ │ └── environment.ts │ │ │ └── main.ts │ │ ├── webpack.config.js │ │ ├── .editorconfig │ │ ├── tsconfig.app.json │ │ ├── tsconfig.spec.json │ │ ├── .browserslistrc │ │ ├── .gitignore │ │ └── tsconfig.json │ ├── DashboardConfiguration.cs │ └── DashboardConfigurationBuilder.cs ├── KafkaFlow.LogHandler.Microsoft │ ├── AssemblyInfo.cs │ ├── ExtensionMethods.cs │ └── KafkaFlow.LogHandler.Microsoft.csproj ├── KafkaFlow.Serializer.SchemaRegistry.ConfluentAvro │ ├── AssemblyInfo.cs │ └── ConfluentAvroTypeNameResolver.cs ├── KafkaFlow.Serializer.SchemaRegistry.ConfluentProtobuf │ ├── AssemblyInfo.cs │ └── ConfluentProtobufDeserializer.cs ├── KafkaFlow │ ├── AssemblyInfo.cs │ ├── Consumers │ │ ├── IWorkerPoolFeeder.cs │ │ ├── ConsumerMiddlewareContext.cs │ │ ├── IOffsetManager.cs │ │ ├── IConsumerManager.cs │ │ ├── IOffsetCommitter.cs │ │ ├── NullOffsetManager.cs │ │ ├── IConsumerWorker.cs │ │ ├── IConsumerWorkerPool.cs │ │ ├── IConsumerMiddlewareContext.cs │ │ ├── NullOffsetCommitter.cs │ │ ├── ConsumerStatus.cs │ │ ├── ConsumerAccessor.cs │ │ ├── IConsumerAccessor.cs │ │ ├── IConsumerFlowManager.cs │ │ ├── OffsetsWatermark.cs │ │ └── TopicPartitionLag.cs │ ├── TopicPartitionMetadata.cs │ ├── DateTimeProvider.cs │ ├── IMiddlewareExecutor.cs │ ├── Configuration │ │ ├── IMiddlewareInstanceContainer.cs │ │ ├── ConsumerMiddlewareConfigurationBuilder.cs │ │ ├── ProducerMiddlewareConfigurationBuilder.cs │ │ ├── KafkaConfiguration.cs │ │ └── MiddlewareInstanceContainer.cs │ ├── IConsumerManagerFactory.cs │ ├── Extensions │ │ ├── MessageContextExtensions.cs │ │ ├── StrategyExtensions.cs │ │ └── ChannelExtensions.cs │ ├── Middlewares │ │ ├── ConsumerThrottling │ │ │ ├── IConsumerThrottlingThreshold.cs │ │ │ ├── ConsumerThrottlingDelayAction.cs │ │ │ ├── IConsumerThrottlingAction.cs │ │ │ ├── IConsumerThrottlingMetric.cs │ │ │ ├── Configuration │ │ │ │ ├── IConsumerThrottlingActionConfigurationBuilder.cs │ │ │ │ ├── IConsumerThrottlingActionsConfigurationBuilder.cs │ │ │ │ ├── ConsumerThrottlingConfiguration.cs │ │ │ │ ├── ConsumerThrottlingActionConfigurationBuilder.cs │ │ │ │ └── IConsumerThrottlingThresholdActionConfigurationBuilder.cs │ │ │ ├── ConsumerThrottlingThreshold.cs │ │ │ └── ConsumerThrottlingKafkaLagMetric.cs │ │ ├── TypedHandler │ │ │ ├── Configuration │ │ │ │ └── TypedHandlerConfiguration.cs │ │ │ ├── HandlerTypeMapping.cs │ │ │ └── HandlerExecutor.cs │ │ └── Serializer │ │ │ └── Resolvers │ │ │ └── IMessageTypeResolver.cs │ ├── EventSubscription.cs │ ├── TopicMetadata.cs │ ├── Producers │ │ ├── ProducerContext.cs │ │ ├── BatchProduceException.cs │ │ └── ProducerAccessor.cs │ ├── Clusters │ │ ├── ClusterManagerAccessor.cs │ │ └── IClusterManagerAccessor.cs │ ├── Authentication │ │ └── OAuthBearerAuthenticator.cs │ ├── Delegates.cs │ ├── KafkaFlow.csproj │ └── IKafkaBus.cs ├── KafkaFlow.Admin │ ├── AssemblyInfo.cs │ ├── Messages │ │ ├── IAdminMessage.cs │ │ ├── StartConsumerByName.cs │ │ ├── StopConsumerByName.cs │ │ ├── RestartConsumerByName.cs │ │ ├── ChangeConsumerWorkersCount.cs │ │ ├── PauseConsumersByGroup.cs │ │ ├── PauseConsumerByName.cs │ │ ├── ResumeConsumersByGroup.cs │ │ ├── ResumeConsumerByName.cs │ │ ├── ResetConsumerOffset.cs │ │ └── RewindConsumerOffsetToDateTime.cs │ ├── ITelemetryScheduler.cs │ ├── Extensions │ │ └── MessageConsumerExtensions.cs │ ├── IAdminProducer.cs │ ├── Handlers │ │ ├── ConsumerTelemetryMetricHandler.cs │ │ ├── StopConsumerByNameHandler.cs │ │ ├── StartConsumerByNameHandler.cs │ │ ├── RestartConsumerByNameHandler.cs │ │ ├── ChangeConsumerWorkersCountHandler.cs │ │ ├── PauseConsumerByNameHandler.cs │ │ └── ResumeConsumerByNameHandler.cs │ ├── KafkaFlow.Admin.csproj │ ├── AdminProducer.cs │ └── ITelemetryStorage.cs ├── KafkaFlow.Abstractions │ ├── Configuration │ │ ├── SaslOauthbearerMethod.cs │ │ ├── IConsumerMiddlewareConfigurationBuilder.cs │ │ ├── IProducerMiddlewareConfigurationBuilder.cs │ │ ├── SslEndpointIdentificationAlgorithm.cs │ │ ├── SecurityProtocol.cs │ │ ├── TopicPartitions.cs │ │ └── SaslMechanism.cs │ ├── IEventSubscription.cs │ ├── AutoOffsetReset.cs │ ├── ISerializerContext.cs │ ├── Acks.cs │ ├── IDateTimeProvider.cs │ ├── IDependencyResolverScope.cs │ ├── ICompressor.cs │ ├── IDecompressor.cs │ ├── ConsumerInitialState.cs │ ├── IMessageMiddleware.cs │ ├── KafkaFlow.Abstractions.csproj │ ├── IOffsetsWatermark.cs │ ├── Delegates.cs │ ├── Extensions │ │ └── DependencyResolverExtensions.cs │ ├── InstanceLifetime.cs │ ├── MessageEventContext.cs │ ├── ISerializer.cs │ ├── IDeserializer.cs │ ├── IMessageHeaders.cs │ ├── Message.cs │ ├── MessageErrorEventContext.cs │ ├── SerializerContext.cs │ ├── IMessageHandler.cs │ ├── NullLogHandler.cs │ ├── Consumers │ │ └── IWorker.cs │ ├── TopicPartition.cs │ ├── IProducerContext.cs │ ├── IWorkerDistributionStrategy.cs │ └── IEvent.cs ├── KafkaFlow.Admin.WebApi │ ├── Contracts │ │ ├── ResetOffsetsRequest.cs │ │ ├── ChangeWorkersCountRequest.cs │ │ ├── RewindOffsetsToDateRequest.cs │ │ ├── GroupsResponse.cs │ │ ├── ConsumersResponse.cs │ │ └── GroupResponse.cs │ ├── KafkaFlow.Admin.WebApi.csproj │ ├── Properties │ │ └── launchSettings.json │ └── Adapters │ │ └── ConsumerResponseAdapter.cs ├── KafkaFlow.OpenTelemetry │ ├── KafkaFlow.OpenTelemetry.csproj │ ├── KafkaFlowInstrumentationOptions.cs │ └── KafkaFlowInstrumentation.cs ├── KafkaFlow.Compressor.Gzip │ ├── KafkaFlow.Compressor.Gzip.csproj │ ├── GzipMessageCompressor.cs │ └── GzipMessageDecompressor.cs ├── KafkaFlow.LogHandler.Console │ ├── KafkaFlow.LogHandler.Console.csproj │ └── ExtensionMethods.cs ├── KafkaFlow.Serializer.ProtobufNet │ ├── ProtobufNetSerializer.cs │ ├── ProtobufNetDeserializer.cs │ └── KafkaFlow.Serializer.ProtobufNet.csproj ├── KafkaFlow.Unity │ ├── UnityDependencyResolverScope.cs │ └── KafkaFlow.Unity.csproj ├── KafkaFlow.SchemaRegistry │ ├── ISchemaRegistryTypeNameResolver.cs │ ├── KafkaFlow.SchemaRegistry.csproj │ └── ClusterConfigurationBuilderExtensions.cs ├── KafkaFlow.Microsoft.DependencyInjection │ ├── MicrosoftDependencyResolverScope.cs │ ├── KafkaFlow.Microsoft.DependencyInjection.csproj │ ├── ServiceProviderExtensions.cs │ ├── MicrosoftDependencyResolver.cs │ └── ServiceCollectionExtensions.cs ├── KafkaFlow.Extensions.Hosting │ ├── KafkaFlowHostedService.cs │ ├── KafkaFlow.Extensions.Hosting.csproj │ └── ServiceCollectionExtensions.cs ├── stylecop.json ├── KafkaFlow.Serializer.JsonCore │ └── KafkaFlow.Serializer.JsonCore.csproj ├── KafkaFlow.Serializer.NewtonsoftJson │ └── KafkaFlow.Serializer.NewtonsoftJson.csproj └── KafkaFlow.Serializer.SchemaRegistry.ConfluentJson │ ├── KafkaFlow.Serializer.SchemaRegistry.ConfluentJson.csproj │ └── ConsumerConfigurationBuilderExtensions.cs ├── samples ├── KafkaFlow.Sample.ConsumerThrottling │ ├── TestMessage.cs │ └── ProcessMessagesMiddleware.cs ├── KafkaFlow.Sample.BatchOperations │ ├── SampleBatchMessage.cs │ ├── PrintConsoleMiddleware.cs │ └── README.md ├── KafkaFlow.Sample.PauseConsumerOnError │ ├── WelcomeMessage.cs │ └── MessageHandler.cs ├── KafkaFlow.Sample.SchemaRegistry │ ├── MessageTypes │ │ ├── Protobuf │ │ │ └── protobufLogMessage.proto │ │ ├── Avro │ │ │ ├── AvroLogMessage2.avsc │ │ │ ├── AvroLogMessage.avsc │ │ │ └── SchemaRegistry │ │ │ │ └── LogLevel.cs │ │ └── Json │ │ │ └── JsonLogMessage.cs │ ├── Handlers │ │ ├── JsonMessageHandler.cs │ │ ├── AvroMessageHandler2.cs │ │ ├── ProtobufMessageHandler.cs │ │ └── AvroMessageHandler.cs │ └── README.md ├── KafkaFlow.Sample.FlowControl │ ├── SampleMessage.cs │ ├── PrintConsoleMiddleware.cs │ └── README.md ├── KafkaFlow.Sample │ ├── TestMessage.cs │ ├── PrintConsoleHandler.cs │ └── README.md ├── KafkaFlow.Sample.Dashboard │ ├── appsettings.Development.json │ ├── appsettings.json │ ├── Program.cs │ ├── Properties │ │ └── launchSettings.json │ ├── KafkaFlow.Sample.Dashboard.csproj │ └── README.md ├── KafkaFlow.Sample.WebApi │ ├── appsettings.Development.json │ ├── appsettings.json │ ├── Properties │ │ └── launchSettings.json │ └── README.md ├── KafkaFlow.Sample.OpenTelemetry │ ├── TestMessage.cs │ └── PrintConsoleHandler.cs ├── KafkaFlow.Sample.CooperativeSticky │ ├── TestMessage.cs │ ├── PrintConsoleHandler.cs │ └── README.md └── KafkaFlow.Sample.WildcardConsumer │ └── PrintConsoleMiddleware.cs ├── tests ├── KafkaFlow.IntegrationTests │ ├── Core │ │ ├── Producers │ │ │ ├── NullProducer.cs │ │ │ ├── AvroProducer.cs │ │ │ ├── GzipProducer.cs │ │ │ ├── JsonProducer.cs │ │ │ ├── JsonProducer2.cs │ │ │ ├── JsonGzipProducer.cs │ │ │ ├── ProtobufProducer.cs │ │ │ ├── ConfluentJsonProducer.cs │ │ │ ├── ProtobufGzipProducer.cs │ │ │ ├── ProtobufGzipProducer2.cs │ │ │ └── ConfluentProtobufProducer.cs │ │ ├── Messages │ │ │ ├── ITestMessage.cs │ │ │ ├── logmessages2.avsc │ │ │ ├── TestMessage3.cs │ │ │ ├── TestMessage1.cs │ │ │ ├── TestMessage2.cs │ │ │ └── PauseResumeMessage.cs │ │ ├── Exceptions │ │ │ ├── ErrorExecutingMiddlewareException.cs │ │ │ └── PartitionAssignmentException.cs │ │ ├── Handlers │ │ │ ├── AvroMessageHandler.cs │ │ │ ├── MessageHandler.cs │ │ │ ├── MessageHandler1.cs │ │ │ ├── ConfluentJsonMessageHandler.cs │ │ │ ├── ConfluentProtobufMessageHandler.cs │ │ │ ├── MessageHandler2.cs │ │ │ └── PauseResumeHandler.cs │ │ └── Middlewares │ │ │ ├── NullHandlerMiddleware.cs │ │ │ └── GzipMiddleware.cs │ └── conf │ │ └── appsettings.json └── KafkaFlow.UnitTests │ ├── DummyObjects │ └── DummyProtobufObject.proto │ ├── ExtensionHelpers.cs │ ├── ConfigurationBuilders │ └── KafkaConfigurationBuilderTests.cs │ └── Factories │ └── WorkerFactory.cs ├── commitlint.config.mjs ├── .github ├── ISSUE_TEMPLATE │ └── config.yml └── PULL_REQUEST_TEMPLATE.md └── Makefile /website/static/.nojekyll: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /website/versions.json: -------------------------------------------------------------------------------- 1 | [ 2 | "2.x" 3 | ] -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/app.component.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.ConsumerThrottling/TestMessage.cs: -------------------------------------------------------------------------------- 1 | public record TestMessage(string Text); 2 | -------------------------------------------------------------------------------- /website/static/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Farfetch/kafkaflow/HEAD/website/static/img/favicon.ico -------------------------------------------------------------------------------- /website/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [require.resolve('@docusaurus/core/lib/babel/preset')], 3 | }; 4 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/styles.scss: -------------------------------------------------------------------------------- 1 | 2 | /* Importing Bootstrap SCSS file. */ 3 | @import '~bootstrap/scss/bootstrap'; 4 | -------------------------------------------------------------------------------- /website/docs/guides/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "Guides", 3 | "position": 3, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/KafkaFlow.LogHandler.Microsoft/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.CompilerServices; 2 | 3 | [assembly: InternalsVisibleTo("KafkaFlow.UnitTests")] 4 | -------------------------------------------------------------------------------- /website/docs/reference/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "Reference", 3 | "position": 4, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/environments/environment.prod.ts: -------------------------------------------------------------------------------- 1 | export const environment = { 2 | production: true, 3 | apiUrl: "/kafkaflow" 4 | }; 5 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Producers/NullProducer.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.IntegrationTests.Core.Producers; 2 | 3 | public class NullProducer 4 | { 5 | 6 | } -------------------------------------------------------------------------------- /website/docs/guides/middlewares/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "Middlewares", 3 | "position": 4, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/docs/reference/KafkaFlow/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow", 3 | "position": 1, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Producers/AvroProducer.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.IntegrationTests.Core.Producers; 2 | 3 | internal class AvroProducer 4 | { 5 | } 6 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Producers/GzipProducer.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.IntegrationTests.Core.Producers; 2 | 3 | internal class GzipProducer 4 | { 5 | } 6 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Producers/JsonProducer.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.IntegrationTests.Core.Producers; 2 | 3 | internal class JsonProducer 4 | { 5 | } 6 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Producers/JsonProducer2.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.IntegrationTests.Core.Producers; 2 | 3 | internal class JsonProducer2 4 | { 5 | } 6 | -------------------------------------------------------------------------------- /website/docs/getting-started/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "Getting Started", 3 | "position": 2, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/docs/guides/admin/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "Administration", 3 | "position": 8, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } -------------------------------------------------------------------------------- /website/docs/guides/consumers/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "Consumers", 3 | "position": 3, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } -------------------------------------------------------------------------------- /website/docs/guides/migration/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "Migration", 3 | "position": 12, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/guides/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "Guides", 3 | "position": 3, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/KafkaFlow.Serializer.SchemaRegistry.ConfluentAvro/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.CompilerServices; 2 | 3 | [assembly: InternalsVisibleTo("KafkaFlow.UnitTests")] 4 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Producers/JsonGzipProducer.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.IntegrationTests.Core.Producers; 2 | 3 | internal class JsonGzipProducer 4 | { 5 | } 6 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Producers/ProtobufProducer.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.IntegrationTests.Core.Producers; 2 | 3 | internal class ProtobufProducer 4 | { 5 | } 6 | -------------------------------------------------------------------------------- /src/KafkaFlow.Serializer.SchemaRegistry.ConfluentProtobuf/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.CompilerServices; 2 | 3 | [assembly: InternalsVisibleTo("KafkaFlow.UnitTests")] 4 | -------------------------------------------------------------------------------- /website/docs/reference/KafkaFlow.Admin/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Admin", 3 | "position": 2, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/docs/reference/KafkaFlow.Unity/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Unity", 3 | "position": 19, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/src/pages/markdown-page.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Markdown page example 3 | --- 4 | 5 | # Markdown page example 6 | 7 | You don't need React to write simple standalone pages. 8 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/reference/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "Reference", 3 | "position": 4, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Producers/ConfluentJsonProducer.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.IntegrationTests.Core.Producers; 2 | 3 | internal class ConfluentJsonProducer 4 | { 5 | } 6 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Producers/ProtobufGzipProducer.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.IntegrationTests.Core.Producers; 2 | 3 | internal class ProtobufGzipProducer 4 | { 5 | } 6 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Producers/ProtobufGzipProducer2.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.IntegrationTests.Core.Producers; 2 | 3 | internal class ProtobufGzipProducer2 4 | { 5 | } 6 | -------------------------------------------------------------------------------- /website/docs/reference/KafkaFlow.BatchConsume/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.BatchConsume", 3 | "position": 3, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/docs/reference/KafkaFlow.Compressor/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Compressor", 3 | "position": 4, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/docs/reference/KafkaFlow.Serializer/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Serializer", 3 | "position": 11, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/getting-started/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "Getting Started", 3 | "position": 2, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/guides/consumers/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "Consumers", 3 | "position": 3, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/guides/middlewares/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "Middlewares", 3 | "position": 4, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/reference/KafkaFlow/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow", 3 | "position": 1, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/KafkaFlow/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.CompilerServices; 2 | 3 | [assembly: InternalsVisibleTo("KafkaFlow.UnitTests")] 4 | [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] 5 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Producers/ConfluentProtobufProducer.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.IntegrationTests.Core.Producers; 2 | 3 | internal class ConfluentProtobufProducer 4 | { 5 | } 6 | -------------------------------------------------------------------------------- /website/docs/reference/KafkaFlow.SchemaRegistry/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.SchemaRegistry", 3 | "position": 10, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/docs/reference/KafkaFlow.TypedHandler/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.TypedHandler", 3 | "position": 18, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/guides/admin/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "Administration", 3 | "position": 8, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.BatchOperations/SampleBatchMessage.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Sample.BatchOperations; 2 | 3 | internal class SampleBatchMessage 4 | { 5 | public string Text { get; set; } 6 | } -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.PauseConsumerOnError/WelcomeMessage.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Sample.PauseConsumerOnError; 2 | 3 | public class WelcomeMessage 4 | { 5 | public string? Text { get; set; } 6 | } -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/reference/KafkaFlow.Admin/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Admin", 3 | "position": 2, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/reference/KafkaFlow.Unity/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Unity", 3 | "position": 19, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.CompilerServices; 2 | 3 | [assembly: InternalsVisibleTo("KafkaFlow.UnitTests")] 4 | [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] 5 | -------------------------------------------------------------------------------- /website/docs/reference/KafkaFlow.Extensions.Hosting/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Extensions.Hosting", 3 | "position": 6, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/docs/reference/KafkaFlow.LogHandler.Console/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.LogHandler.Console", 3 | "position": 7, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/docs/reference/KafkaFlow.Serializer.JsonCore/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Serializer.JsonCore", 3 | "position": 12, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/docs/reference/KafkaFlow.LogHandler.Microsoft/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.LogHandler.Microsoft", 3 | "position": 8, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/reference/KafkaFlow.Compressor/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Compressor", 3 | "position": 4, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/reference/KafkaFlow.Serializer/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Serializer", 3 | "position": 11, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/docs/reference/KafkaFlow.Serializer.ProtobufNet/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Serializer.ProtobufNet", 3 | "position": 14, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/reference/KafkaFlow.BatchConsume/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.BatchConsume", 3 | "position": 3, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/reference/KafkaFlow.TypedHandler/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.TypedHandler", 3 | "position": 18, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/docs/reference/KafkaFlow.Serializer.NewtonsoftJson/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Serializer.NewtonsoftJson", 3 | "position": 13, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/reference/KafkaFlow.SchemaRegistry/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.SchemaRegistry", 3 | "position": 10, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/docs/reference/KafkaFlow.Microsoft.DependencyInjection/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Microsoft.DependencyInjection", 3 | "position": 9, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/reference/KafkaFlow.Extensions.Hosting/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Extensions.Hosting", 3 | "position": 6, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/reference/KafkaFlow.LogHandler.Console/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.LogHandler.Console", 3 | "position": 7, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/conf/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Kafka": { 3 | "Brokers": "localhost:9092", 4 | "SecurityProtocol": "Plaintext" 5 | }, 6 | "SchemaRegistry": { 7 | "Url": "localhost:8081" 8 | } 9 | } -------------------------------------------------------------------------------- /website/docs/guides/consumers/built-in-workers-algorithms/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "Built-in Dynamic Workers Algorithms", 3 | "position": 4, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/reference/KafkaFlow.LogHandler.Microsoft/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.LogHandler.Microsoft", 3 | "position": 8, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/reference/KafkaFlow.Serializer.JsonCore/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Serializer.JsonCore", 3 | "position": 12, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.SchemaRegistry/MessageTypes/Protobuf/protobufLogMessage.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package SchemaRegistry; 4 | 5 | message ProtobufLogMessage { 6 | string Message = 1; 7 | int32 Code = 2; 8 | } 9 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/Messages/IAdminMessage.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Admin.Messages; 2 | 3 | /// 4 | /// The interface that every admin message must implement 5 | /// 6 | public interface IAdminMessage 7 | { 8 | } 9 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/reference/KafkaFlow.Serializer.ProtobufNet/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Serializer.ProtobufNet", 3 | "position": 14, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/KafkaFlow/Consumers/IWorkerPoolFeeder.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace KafkaFlow.Consumers; 4 | 5 | internal interface IWorkerPoolFeeder 6 | { 7 | void Start(); 8 | 9 | Task StopAsync(); 10 | } 11 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/reference/KafkaFlow.Serializer.NewtonsoftJson/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Serializer.NewtonsoftJson", 3 | "position": 13, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/docs/reference/KafkaFlow.Serializer.SchemaRegistry.ConfluentAvro/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Serializer.SchemaRegistry.ConfluentAvro", 3 | "position": 15, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/docs/reference/KafkaFlow.Serializer.SchemaRegistry.ConfluentJson/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Serializer.SchemaRegistry.ConfluentJson", 3 | "position": 16, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /commitlint.config.mjs: -------------------------------------------------------------------------------- 1 | export default { 2 | extends: ['@commitlint/config-conventional'], 3 | ignores: [(message) => /^Bumps \[.+]\(.+\) from .+ to .+\.$/m.test(message)], 4 | rules: { 5 | 'body-max-line-length': [0, 'always'], 6 | } 7 | } -------------------------------------------------------------------------------- /website/src/components/HomepageFeatures/styles.module.css: -------------------------------------------------------------------------------- 1 | .features { 2 | display: flex; 3 | align-items: center; 4 | padding: 2rem 0; 5 | width: 100%; 6 | } 7 | 8 | .featureSvg { 9 | height: 200px; 10 | width: 200px; 11 | } 12 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/reference/KafkaFlow.Microsoft.DependencyInjection/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Microsoft.DependencyInjection", 3 | "position": 9, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.FlowControl/SampleMessage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace KafkaFlow.Sample.FlowControl; 4 | 5 | internal class SampleMessage 6 | { 7 | public Guid Id { get; set; } 8 | 9 | public string Name { get; set; } 10 | } -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/ITelemetryScheduler.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Admin; 2 | 3 | internal interface ITelemetryScheduler 4 | { 5 | void Start(string telemetryId, string topicName, int topicPartition); 6 | 7 | void Stop(string telemetryId); 8 | } 9 | -------------------------------------------------------------------------------- /website/docs/reference/KafkaFlow.Serializer.SchemaRegistry.ConfluentProtobuf/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Serializer.SchemaRegistry.ConfluentProtobuf", 3 | "position": 17, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample/TestMessage.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.Serialization; 2 | 3 | namespace KafkaFlow.Sample; 4 | 5 | [DataContract] 6 | public class TestMessage 7 | { 8 | [DataMember(Order = 1)] 9 | public string Text { get; set; } 10 | } -------------------------------------------------------------------------------- /src/KafkaFlow/TopicPartitionMetadata.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow; 2 | 3 | public class TopicPartitionMetadata 4 | { 5 | public TopicPartitionMetadata(int id) 6 | { 7 | this.Id = id; 8 | } 9 | 10 | public int Id { get; } 11 | } 12 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.Dashboard/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft": "Warning", 6 | "Microsoft.Hosting.Lifetime": "Information" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.WebApi/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft": "Warning", 6 | "Microsoft.Hosting.Lifetime": "Information" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/reference/KafkaFlow.Serializer.SchemaRegistry.ConfluentAvro/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Serializer.SchemaRegistry.ConfluentAvro", 3 | "position": 15, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/reference/KafkaFlow.Serializer.SchemaRegistry.ConfluentJson/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Serializer.SchemaRegistry.ConfluentJson", 3 | "position": 16, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/api/models/consumer-group.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /* eslint-disable */ 3 | import { Consumer } from './consumer'; 4 | export interface ConsumerGroup { 5 | consumers?: null | Array; 6 | groupId: string; 7 | } 8 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/api/models/telemetry-response.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /* eslint-disable */ 3 | import { ConsumerGroup } from './consumer-group'; 4 | export interface TelemetryResponse { 5 | groups?: null | Array; 6 | } 7 | -------------------------------------------------------------------------------- /src/KafkaFlow/Consumers/ConsumerMiddlewareContext.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Consumers; 2 | 3 | internal class ConsumerMiddlewareContext : IConsumerMiddlewareContext 4 | { 5 | public IWorker Worker { get; set; } 6 | 7 | public IConsumer Consumer { get; set; } 8 | } 9 | -------------------------------------------------------------------------------- /src/KafkaFlow/DateTimeProvider.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace KafkaFlow; 4 | 5 | internal class DateTimeProvider : IDateTimeProvider 6 | { 7 | public DateTime UtcNow => DateTime.UtcNow; 8 | 9 | public DateTime MinValue => DateTime.MinValue; 10 | } 11 | -------------------------------------------------------------------------------- /src/KafkaFlow/IMiddlewareExecutor.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | 4 | namespace KafkaFlow; 5 | 6 | internal interface IMiddlewareExecutor 7 | { 8 | Task Execute(IMessageContext context, Func nextOperation); 9 | } 10 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/reference/KafkaFlow.Serializer.SchemaRegistry.ConfluentProtobuf/_category_.json: -------------------------------------------------------------------------------- 1 | { 2 | "label": "KafkaFlow.Serializer.SchemaRegistry.ConfluentProtobuf", 3 | "position": 17, 4 | "link": { 5 | "type": "generated-index" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.WebApi/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft": "Warning", 6 | "Microsoft.Hosting.Lifetime": "Information" 7 | } 8 | }, 9 | "AllowedHosts": "*" 10 | } 11 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.Dashboard/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft": "Warning", 6 | "Microsoft.Hosting.Lifetime": "Information" 7 | } 8 | }, 9 | "AllowedHosts": "*" 10 | } 11 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.OpenTelemetry/TestMessage.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.Serialization; 2 | 3 | namespace KafkaFlow.Sample.OpenTelemetry; 4 | 5 | [DataContract] 6 | public class TestMessage 7 | { 8 | [DataMember(Order = 1)] 9 | public string Text { get; set; } 10 | } -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/app.component.ts: -------------------------------------------------------------------------------- 1 | import { Component } from '@angular/core'; 2 | 3 | @Component({ 4 | selector: 'app-root', 5 | templateUrl: './app.component.html' 6 | }) 7 | export class AppComponent { 8 | title = 'dashboard'; 9 | } 10 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.CooperativeSticky/TestMessage.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.Serialization; 2 | 3 | namespace KafkaFlow.Sample.CooperativeSticky; 4 | 5 | [DataContract] 6 | public class TestMessage 7 | { 8 | [DataMember(Order = 1)] 9 | public string Text { get; set; } 10 | } -------------------------------------------------------------------------------- /src/KafkaFlow/Configuration/IMiddlewareInstanceContainer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace KafkaFlow.Configuration; 4 | 5 | internal interface IMiddlewareInstanceContainer 6 | { 7 | Guid Id { get; } 8 | 9 | IMessageMiddleware GetInstance(IDependencyResolver resolver); 10 | } 11 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/home/home.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, OnInit } from '@angular/core'; 2 | 3 | @Component({ 4 | selector: 'app-home', 5 | templateUrl: './home.component.html' 6 | }) 7 | export class HomeComponent { 8 | constructor() { 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/webpack.config.js: -------------------------------------------------------------------------------- 1 | const webpack = require('webpack'); 2 | module.exports = { 3 | plugins: [ 4 | new webpack.IgnorePlugin({ 5 | resourceRegExp: /^\.\/locale$/, 6 | contextRegExp: /moment$/, 7 | }) 8 | ], 9 | }; 10 | -------------------------------------------------------------------------------- /src/KafkaFlow/IConsumerManagerFactory.cs: -------------------------------------------------------------------------------- 1 | using KafkaFlow.Configuration; 2 | using KafkaFlow.Consumers; 3 | 4 | namespace KafkaFlow; 5 | 6 | internal interface IConsumerManagerFactory 7 | { 8 | IConsumerManager Create(IConsumerConfiguration configuration, IDependencyResolver resolver); 9 | } 10 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/api/models/consumer.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /* eslint-disable */ 3 | import { TopicPartitionAssignment } from './topic-partition-assignment'; 4 | export interface Consumer { 5 | assignments?: null | Array; 6 | name: string; 7 | } 8 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Messages/ITestMessage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace KafkaFlow.IntegrationTests.Core.Messages; 4 | 5 | internal interface ITestMessage 6 | { 7 | Guid Id { get; set; } 8 | 9 | string Value { get; set; } 10 | 11 | int Version { get; set; } 12 | } 13 | -------------------------------------------------------------------------------- /tests/KafkaFlow.UnitTests/DummyObjects/DummyProtobufObject.proto: -------------------------------------------------------------------------------- 1 | // can be generated using 2 | // protoc --csharp_out=. DummyProtobufObject.proto 3 | 4 | syntax = "proto3"; 5 | 6 | package KafkaFlow.UnitTests; 7 | 8 | message DummyProtobufObject { 9 | string field1 = 1; 10 | int32 field2 = 2; 11 | } 12 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.SchemaRegistry/MessageTypes/Avro/AvroLogMessage2.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "AvroLogMessage2", 4 | "namespace": "SchemaRegistry", 5 | "doc": "A simple log message.", 6 | "fields": [ 7 | { 8 | "name": "Message", 9 | "type": "string" 10 | } 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/Configuration/SaslOauthbearerMethod.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Configuration; 2 | 3 | /// SaslOauthbearerMethod enum values 4 | public enum SaslOauthbearerMethod 5 | { 6 | /// Default 7 | Default, 8 | 9 | /// Oidc 10 | Oidc, 11 | } 12 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/IEventSubscription.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow; 2 | 3 | /// 4 | /// Represents an Event subscription. 5 | /// 6 | public interface IEventSubscription 7 | { 8 | /// 9 | /// Cancels the subscription to the event. 10 | /// 11 | void Cancel(); 12 | } 13 | -------------------------------------------------------------------------------- /src/KafkaFlow/Consumers/IOffsetManager.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace KafkaFlow.Consumers; 4 | 5 | internal interface IOffsetManager 6 | { 7 | void Enqueue(IConsumerContext context); 8 | 9 | void MarkAsProcessed(IConsumerContext offset); 10 | 11 | Task WaitContextsCompletionAsync(); 12 | } 13 | -------------------------------------------------------------------------------- /src/KafkaFlow/Extensions/MessageContextExtensions.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Extensions; 2 | 3 | internal static class MessageContextExtensions 4 | { 5 | public static void Discard(this IConsumerContext context) 6 | { 7 | context.ShouldStoreOffset = false; 8 | context.Complete(); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Messages/logmessages2.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "LogMessages2", 4 | "namespace": "MessageTypes", 5 | "doc": "A simple log message type as used by this blog post.", 6 | "fields": [ 7 | { 8 | "name": "Message", 9 | "type": "string" 10 | } 11 | ] 12 | } -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Messages/TestMessage3.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace KafkaFlow.IntegrationTests.Core.Messages; 4 | 5 | internal class TestMessage3 : ITestMessage 6 | { 7 | public Guid Id { get; set; } 8 | 9 | public string Value { get; set; } 10 | 11 | public int Version { get; set; } 12 | } 13 | -------------------------------------------------------------------------------- /src/KafkaFlow/Middlewares/ConsumerThrottling/IConsumerThrottlingThreshold.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace KafkaFlow.Middlewares.ConsumerThrottling; 4 | 5 | internal interface IConsumerThrottlingThreshold 6 | { 7 | long ThresholdValue { get; } 8 | 9 | Task TryExecuteActionAsync(long metricValue); 10 | } 11 | -------------------------------------------------------------------------------- /website/.gitignore: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | /node_modules 3 | 4 | # Production 5 | /build 6 | 7 | # Generated files 8 | .docusaurus 9 | .cache-loader 10 | 11 | # Misc 12 | .DS_Store 13 | .env.local 14 | .env.development.local 15 | .env.test.local 16 | .env.production.local 17 | 18 | npm-debug.log* 19 | yarn-debug.log* 20 | yarn-error.log* 21 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/Configuration/IConsumerMiddlewareConfigurationBuilder.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Configuration; 2 | 3 | /// 4 | public interface IConsumerMiddlewareConfigurationBuilder 5 | : IMiddlewareConfigurationBuilder 6 | { 7 | } 8 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/Configuration/IProducerMiddlewareConfigurationBuilder.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Configuration; 2 | 3 | /// 4 | public interface IProducerMiddlewareConfigurationBuilder 5 | : IMiddlewareConfigurationBuilder 6 | { 7 | } 8 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/AutoOffsetReset.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow; 2 | 3 | /// AutoOffsetReset enum values 4 | public enum AutoOffsetReset 5 | { 6 | /// Only reads new messages in the topic 7 | Latest, 8 | 9 | /// Reads the topic from the beginning 10 | Earliest, 11 | } 12 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/app-routing.module.ts: -------------------------------------------------------------------------------- 1 | import { NgModule } from '@angular/core'; 2 | import { RouterModule, Routes } from '@angular/router'; 3 | 4 | const routes: Routes = []; 5 | 6 | @NgModule({ 7 | imports: [RouterModule.forRoot(routes)], 8 | exports: [RouterModule] 9 | }) 10 | export class AppRoutingModule { } 11 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/Configuration/SslEndpointIdentificationAlgorithm.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Configuration; 2 | 3 | /// SslEndpointIdentificationAlgorithm enum values 4 | public enum SslEndpointIdentificationAlgorithm 5 | { 6 | /// None 7 | None, 8 | 9 | /// Https 10 | Https, 11 | } 12 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/ISerializerContext.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow; 2 | 3 | /// 4 | /// Context for serialization and deserialization operations. 5 | /// 6 | public interface ISerializerContext 7 | { 8 | /// 9 | /// Gets the topic associated with the message 10 | /// 11 | string Topic { get; } 12 | } 13 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/Acks.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow; 2 | 3 | /// Acknowledge type 4 | public enum Acks 5 | { 6 | /// Only waits leader's acknowledge 7 | Leader, 8 | 9 | /// Waits acknowledge from all brokers 10 | All, 11 | 12 | /// Don't wait acknowledge 13 | None, 14 | } 15 | -------------------------------------------------------------------------------- /src/KafkaFlow/Consumers/IConsumerManager.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace KafkaFlow.Consumers; 4 | 5 | internal interface IConsumerManager 6 | { 7 | IWorkerPoolFeeder Feeder { get; } 8 | 9 | IConsumerWorkerPool WorkerPool { get; } 10 | 11 | IConsumer Consumer { get; } 12 | 13 | Task StartAsync(); 14 | 15 | Task StopAsync(); 16 | } 17 | -------------------------------------------------------------------------------- /src/KafkaFlow/Middlewares/TypedHandler/Configuration/TypedHandlerConfiguration.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace KafkaFlow.Middlewares.TypedHandler.Configuration; 4 | 5 | internal class TypedHandlerConfiguration 6 | { 7 | public HandlerTypeMapping HandlerMapping { get; } = new(); 8 | 9 | public Action OnNoHandlerFound { get; set; } = (_) => { }; 10 | } 11 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.SchemaRegistry/MessageTypes/Json/JsonLogMessage.cs: -------------------------------------------------------------------------------- 1 | using Newtonsoft.Json; 2 | 3 | namespace SchemaRegistry; 4 | 5 | /// 6 | /// A simple log message. 7 | /// 8 | public class JsonLogMessage 9 | { 10 | [JsonProperty] 11 | public string Message { get; set; } 12 | 13 | [JsonProperty] 14 | public string Type { get; set; } 15 | } 16 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/.editorconfig: -------------------------------------------------------------------------------- 1 | # Editor configuration, see https://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | charset = utf-8 6 | indent_style = space 7 | indent_size = 2 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [*.ts] 12 | quote_type = single 13 | 14 | [*.md] 15 | max_line_length = off 16 | trim_trailing_whitespace = false 17 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.WebApi/Contracts/ResetOffsetsRequest.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Admin.WebApi.Contracts; 2 | 3 | /// 4 | /// The request to reset the offsets 5 | /// 6 | public class ResetOffsetsRequest 7 | { 8 | /// 9 | /// Gets or sets a value indicating whether the confirmation 10 | /// 11 | public bool Confirm { get; set; } 12 | } 13 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.WebApi/Contracts/ChangeWorkersCountRequest.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Admin.WebApi.Contracts; 2 | 3 | /// 4 | /// The request to change the number of workers 5 | /// 6 | public class ChangeWorkersCountRequest 7 | { 8 | /// 9 | /// Gets or sets the workers count 10 | /// 11 | public int WorkersCount { get; set; } 12 | } 13 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Exceptions/ErrorExecutingMiddlewareException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace KafkaFlow.IntegrationTests.Core.Exceptions; 4 | 5 | public class ErrorExecutingMiddlewareException : Exception 6 | { 7 | public ErrorExecutingMiddlewareException(string middlewareName) 8 | : base($"Exception thrown executing {middlewareName}") 9 | { 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/tsconfig.app.json: -------------------------------------------------------------------------------- 1 | /* To learn more about this file see: https://angular.io/config/tsconfig. */ 2 | { 3 | "extends": "./tsconfig.json", 4 | "compilerOptions": { 5 | "outDir": "./out-tsc/app", 6 | "types": [] 7 | }, 8 | "files": [ 9 | "src/main.ts", 10 | "src/polyfills.ts" 11 | ], 12 | "include": [ 13 | "src/**/*.d.ts" 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/IDateTimeProvider.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace KafkaFlow; 4 | 5 | /// 6 | /// Provides access to DateTime static members 7 | /// 8 | public interface IDateTimeProvider 9 | { 10 | /// 11 | DateTime UtcNow { get; } 12 | 13 | /// 14 | DateTime MinValue { get; } 15 | } 16 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/api/api-configuration.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /* eslint-disable */ 3 | import { Injectable } from '@angular/core'; 4 | import { environment } from 'src/environments/environment'; 5 | 6 | /** 7 | * Global configuration 8 | */ 9 | @Injectable({ 10 | providedIn: 'root', 11 | }) 12 | export class ApiConfiguration { 13 | rootUrl: string = environment.apiUrl; 14 | } 15 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.WebApi/Contracts/RewindOffsetsToDateRequest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace KafkaFlow.Admin.WebApi.Contracts; 4 | 5 | /// 6 | /// The request to rewind offsets to a point in time 7 | /// 8 | public class RewindOffsetsToDateRequest 9 | { 10 | /// 11 | /// Gets or sets the point in time 12 | /// 13 | public DateTime Date { get; set; } 14 | } 15 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/IDependencyResolverScope.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace KafkaFlow; 4 | 5 | /// 6 | /// Represents the interface of a dependency injection resolver scope 7 | /// 8 | public interface IDependencyResolverScope : IDisposable 9 | { 10 | /// 11 | /// Gets the dependency injection resolver 12 | /// 13 | IDependencyResolver Resolver { get; } 14 | } 15 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/api/models/topic-partition-assignment.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /* eslint-disable */ 3 | export interface TopicPartitionAssignment { 4 | instanceName: string; 5 | lag?: number; 6 | workers: number; 7 | lastUpdate?: string; 8 | pausedPartitions?: null | Array; 9 | runningPartitions?: null | Array; 10 | status: string; 11 | topicName: string; 12 | } 13 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.WebApi/Contracts/GroupsResponse.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace KafkaFlow.Admin.WebApi.Contracts; 4 | 5 | /// 6 | /// The response of the consumer groups 7 | /// 8 | public class GroupsResponse 9 | { 10 | /// 11 | /// Gets or sets the groups collection 12 | /// 13 | public IEnumerable Groups { get; set; } 14 | } 15 | -------------------------------------------------------------------------------- /src/KafkaFlow/EventSubscription.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace KafkaFlow; 4 | 5 | internal class EventSubscription : IEventSubscription 6 | { 7 | private readonly Action _cancelDelegate; 8 | 9 | public EventSubscription(Action cancelDelegate) 10 | { 11 | _cancelDelegate = cancelDelegate; 12 | } 13 | 14 | public void Cancel() 15 | { 16 | _cancelDelegate.Invoke(); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Exceptions/PartitionAssignmentException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace KafkaFlow.IntegrationTests.Core.Exceptions; 4 | 5 | public class PartitionAssignmentException : Exception 6 | { 7 | private const string ExceptionMessage = "Partition assignment hasn't occurred yet."; 8 | 9 | public PartitionAssignmentException() 10 | : base(ExceptionMessage) 11 | { 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.WebApi/Contracts/ConsumersResponse.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace KafkaFlow.Admin.WebApi.Contracts; 4 | 5 | /// 6 | /// The response of the consumers 7 | /// 8 | public class ConsumersResponse 9 | { 10 | /// 11 | /// Gets or sets the consumers collection 12 | /// 13 | public IEnumerable Consumers { get; set; } 14 | } 15 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.Dashboard/Program.cs: -------------------------------------------------------------------------------- 1 | using KafkaFlow.Sample.Dashboard; 2 | using Microsoft.AspNetCore.Hosting; 3 | using Microsoft.Extensions.Hosting; 4 | 5 | await CreateHostBuilder(args) 6 | .Build() 7 | .RunAsync(); 8 | 9 | 10 | static IHostBuilder CreateHostBuilder(string[] args) => 11 | Host 12 | .CreateDefaultBuilder(args) 13 | .ConfigureWebHostDefaults(webBuilder => { webBuilder.UseStartup(); }); -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/ICompressor.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow; 2 | 3 | /// 4 | /// Used to create a message compressor 5 | /// 6 | public interface ICompressor 7 | { 8 | /// 9 | /// Compress the given message 10 | /// 11 | /// The message to be compressed 12 | /// The compressed message 13 | byte[] Compress(byte[] message); 14 | } 15 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Handlers/AvroMessageHandler.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using MessageTypes; 3 | 4 | namespace KafkaFlow.IntegrationTests.Core.Handlers; 5 | 6 | internal class AvroMessageHandler : IMessageHandler 7 | { 8 | public Task Handle(IMessageContext context, LogMessages2 message) 9 | { 10 | MessageStorage.Add(message); 11 | return Task.CompletedTask; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/Configuration/SecurityProtocol.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Configuration; 2 | 3 | /// SecurityProtocol enum values 4 | public enum SecurityProtocol 5 | { 6 | /// Plaintext 7 | Plaintext, 8 | 9 | /// Ssl 10 | Ssl, 11 | 12 | /// SaslPlaintext 13 | SaslPlaintext, 14 | 15 | /// SaslSsl 16 | SaslSsl, 17 | } 18 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/Configuration/TopicPartitions.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace KafkaFlow.Configuration; 4 | 5 | public class TopicPartitions 6 | { 7 | public TopicPartitions(string name, IEnumerable partitions) 8 | { 9 | this.Name = name; 10 | this.Partitions = partitions; 11 | } 12 | 13 | public string Name { get; } 14 | 15 | public IEnumerable Partitions { get; } 16 | } 17 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/IDecompressor.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow; 2 | 3 | /// 4 | /// Used to create a message decompressor 5 | /// 6 | public interface IDecompressor 7 | { 8 | /// 9 | /// Decompress the given message 10 | /// 11 | /// The message to be decompressed 12 | /// The decompressed message 13 | byte[] Decompress(byte[] message); 14 | } 15 | -------------------------------------------------------------------------------- /src/KafkaFlow/Consumers/IOffsetCommitter.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Threading.Tasks; 3 | using KafkaFlow.Configuration; 4 | 5 | namespace KafkaFlow.Consumers; 6 | 7 | internal interface IOffsetCommitter 8 | { 9 | List PendingOffsetsStatisticsHandlers { get; } 10 | 11 | void MarkAsProcessed(TopicPartitionOffset tpo); 12 | 13 | Task StartAsync(); 14 | 15 | Task StopAsync(); 16 | } 17 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/ConsumerInitialState.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow; 2 | 3 | /// 4 | /// Consumer initial state options 5 | /// 6 | public enum ConsumerInitialState 7 | { 8 | /// 9 | /// Automatically start the consumer when the bus is started 10 | /// 11 | Running, 12 | 13 | /// 14 | /// Do not start the consumer when the bus is started 15 | /// 16 | Stopped, 17 | } 18 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/tsconfig.spec.json: -------------------------------------------------------------------------------- 1 | /* To learn more about this file see: https://angular.io/config/tsconfig. */ 2 | { 3 | "extends": "./tsconfig.json", 4 | "compilerOptions": { 5 | "outDir": "./out-tsc/spec", 6 | "types": [ 7 | "jasmine" 8 | ] 9 | }, 10 | "files": [ 11 | "src/test.ts", 12 | "src/polyfills.ts" 13 | ], 14 | "include": [ 15 | "src/**/*.spec.ts", 16 | "src/**/*.d.ts" 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /src/KafkaFlow/TopicMetadata.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace KafkaFlow; 4 | 5 | public record TopicMetadata 6 | { 7 | public TopicMetadata(string name, IReadOnlyCollection partitions) 8 | { 9 | this.Name = name; 10 | this.Partitions = partitions; 11 | } 12 | 13 | public string Name { get; } 14 | 15 | public IReadOnlyCollection Partitions { get; } 16 | } 17 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Handlers/MessageHandler.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using KafkaFlow.IntegrationTests.Core.Messages; 3 | 4 | namespace KafkaFlow.IntegrationTests.Core.Handlers; 5 | 6 | internal class MessageHandler : IMessageHandler 7 | { 8 | public Task Handle(IMessageContext context, TestMessage1 message) 9 | { 10 | MessageStorage.Add(message); 11 | return Task.CompletedTask; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/main.ts: -------------------------------------------------------------------------------- 1 | import { enableProdMode } from '@angular/core'; 2 | import { platformBrowserDynamic } from '@angular/platform-browser-dynamic'; 3 | 4 | import { AppModule } from './app/app.module'; 5 | import { environment } from './environments/environment'; 6 | 7 | if (environment.production) { 8 | enableProdMode(); 9 | } 10 | 11 | platformBrowserDynamic().bootstrapModule(AppModule) 12 | .catch(err => console.error(err)); 13 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Handlers/MessageHandler1.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using KafkaFlow.IntegrationTests.Core.Messages; 3 | 4 | namespace KafkaFlow.IntegrationTests.Core.Handlers; 5 | 6 | internal class MessageHandler1 : IMessageHandler 7 | { 8 | public Task Handle(IMessageContext context, TestMessage1 message) 9 | { 10 | MessageStorage.Add(message); 11 | return Task.CompletedTask; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/KafkaFlow.OpenTelemetry/KafkaFlow.OpenTelemetry.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /src/KafkaFlow/Configuration/ConsumerMiddlewareConfigurationBuilder.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Configuration; 2 | 3 | internal class ConsumerMiddlewareConfigurationBuilder 4 | : MiddlewareConfigurationBuilder, 5 | IConsumerMiddlewareConfigurationBuilder 6 | { 7 | public ConsumerMiddlewareConfigurationBuilder(IDependencyConfigurator dependencyConfigurator) 8 | : base(dependencyConfigurator) 9 | { 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/KafkaFlow/Configuration/ProducerMiddlewareConfigurationBuilder.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Configuration; 2 | 3 | internal class ProducerMiddlewareConfigurationBuilder 4 | : MiddlewareConfigurationBuilder, 5 | IProducerMiddlewareConfigurationBuilder 6 | { 7 | public ProducerMiddlewareConfigurationBuilder(IDependencyConfigurator dependencyConfigurator) 8 | : base(dependencyConfigurator) 9 | { 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/KafkaFlow/Consumers/NullOffsetManager.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace KafkaFlow.Consumers; 4 | 5 | internal class NullOffsetManager : IOffsetManager 6 | { 7 | public void Enqueue(IConsumerContext context) 8 | { 9 | // Do nothing 10 | } 11 | 12 | public void MarkAsProcessed(IConsumerContext offset) 13 | { 14 | // Do nothing 15 | } 16 | 17 | public Task WaitContextsCompletionAsync() => Task.CompletedTask; 18 | } 19 | -------------------------------------------------------------------------------- /website/docs/guides/admin/admin.md: -------------------------------------------------------------------------------- 1 | --- 2 | sidebar_position: 1 3 | --- 4 | 5 | # Administration 6 | 7 | In this section, we will learn about KafkaFlow administration features. 8 | 9 | KafkaFlow provides an administrative infrastructure that you can use to get details about KafkaFlow consumers and perform administration tasks over them. 10 | 11 | There are two methods you can use: 12 | - The [Administration Web API](web-api). 13 | - The [Dashboard](dashboard). 14 | 15 | 16 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Handlers/ConfluentJsonMessageHandler.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using KafkaFlow.IntegrationTests.Core.Messages; 3 | 4 | namespace KafkaFlow.IntegrationTests.Core.Handlers; 5 | 6 | internal class ConfluentJsonMessageHandler : IMessageHandler 7 | { 8 | public Task Handle(IMessageContext context, TestMessage3 message) 9 | { 10 | MessageStorage.Add(message); 11 | return Task.CompletedTask; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Messages/TestMessage1.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Runtime.Serialization; 3 | 4 | namespace KafkaFlow.IntegrationTests.Core.Messages; 5 | 6 | [DataContract] 7 | internal class TestMessage1 : ITestMessage 8 | { 9 | [DataMember(Order = 1)] 10 | public Guid Id { get; set; } 11 | 12 | [DataMember(Order = 2)] 13 | public string Value { get; set; } 14 | 15 | [DataMember(Order = 3)] 16 | public int Version { get; set; } 17 | } 18 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Messages/TestMessage2.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Runtime.Serialization; 3 | 4 | namespace KafkaFlow.IntegrationTests.Core.Messages; 5 | 6 | [DataContract] 7 | internal class TestMessage2 : ITestMessage 8 | { 9 | [DataMember(Order = 1)] 10 | public Guid Id { get; set; } 11 | 12 | [DataMember(Order = 2)] 13 | public string Value { get; set; } 14 | 15 | [DataMember(Order = 3)] 16 | public int Version { get; set; } 17 | } 18 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.FlowControl/PrintConsoleMiddleware.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using Newtonsoft.Json; 4 | 5 | namespace KafkaFlow.Sample.FlowControl; 6 | 7 | internal class PrintConsoleMiddleware : IMessageMiddleware 8 | { 9 | public Task Invoke(IMessageContext context, MiddlewareDelegate next) 10 | { 11 | Console.WriteLine($"Message: {JsonConvert.SerializeObject(context.Message.Value)}"); 12 | return Task.CompletedTask; 13 | } 14 | } -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/Messages/StartConsumerByName.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.Serialization; 2 | 3 | namespace KafkaFlow.Admin.Messages; 4 | 5 | /// 6 | /// The message that starts a consumer 7 | /// 8 | [DataContract] 9 | public class StartConsumerByName : IAdminMessage 10 | { 11 | /// 12 | /// Gets or sets the consumer name that will be started 13 | /// 14 | [DataMember(Order = 1)] 15 | public string ConsumerName { get; set; } 16 | } 17 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/Messages/StopConsumerByName.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.Serialization; 2 | 3 | namespace KafkaFlow.Admin.Messages; 4 | 5 | /// 6 | /// The message that stops a consumer 7 | /// 8 | [DataContract] 9 | public class StopConsumerByName : IAdminMessage 10 | { 11 | /// 12 | /// Gets or sets the consumer name that will be stopped 13 | /// 14 | [DataMember(Order = 1)] 15 | public string ConsumerName { get; set; } 16 | } 17 | -------------------------------------------------------------------------------- /tests/KafkaFlow.UnitTests/ExtensionHelpers.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | 4 | namespace KafkaFlow.UnitTests; 5 | 6 | public static class ExtensionHelpers 7 | { 8 | public static TaskCompletionSource WithTimeout(this TaskCompletionSource taskCompletionSource, int milliseconds) 9 | { 10 | Task.Delay(milliseconds).ContinueWith(_ => taskCompletionSource.TrySetException(new TimeoutException())); 11 | return taskCompletionSource; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.WebApi/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json.schemastore.org/launchsettings.json", 3 | "profiles": { 4 | "KafkaFlow.Admin.WebApi.Sample": { 5 | "commandName": "Project", 6 | "launchBrowser": true, 7 | "launchUrl": "swagger", 8 | "applicationUrl": "https://localhost:5001;http://localhost:5000", 9 | "environmentVariables": { 10 | "ASPNETCORE_ENVIRONMENT": "Development" 11 | } 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/guides/admin/admin.md: -------------------------------------------------------------------------------- 1 | --- 2 | sidebar_position: 1 3 | --- 4 | 5 | # Administration 6 | 7 | In this section, we will learn about KafkaFlow administration features. 8 | 9 | KafkaFlow provides an administrative infrastructure that you can use to get details about KafkaFlow consumers and perform administration tasks over them. 10 | 11 | There are two methods you can use: 12 | - The [Administration Web API](web-api). 13 | - The [Dashboard](dashboard). 14 | 15 | 16 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.ConsumerThrottling/ProcessMessagesMiddleware.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using KafkaFlow; 4 | 5 | public class ProcessMessagesMiddleware : IMessageMiddleware 6 | { 7 | public async Task Invoke(IMessageContext context, MiddlewareDelegate next) 8 | { 9 | await Task.Delay(1000); // simulate a message processing 10 | await Console.Out.WriteLineAsync($"{context.ConsumerContext.ConsumerName} messaged processed"); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/Messages/RestartConsumerByName.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.Serialization; 2 | 3 | namespace KafkaFlow.Admin.Messages; 4 | 5 | /// 6 | /// The message that destroy and recreates the internal consumer 7 | /// 8 | [DataContract] 9 | public class RestartConsumerByName : IAdminMessage 10 | { 11 | /// 12 | /// Gets or sets the consumer name 13 | /// 14 | [DataMember(Order = 1)] 15 | public string ConsumerName { get; set; } 16 | } 17 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Messages/PauseResumeMessage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Runtime.Serialization; 3 | 4 | namespace KafkaFlow.IntegrationTests.Core.Messages; 5 | 6 | [DataContract] 7 | internal class PauseResumeMessage : ITestMessage 8 | { 9 | [DataMember(Order = 1)] 10 | public Guid Id { get; set; } 11 | 12 | [DataMember(Order = 2)] 13 | public string Value { get; set; } 14 | 15 | [DataMember(Order = 3)] 16 | public int Version { get; set; } 17 | } 18 | -------------------------------------------------------------------------------- /src/KafkaFlow.Compressor.Gzip/KafkaFlow.Compressor.Gzip.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | KafkaFlow.Compressor.Gzip 6 | Gzip implementation for KafkaFlow compression middleware 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Handlers/ConfluentProtobufMessageHandler.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using KafkaFlow.IntegrationTests.Core.Messages; 3 | 4 | namespace KafkaFlow.IntegrationTests.Core.Handlers; 5 | 6 | internal class ConfluentProtobufMessageHandler : IMessageHandler 7 | { 8 | public Task Handle(IMessageContext context, TestProtoMessage message) 9 | { 10 | MessageStorage.Add(message); 11 | return Task.CompletedTask; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/Configuration/SaslMechanism.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Configuration; 2 | 3 | /// SaslMechanism enum values 4 | public enum SaslMechanism 5 | { 6 | /// GSSAPI 7 | Gssapi, 8 | 9 | /// PLAIN 10 | Plain, 11 | 12 | /// SCRAM-SHA-256 13 | ScramSha256, 14 | 15 | /// SCRAM-SHA-512 16 | ScramSha512, 17 | 18 | /// OAUTH-BEARER 19 | OAuthBearer, 20 | } 21 | -------------------------------------------------------------------------------- /src/KafkaFlow/Consumers/IConsumerWorker.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading; 3 | using System.Threading.Tasks; 4 | 5 | namespace KafkaFlow.Consumers; 6 | 7 | internal interface IConsumerWorker : IWorker, IDisposable 8 | { 9 | CancellationToken StopCancellationToken { get; } 10 | 11 | IDependencyResolver WorkerDependencyResolver { get; } 12 | 13 | ValueTask EnqueueAsync(IMessageContext context); 14 | 15 | Task StartAsync(CancellationToken stopCancellationToken); 16 | 17 | Task StopAsync(); 18 | } -------------------------------------------------------------------------------- /src/KafkaFlow.LogHandler.Console/KafkaFlow.LogHandler.Console.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | KafkaFlow 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Handlers/MessageHandler2.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using KafkaFlow.IntegrationTests.Core.Messages; 4 | 5 | namespace KafkaFlow.IntegrationTests.Core.Handlers; 6 | 7 | internal class MessageHandler2 : IMessageHandler 8 | { 9 | public async Task Handle(IMessageContext context, TestMessage2 message) 10 | { 11 | await Task.Delay(new Random().Next(1000)).ConfigureAwait(false); 12 | 13 | MessageStorage.Add(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/KafkaFlow/Middlewares/ConsumerThrottling/ConsumerThrottlingDelayAction.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | 4 | namespace KafkaFlow.Middlewares.ConsumerThrottling; 5 | 6 | internal class ConsumerThrottlingDelayAction : IConsumerThrottlingAction 7 | { 8 | private readonly TimeSpan _delay; 9 | 10 | public ConsumerThrottlingDelayAction(TimeSpan delay) 11 | { 12 | _delay = delay; 13 | } 14 | 15 | public Task ExecuteAsync() 16 | { 17 | return Task.Delay(_delay); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/consumer/shared/stop-modal/stop-modal.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, Input, OnInit } from '@angular/core'; 2 | import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'; 3 | 4 | @Component({ 5 | selector: 'app-stop-modal', 6 | templateUrl: './stop-modal.component.html' 7 | }) 8 | export class StopModalComponent { 9 | @Input() public groupId: string | undefined; 10 | @Input() public consumerName: string | undefined; 11 | 12 | constructor(public activeModal: NgbActiveModal) { } 13 | } 14 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/consumer/shared/start-modal/start-modal.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, Input, OnInit } from '@angular/core'; 2 | import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'; 3 | 4 | @Component({ 5 | selector: 'app-start-modal', 6 | templateUrl: './start-modal.component.html' 7 | }) 8 | export class StartModalComponent { 9 | @Input() public groupId: string | undefined; 10 | @Input() public consumerName: string | undefined; 11 | 12 | constructor(public activeModal: NgbActiveModal) { } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/consumer/shared/restart-modal/restart-modal.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, Input, OnInit } from '@angular/core'; 2 | import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'; 3 | 4 | @Component({ 5 | selector: 'app-restart-modal', 6 | templateUrl: './restart-modal.component.html' 7 | }) 8 | export class RestartModalComponent { 9 | @Input() public groupId: string | undefined; 10 | @Input() public consumerName: string | undefined; 11 | 12 | constructor(public activeModal: NgbActiveModal) { } 13 | } 14 | -------------------------------------------------------------------------------- /src/KafkaFlow.Serializer.ProtobufNet/ProtobufNetSerializer.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | using System.Threading.Tasks; 3 | 4 | namespace KafkaFlow.Serializer; 5 | 6 | /// 7 | /// A message serializer using protobuf-net library 8 | /// 9 | public class ProtobufNetSerializer : ISerializer 10 | { 11 | /// 12 | public Task SerializeAsync(object message, Stream output, ISerializerContext context) 13 | { 14 | ProtoBuf.Serializer.Serialize(output, message); 15 | 16 | return Task.CompletedTask; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/KafkaFlow/Producers/ProducerContext.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Producers; 2 | 3 | internal class ProducerContext : IProducerContext 4 | { 5 | public ProducerContext(string topic, IDependencyResolver producerDependencyResolver) 6 | { 7 | this.Topic = topic; 8 | this.DependencyResolver = producerDependencyResolver; 9 | } 10 | 11 | public string Topic { get; } 12 | 13 | public int? Partition { get; set; } 14 | 15 | public long? Offset { get; set; } 16 | 17 | public IDependencyResolver DependencyResolver { get; } 18 | } 19 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/Extensions/MessageConsumerExtensions.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using Confluent.Kafka; 4 | using KafkaFlow.Consumers; 5 | 6 | namespace KafkaFlow.Admin.Extensions; 7 | 8 | internal static class MessageConsumerExtensions 9 | { 10 | public static IReadOnlyList FilterAssigment(this IMessageConsumer consumer, IList topics) 11 | { 12 | return topics.Any() ? consumer.Assignment.Where(a => topics.Contains(a.Topic)).ToList() : consumer.Assignment; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/KafkaFlow.Serializer.ProtobufNet/ProtobufNetDeserializer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Threading.Tasks; 4 | 5 | namespace KafkaFlow.Serializer; 6 | 7 | /// 8 | /// A message deserializer using protobuf-net library 9 | /// 10 | public class ProtobufNetDeserializer : IDeserializer 11 | { 12 | /// 13 | public Task DeserializeAsync(Stream input, Type type, ISerializerContext context) 14 | { 15 | return Task.FromResult(ProtoBuf.Serializer.Deserialize(type, input)); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | contact_links: 3 | - name: Documentation 📚 4 | url: https://farfetch.github.io/kafkaflow/docs/ 5 | about: Check out the official docs for answers to common questions 6 | - name: Questions & discussions 🤔 7 | url: https://github.com/Farfetch/kafkaflow/discussions 8 | about: Ask questions, request features & discuss RFCs 9 | - name: Community Slack 💬 10 | url: https://join.slack.com/t/kafkaflow/shared_invite/zt-puihrtcl-NnnylPZloAiVlQfsw~RD6Q 11 | about: Community discussions and support -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample/PrintConsoleHandler.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | 4 | namespace KafkaFlow.Sample; 5 | 6 | public class PrintConsoleHandler : IMessageHandler 7 | { 8 | public Task Handle(IMessageContext context, TestMessage message) 9 | { 10 | Console.WriteLine( 11 | "Partition: {0} | Offset: {1} | Message: {2}", 12 | context.ConsumerContext.Partition, 13 | context.ConsumerContext.Offset, 14 | message.Text); 15 | 16 | return Task.CompletedTask; 17 | } 18 | } -------------------------------------------------------------------------------- /src/KafkaFlow.Unity/UnityDependencyResolverScope.cs: -------------------------------------------------------------------------------- 1 | using global::Unity; 2 | 3 | namespace KafkaFlow.Unity; 4 | 5 | internal class UnityDependencyResolverScope : IDependencyResolverScope 6 | { 7 | private readonly IUnityContainer _container; 8 | 9 | public UnityDependencyResolverScope(IUnityContainer container) 10 | { 11 | _container = container; 12 | this.Resolver = new UnityDependencyResolver(container); 13 | } 14 | 15 | public IDependencyResolver Resolver { get; } 16 | 17 | public void Dispose() => _container.Dispose(); 18 | } 19 | -------------------------------------------------------------------------------- /src/KafkaFlow.Unity/KafkaFlow.Unity.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | KafkaFlow.Unity 6 | Adapts KafkaFlow to use Unity Dependency Injection 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /src/KafkaFlow.SchemaRegistry/ISchemaRegistryTypeNameResolver.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace KafkaFlow; 4 | 5 | /// 6 | /// An interface to implement a type name resolver to messages serialized with schema registry serializers 7 | /// 8 | public interface ISchemaRegistryTypeNameResolver 9 | { 10 | /// 11 | /// Resolve the message type name of a schema 12 | /// 13 | /// Identifier of the schema 14 | /// 15 | Task ResolveAsync(int schemaId); 16 | } 17 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Middlewares/NullHandlerMiddleware.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using KafkaFlow.IntegrationTests.Core.Handlers; 3 | 4 | namespace KafkaFlow.IntegrationTests.Core.Middlewares; 5 | 6 | public sealed class NullHandlerMiddleware : IMessageMiddleware 7 | { 8 | public Task Invoke(IMessageContext context, MiddlewareDelegate next) 9 | { 10 | if (context.Message.Value is null) 11 | { 12 | MessageStorage.AddNullMessage(context.Message); 13 | } 14 | return next(context); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/IAdminProducer.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using KafkaFlow.Admin.Messages; 3 | 4 | namespace KafkaFlow.Admin; 5 | 6 | /// 7 | /// A special producer to publish admin messages 8 | /// 9 | internal interface IAdminProducer 10 | { 11 | /// 12 | /// Produces admin messages to be listened by other application instances 13 | /// 14 | /// A message that implements the 15 | /// 16 | Task ProduceAsync(IAdminMessage message); 17 | } 18 | -------------------------------------------------------------------------------- /src/KafkaFlow/Consumers/IConsumerWorkerPool.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Threading; 3 | using System.Threading.Tasks; 4 | using Confluent.Kafka; 5 | 6 | namespace KafkaFlow.Consumers; 7 | 8 | internal interface IConsumerWorkerPool 9 | { 10 | int CurrentWorkersCount { get; } 11 | 12 | Task StartAsync(IReadOnlyCollection partitions, int workersCount); 13 | 14 | Task StopAsync(); 15 | 16 | Task EnqueueAsync( 17 | ConsumeResult message, 18 | CancellationToken stopCancellationToken); 19 | } 20 | -------------------------------------------------------------------------------- /src/KafkaFlow/Middlewares/ConsumerThrottling/IConsumerThrottlingAction.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace KafkaFlow.Middlewares.ConsumerThrottling; 4 | 5 | /// 6 | /// Defines a throttling action that can be executed by a KafkaFlow consumer. 7 | /// 8 | public interface IConsumerThrottlingAction 9 | { 10 | /// 11 | /// Executes the action defined in the implementation of this interface. 12 | /// 13 | /// A representing the asynchronous operation. 14 | Task ExecuteAsync(); 15 | } 16 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.OpenTelemetry/PrintConsoleHandler.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | 4 | namespace KafkaFlow.Sample.OpenTelemetry; 5 | 6 | public class PrintConsoleHandler : IMessageHandler 7 | { 8 | public Task Handle(IMessageContext context, TestMessage message) 9 | { 10 | Console.WriteLine( 11 | "Partition: {0} | Offset: {1} | Message: {2}", 12 | context.ConsumerContext.Partition, 13 | context.ConsumerContext.Offset, 14 | message.Text); 15 | 16 | return Task.CompletedTask; 17 | } 18 | } -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/IMessageMiddleware.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace KafkaFlow; 4 | 5 | /// 6 | /// Used to create a message middleware 7 | /// 8 | public interface IMessageMiddleware 9 | { 10 | /// 11 | /// The method that is called when the middleware is invoked 12 | /// 13 | /// The message context 14 | /// A delegate to the next middleware 15 | /// 16 | Task Invoke(IMessageContext context, MiddlewareDelegate next); 17 | } 18 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/KafkaFlow.Abstractions.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | KafkaFlow 6 | KafkaFlow.Abstractions 7 | Contains all KafkaFlow extendable interfaces 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/consumer/shared/pause-modal/pause-modal.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, Input, OnInit } from '@angular/core'; 2 | import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'; 3 | 4 | @Component({ 5 | selector: 'app-pause-modal', 6 | templateUrl: './pause-modal.component.html' 7 | }) 8 | export class PauseModalComponent { 9 | @Input() public groupId: string | undefined; 10 | @Input() public consumerName: string | undefined; 11 | @Input() public topic: string | undefined; 12 | 13 | constructor(public activeModal: NgbActiveModal) { } 14 | } 15 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/consumer/shared/reset-modal/reset-modal.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, Input, OnInit } from '@angular/core'; 2 | import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'; 3 | 4 | @Component({ 5 | selector: 'app-reset-modal', 6 | templateUrl: './reset-modal.component.html' 7 | }) 8 | export class ResetModalComponent { 9 | @Input() public groupId: string | undefined; 10 | @Input() public consumerName: string | undefined; 11 | @Input() public topic: string | undefined; 12 | 13 | constructor(public activeModal: NgbActiveModal) { } 14 | } 15 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/consumer/shared/resume-modal/resume-modal.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, Input, OnInit } from '@angular/core'; 2 | import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'; 3 | 4 | @Component({ 5 | selector: 'app-resume-modal', 6 | templateUrl: './resume-modal.component.html' 7 | }) 8 | export class ResumeModalComponent { 9 | @Input() public groupId: string | undefined; 10 | @Input() public consumerName: string | undefined; 11 | @Input() public topic: string | undefined; 12 | 13 | constructor(public activeModal: NgbActiveModal) { } 14 | } 15 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.CooperativeSticky/PrintConsoleHandler.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | 4 | namespace KafkaFlow.Sample.CooperativeSticky; 5 | 6 | public class PrintConsoleHandler : IMessageHandler 7 | { 8 | public Task Handle(IMessageContext context, TestMessage message) 9 | { 10 | Console.WriteLine( 11 | "Partition: {0} | Offset: {1} | Message: {2}", 12 | context.ConsumerContext.Partition, 13 | context.ConsumerContext.Offset, 14 | message.Text); 15 | 16 | return Task.CompletedTask; 17 | } 18 | } -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.SchemaRegistry/MessageTypes/Avro/AvroLogMessage.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "AvroLogMessage", 4 | "namespace": "SchemaRegistry", 5 | "doc": "A simple log message.", 6 | "fields": [ 7 | { 8 | "name": "Severity", 9 | "type": { 10 | "type": "enum", 11 | "name": "LogLevel", 12 | "doc": "Enumerates the set of allowable log levels.", 13 | "symbols": [ 14 | "None", 15 | "Verbose", 16 | "Info", 17 | "Warning", 18 | "Error" 19 | ] 20 | } 21 | } 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /src/KafkaFlow.LogHandler.Console/ExtensionMethods.cs: -------------------------------------------------------------------------------- 1 | using KafkaFlow.Configuration; 2 | 3 | namespace KafkaFlow; 4 | 5 | /// 6 | /// No needed 7 | /// 8 | public static class ExtensionMethods 9 | { 10 | /// 11 | /// Configure KafkaFlow to use the Console to log messages 12 | /// 13 | /// The Kafka configuration builder 14 | /// 15 | public static IKafkaConfigurationBuilder UseConsoleLog(this IKafkaConfigurationBuilder builder) => 16 | builder.UseLogHandler(); 17 | } 18 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/IOffsetsWatermark.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow; 2 | 3 | /// 4 | /// Interface that represents the low and high watermark offsets of a topic/partition 5 | /// 6 | public interface IOffsetsWatermark 7 | { 8 | /// 9 | /// Gets the high watermark offset, which is the offset of the latest message in the topic/partition available for consumption + 1 10 | /// 11 | long High { get; } 12 | 13 | /// 14 | /// Gets the offset of the earliest message in the topic/partition 15 | /// 16 | long Low { get; } 17 | } 18 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/Handlers/ConsumerTelemetryMetricHandler.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using KafkaFlow.Admin.Messages; 3 | 4 | namespace KafkaFlow.Admin.Handlers; 5 | 6 | internal class ConsumerTelemetryMetricHandler : IMessageHandler 7 | { 8 | private readonly ITelemetryStorage _storage; 9 | 10 | public ConsumerTelemetryMetricHandler(ITelemetryStorage storage) => _storage = storage; 11 | 12 | public Task Handle(IMessageContext context, ConsumerTelemetryMetric message) 13 | { 14 | _storage.Put(message); 15 | return Task.CompletedTask; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Handlers/PauseResumeHandler.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using KafkaFlow.IntegrationTests.Core.Messages; 3 | 4 | namespace KafkaFlow.IntegrationTests.Core.Handlers; 5 | 6 | internal class PauseResumeHandler : IMessageHandler 7 | { 8 | public async Task Handle(IMessageContext context, PauseResumeMessage message) 9 | { 10 | context.ConsumerContext.Pause(); 11 | 12 | await Task.Delay(Bootstrapper.MaxPollIntervalMs + 1000); 13 | 14 | MessageStorage.Add(message); 15 | 16 | context.ConsumerContext.Resume(); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/Delegates.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace KafkaFlow; 4 | 5 | /// 6 | /// The delegate used to call the next middleware 7 | /// 8 | /// The message context to be passed to the next middleware 9 | public delegate Task MiddlewareDelegate(IMessageContext context); 10 | 11 | /// 12 | /// Defines a factory to create an instance of type 13 | /// 14 | /// A instance 15 | public delegate T Factory(IDependencyResolver resolver); 16 | -------------------------------------------------------------------------------- /src/KafkaFlow.LogHandler.Microsoft/ExtensionMethods.cs: -------------------------------------------------------------------------------- 1 | using KafkaFlow.Configuration; 2 | 3 | namespace KafkaFlow; 4 | 5 | /// 6 | /// No needed 7 | /// 8 | public static class ExtensionMethods 9 | { 10 | /// 11 | /// Configure KafkaFlow to use the Microsoft Logging framework to log messages. 12 | /// 13 | /// The Kafka configuration builder 14 | /// 15 | public static IKafkaConfigurationBuilder UseMicrosoftLog(this IKafkaConfigurationBuilder builder) => 16 | builder.UseLogHandler(); 17 | } 18 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.BatchOperations/PrintConsoleMiddleware.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Threading.Tasks; 4 | 5 | namespace KafkaFlow.Sample.BatchOperations; 6 | 7 | internal class PrintConsoleMiddleware : IMessageMiddleware 8 | { 9 | public Task Invoke(IMessageContext context, MiddlewareDelegate next) 10 | { 11 | var batch = context.GetMessagesBatch(); 12 | 13 | var text = string.Join( 14 | '\n', 15 | batch.Select(ctx => ((SampleBatchMessage)ctx.Message.Value).Text)); 16 | 17 | Console.WriteLine(text); 18 | 19 | return Task.CompletedTask; 20 | } 21 | } -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/KafkaFlow.Admin.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | KafkaFlow.Admin 6 | Allows KafkaFlow to use the Admin commands 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /src/KafkaFlow/Consumers/IConsumerMiddlewareContext.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Consumers; 2 | 3 | /// 4 | /// Provides access to the current consumer's middleware context. 5 | /// 6 | public interface IConsumerMiddlewareContext 7 | { 8 | /// 9 | /// Gets the current worker in the context. 10 | /// This property only returns values when inside a middleware with Worker lifetime; otherwise, it will return null. 11 | /// 12 | IWorker Worker { get; } 13 | 14 | /// 15 | /// Gets the current consumer in the context. 16 | /// 17 | IConsumer Consumer { get; } 18 | } 19 | -------------------------------------------------------------------------------- /src/KafkaFlow/Extensions/StrategyExtensions.cs: -------------------------------------------------------------------------------- 1 | using Confluent.Kafka; 2 | 3 | namespace KafkaFlow.Extensions; 4 | 5 | /// 6 | /// Strategy extension methods. 7 | /// 8 | public static class StrategyExtensions 9 | { 10 | /// 11 | /// Determine if the strategy is a "stop the world" behavior. 12 | /// 13 | /// Strategy 14 | /// 15 | public static bool IsStopTheWorldStrategy(this PartitionAssignmentStrategy? strategy) => 16 | strategy is null or PartitionAssignmentStrategy.Range or PartitionAssignmentStrategy.RoundRobin; 17 | } 18 | -------------------------------------------------------------------------------- /website/src/pages/index.module.css: -------------------------------------------------------------------------------- 1 | /** 2 | * CSS files with the .module.css suffix will be treated as CSS modules 3 | * and scoped locally. 4 | */ 5 | 6 | .heroBanner { 7 | padding: 4rem 0; 8 | text-align: center; 9 | position: relative; 10 | overflow: hidden; 11 | } 12 | 13 | @media screen and (max-width: 996px) { 14 | .heroBanner { 15 | padding: 2rem; 16 | } 17 | } 18 | 19 | .buttons { 20 | display: flex; 21 | align-items: center; 22 | justify-content: center; 23 | } 24 | 25 | .videoContainer { 26 | position: relative; 27 | overflow: hidden; 28 | width: 100%; 29 | max-width: 560px; 30 | margin: 0 auto; 31 | } 32 | -------------------------------------------------------------------------------- /src/KafkaFlow.Microsoft.DependencyInjection/MicrosoftDependencyResolverScope.cs: -------------------------------------------------------------------------------- 1 | using global::Microsoft.Extensions.DependencyInjection; 2 | 3 | namespace KafkaFlow; 4 | 5 | internal class MicrosoftDependencyResolverScope : IDependencyResolverScope 6 | { 7 | private readonly IServiceScope _scope; 8 | 9 | public MicrosoftDependencyResolverScope(IServiceScope scope) 10 | { 11 | _scope = scope; 12 | this.Resolver = new MicrosoftDependencyResolver(scope.ServiceProvider); 13 | } 14 | 15 | public IDependencyResolver Resolver { get; } 16 | 17 | public void Dispose() 18 | { 19 | _scope.Dispose(); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.WildcardConsumer/PrintConsoleMiddleware.cs: -------------------------------------------------------------------------------- 1 | using System.Text; 2 | using KafkaFlow; 3 | 4 | public class PrintConsoleMiddleware : IMessageMiddleware 5 | { 6 | public Task Invoke(IMessageContext context, MiddlewareDelegate next) 7 | { 8 | Console.WriteLine( 9 | "Topic: {0} | Partition: {1} | Offset: {2} | Message: {3}", 10 | context.ConsumerContext.Topic, 11 | context.ConsumerContext.Partition, 12 | context.ConsumerContext.Offset, 13 | Encoding.UTF8.GetString( 14 | (byte[])context.Message.Value)); 15 | 16 | return next(context); 17 | } 18 | } -------------------------------------------------------------------------------- /src/KafkaFlow.OpenTelemetry/KafkaFlowInstrumentationOptions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | 4 | namespace KafkaFlow.OpenTelemetry; 5 | 6 | /// 7 | /// The options to be included in KafkaFlow instrumentation 8 | /// 9 | public class KafkaFlowInstrumentationOptions 10 | { 11 | /// 12 | /// Gets or sets the Producer enricher 13 | /// 14 | public Action EnrichProducer { get; set; } 15 | 16 | /// 17 | /// Gets or sets the Consumer enricher 18 | /// 19 | public Action EnrichConsumer { get; set; } 20 | } 21 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.SchemaRegistry/Handlers/JsonMessageHandler.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using global::SchemaRegistry; 4 | 5 | namespace KafkaFlow.Sample.SchemaRegistry.Handlers; 6 | 7 | public class JsonMessageHandler : IMessageHandler 8 | { 9 | public Task Handle(IMessageContext context, JsonLogMessage message) 10 | { 11 | Console.WriteLine( 12 | "Partition: {0} | Offset: {1} | Message: {2} | Json", 13 | context.ConsumerContext.Partition, 14 | context.ConsumerContext.Offset, 15 | message.Message); 16 | 17 | return Task.CompletedTask; 18 | } 19 | } -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/Extensions/DependencyResolverExtensions.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow; 2 | 3 | /// 4 | /// Provides extension methods over 5 | /// 6 | public static class DependencyResolverExtensions 7 | { 8 | /// 9 | /// Resolve an instance of . 10 | /// 11 | /// Instance of 12 | /// The type to be resolved 13 | /// 14 | public static T Resolve(this IDependencyResolver resolver) => (T)resolver.Resolve(typeof(T)); 15 | } 16 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.SchemaRegistry/Handlers/AvroMessageHandler2.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using global::SchemaRegistry; 4 | 5 | namespace KafkaFlow.Sample.SchemaRegistry.Handlers; 6 | 7 | public class AvroMessageHandler2 : IMessageHandler 8 | { 9 | public Task Handle(IMessageContext context, AvroLogMessage2 message) 10 | { 11 | Console.WriteLine( 12 | "Partition: {0} | Offset: {1} | Message: {2} | Avro", 13 | context.ConsumerContext.Partition, 14 | context.ConsumerContext.Offset, 15 | message.Message); 16 | 17 | return Task.CompletedTask; 18 | } 19 | } -------------------------------------------------------------------------------- /src/KafkaFlow.Extensions.Hosting/KafkaFlowHostedService.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading; 3 | using System.Threading.Tasks; 4 | using global::Microsoft.Extensions.Hosting; 5 | 6 | namespace KafkaFlow; 7 | 8 | internal class KafkaFlowHostedService : IHostedService 9 | { 10 | private readonly IKafkaBus _kafkaBus; 11 | 12 | public KafkaFlowHostedService(IServiceProvider serviceProvider) => _kafkaBus = serviceProvider.CreateKafkaBus(); 13 | 14 | public Task StartAsync(CancellationToken cancellationToken) => _kafkaBus.StartAsync(cancellationToken); 15 | 16 | public Task StopAsync(CancellationToken cancellationToken) => _kafkaBus.StopAsync(); 17 | } 18 | -------------------------------------------------------------------------------- /src/stylecop.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/DotNetAnalyzers/StyleCopAnalyzers/master/StyleCop.Analyzers/StyleCop.Analyzers/Settings/stylecop.schema.json", 3 | "settings": { 4 | "documentationRules": { 5 | "documentInterfaces": false, 6 | "documentExposedElements": true, 7 | "documentInternalElements": false, 8 | "documentPrivateElements": false, 9 | "documentPrivateFields": false, 10 | "xmlHeader": false 11 | }, 12 | "layoutRules": { 13 | "newlineAtEndOfFile": "require" 14 | }, 15 | "orderingRules": { 16 | "systemUsingDirectivesFirst": true, 17 | "usingDirectivesPlacement": "outsideNamespace" 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.SchemaRegistry/Handlers/ProtobufMessageHandler.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using SchemaRegistry; 4 | 5 | namespace KafkaFlow.Sample.SchemaRegistry.Handlers; 6 | 7 | public class ProtobufMessageHandler : IMessageHandler 8 | { 9 | public Task Handle(IMessageContext context, ProtobufLogMessage message) 10 | { 11 | Console.WriteLine( 12 | "Partition: {0} | Offset: {1} | Message: {2} | Protobuf", 13 | context.ConsumerContext.Partition, 14 | context.ConsumerContext.Offset, 15 | message.Message); 16 | 17 | return Task.CompletedTask; 18 | } 19 | } -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/InstanceLifetime.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow; 2 | 3 | /// 4 | /// Specifies the lifetime of a class 5 | /// 6 | public enum InstanceLifetime 7 | { 8 | /// 9 | /// Specifies that a single instance of the class will be created for the entire application 10 | /// 11 | Singleton, 12 | 13 | /// 14 | /// Specifies that a new instance of the class will be created for each scope 15 | /// 16 | Scoped, 17 | 18 | /// 19 | /// Specifies that a new instance of the class will be created every time it is requested 20 | /// 21 | Transient, 22 | } 23 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.SchemaRegistry/Handlers/AvroMessageHandler.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using global::SchemaRegistry; 4 | 5 | namespace KafkaFlow.Sample.SchemaRegistry.Handlers; 6 | 7 | public class AvroMessageHandler : IMessageHandler 8 | { 9 | public Task Handle(IMessageContext context, AvroLogMessage message) 10 | { 11 | Console.WriteLine( 12 | "Partition: {0} | Offset: {1} | Message: {2} | Avro", 13 | context.ConsumerContext.Partition, 14 | context.ConsumerContext.Offset, 15 | message.Severity.ToString()); 16 | 17 | return Task.CompletedTask; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/Messages/ChangeConsumerWorkersCount.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.Serialization; 2 | 3 | namespace KafkaFlow.Admin.Messages; 4 | 5 | /// 6 | /// A message to change the worker count 7 | /// 8 | [DataContract] 9 | public class ChangeConsumerWorkersCount : IAdminMessage 10 | { 11 | /// 12 | /// Gets or sets the consumer name that will be affected 13 | /// 14 | [DataMember(Order = 1)] 15 | public string ConsumerName { get; set; } 16 | 17 | /// 18 | /// Gets or sets the number of workers 19 | /// 20 | [DataMember(Order = 2)] 21 | public int WorkersCount { get; set; } 22 | } 23 | -------------------------------------------------------------------------------- /src/KafkaFlow/Consumers/NullOffsetCommitter.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Threading.Tasks; 3 | using KafkaFlow.Configuration; 4 | 5 | namespace KafkaFlow.Consumers; 6 | 7 | internal class NullOffsetCommitter : IOffsetCommitter 8 | { 9 | public List PendingOffsetsStatisticsHandlers { get; } = new(); 10 | 11 | public void Dispose() 12 | { 13 | // Do nothing 14 | } 15 | 16 | public void MarkAsProcessed(TopicPartitionOffset tpo) 17 | { 18 | // Do nothing 19 | } 20 | 21 | public Task StartAsync() => Task.CompletedTask; 22 | 23 | public Task StopAsync() => Task.CompletedTask; 24 | } 25 | -------------------------------------------------------------------------------- /src/KafkaFlow/Middlewares/ConsumerThrottling/IConsumerThrottlingMetric.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace KafkaFlow.Middlewares.ConsumerThrottling; 4 | 5 | /// 6 | /// Defines a metric that is used by the KafkaFlow consumer throttling feature. 7 | /// 8 | public interface IConsumerThrottlingMetric 9 | { 10 | /// 11 | /// Retrieves the value of the metric defined in the implementation of this interface. 12 | /// 13 | /// A representing the asynchronous operation, with a result representing the metric's value. 14 | Task GetValueAsync(); 15 | } 16 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/MessageEventContext.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow; 2 | 3 | /// 4 | /// Represents a message context used in the events 5 | /// 6 | public class MessageEventContext 7 | { 8 | /// 9 | /// Initializes a new instance of the class. 10 | /// 11 | /// The message context 12 | public MessageEventContext(IMessageContext messageContext) 13 | { 14 | this.MessageContext = messageContext; 15 | } 16 | 17 | /// 18 | /// Gets the message context 19 | /// 20 | public IMessageContext MessageContext { get; } 21 | } 22 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/AdminProducer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using KafkaFlow.Admin.Messages; 4 | 5 | namespace KafkaFlow.Admin; 6 | 7 | internal class AdminProducer : IAdminProducer 8 | { 9 | private readonly IMessageProducer _producer; 10 | private readonly int _topicPartition; 11 | 12 | public AdminProducer(IMessageProducer producer, int topicPartition) 13 | { 14 | _producer = producer; 15 | _topicPartition = topicPartition; 16 | } 17 | 18 | public Task ProduceAsync(IAdminMessage message) => 19 | _producer.ProduceAsync(Guid.NewGuid().ToString(), message, partition: _topicPartition); 20 | } 21 | -------------------------------------------------------------------------------- /src/KafkaFlow/Consumers/ConsumerStatus.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Consumers; 2 | 3 | /// 4 | /// An enum with all consumer status 5 | /// 6 | public enum ConsumerStatus 7 | { 8 | /// 9 | /// When the consumer is stopped 10 | /// 11 | Stopped, 12 | 13 | /// 14 | /// When all consumer partitions are running 15 | /// 16 | Running, 17 | 18 | /// 19 | /// When the consumer has paused and running partitions at the same time 20 | /// 21 | PartiallyRunning, 22 | 23 | /// 24 | /// When all consumer partitions are paused 25 | /// 26 | Paused, 27 | } 28 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/ISerializer.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | using System.Threading.Tasks; 3 | 4 | namespace KafkaFlow; 5 | 6 | /// 7 | /// Used to implement a message serializer 8 | /// 9 | public interface ISerializer 10 | { 11 | /// 12 | /// Serializes the given message 13 | /// 14 | /// The message to be serialized 15 | /// A stream to write the serialized data 16 | /// An object containing metadata 17 | /// The serialized message 18 | Task SerializeAsync(object message, Stream output, ISerializerContext context); 19 | } 20 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/Handlers/StopConsumerByNameHandler.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using KafkaFlow.Admin.Messages; 3 | using KafkaFlow.Consumers; 4 | 5 | namespace KafkaFlow.Admin.Handlers; 6 | 7 | internal class StopConsumerByNameHandler : IMessageHandler 8 | { 9 | private readonly IConsumerAccessor _consumerAccessor; 10 | 11 | public StopConsumerByNameHandler(IConsumerAccessor consumerAccessor) => _consumerAccessor = consumerAccessor; 12 | 13 | public async Task Handle(IMessageContext context, StopConsumerByName message) 14 | { 15 | var consumer = _consumerAccessor[message.ConsumerName]; 16 | 17 | await consumer.StopAsync(); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/KafkaFlow.OpenTelemetry/KafkaFlowInstrumentation.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | 3 | namespace KafkaFlow.OpenTelemetry; 4 | 5 | /// 6 | /// KafkaFlow OTEL instrumentation properties 7 | /// 8 | public static class KafkaFlowInstrumentation 9 | { 10 | internal static readonly AssemblyName AssemblyName = typeof(KafkaFlowInstrumentation).Assembly.GetName(); 11 | internal static readonly string Version = AssemblyName.Version.ToString(); 12 | 13 | /// 14 | /// ActivitySource name to be used when adding 15 | /// KafkaFlow as source to an OTEL listener 16 | /// 17 | public static readonly string ActivitySourceName = AssemblyName.Name; 18 | } 19 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/Handlers/StartConsumerByNameHandler.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using KafkaFlow.Admin.Messages; 3 | using KafkaFlow.Consumers; 4 | 5 | namespace KafkaFlow.Admin.Handlers; 6 | 7 | internal class StartConsumerByNameHandler : IMessageHandler 8 | { 9 | private readonly IConsumerAccessor _consumerAccessor; 10 | 11 | public StartConsumerByNameHandler(IConsumerAccessor consumerAccessor) => _consumerAccessor = consumerAccessor; 12 | 13 | public async Task Handle(IMessageContext context, StartConsumerByName message) 14 | { 15 | var consumer = _consumerAccessor[message.ConsumerName]; 16 | 17 | await consumer.StartAsync(); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/KafkaFlow.Serializer.JsonCore/KafkaFlow.Serializer.JsonCore.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | KafkaFlow.Serializer.JsonCore 6 | JSON implementation for KafkaFlow serializer middleware using System.Text.Json 7 | KafkaFlow.Serializer 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /src/KafkaFlow.Serializer.ProtobufNet/KafkaFlow.Serializer.ProtobufNet.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | KafkaFlow.Serializer.ProtobufNet 6 | Protobuf implementation for KafkaFlow serializer middleware using protobuf-net 7 | KafkaFlow.Serializer 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.WebApi/Contracts/GroupResponse.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.ComponentModel.DataAnnotations; 3 | using Newtonsoft.Json; 4 | 5 | namespace KafkaFlow.Admin.WebApi.Contracts; 6 | 7 | /// 8 | /// The response of the consumer group 9 | /// 10 | public class GroupResponse 11 | { 12 | /// 13 | /// Gets or sets the consumer group id 14 | /// 15 | [Required] 16 | [JsonProperty(Required = Required.DisallowNull)] 17 | public string GroupId { get; set; } 18 | 19 | /// 20 | /// Gets or sets the consumers collection 21 | /// 22 | public IEnumerable Consumers { get; set; } 23 | } 24 | -------------------------------------------------------------------------------- /src/KafkaFlow/Consumers/ConsumerAccessor.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace KafkaFlow.Consumers; 4 | 5 | internal class ConsumerAccessor : IConsumerAccessor 6 | { 7 | private readonly IDictionary _consumers = new Dictionary(); 8 | 9 | public IEnumerable All => _consumers.Values; 10 | 11 | public IMessageConsumer this[string name] => this.GetConsumer(name); 12 | 13 | public IMessageConsumer GetConsumer(string name) => 14 | _consumers.TryGetValue(name, out var consumer) ? consumer : null; 15 | 16 | void IConsumerAccessor.Add(IMessageConsumer consumer) => _consumers.Add(consumer.ConsumerName, consumer); 17 | } 18 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/IDeserializer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Threading.Tasks; 4 | 5 | namespace KafkaFlow; 6 | 7 | /// 8 | /// Used to implement a message serializer 9 | /// 10 | public interface IDeserializer 11 | { 12 | /// 13 | /// Deserializes the given message 14 | /// 15 | /// A stream to read the data to be deserialized 16 | /// The type to be created 17 | /// An object containing metadata 18 | /// The deserialized message 19 | Task DeserializeAsync(Stream input, Type type, ISerializerContext context); 20 | } 21 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.WebApi/KafkaFlow.Admin.WebApi.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net8.0 5 | Library 6 | KafkaFlow.Admin.WebApi 7 | Allows KafkaFlow to use the Admin Web API 8 | true 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /src/KafkaFlow.Compressor.Gzip/GzipMessageCompressor.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | using System.IO.Compression; 3 | 4 | namespace KafkaFlow.Compressor.Gzip; 5 | 6 | /// 7 | /// A GZIP message compressor 8 | /// 9 | public class GzipMessageCompressor : ICompressor 10 | { 11 | /// 12 | public byte[] Compress(byte[] message) 13 | { 14 | using var inputStream = new MemoryStream(message); 15 | using var outputStream = new MemoryStream(); 16 | 17 | using (var gzipStream = new GZipStream(outputStream, CompressionLevel.Optimal)) 18 | { 19 | inputStream.CopyTo(gzipStream); 20 | } 21 | 22 | return outputStream.ToArray(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/KafkaFlow.Serializer.NewtonsoftJson/KafkaFlow.Serializer.NewtonsoftJson.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | KafkaFlow.Serializer.NewtonsoftJson 6 | JSON implementation for KafkaFlow serializer middleware using Newtonsoft Json 7 | KafkaFlow.Serializer 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.PauseConsumerOnError/MessageHandler.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Sample.PauseConsumerOnError; 2 | 3 | public class MessageHandler : IMessageHandler 4 | { 5 | public Task Handle(IMessageContext context, WelcomeMessage message) 6 | { 7 | if (string.IsNullOrEmpty(message.Text)) 8 | throw new InvalidDataException("Missing Message Text"); 9 | 10 | // Handle message 11 | // ... 12 | 13 | Console.WriteLine( 14 | "Partition: {0} / Offset: {1} / Message: {2}", 15 | context.ConsumerContext.Partition, 16 | context.ConsumerContext.Offset, 17 | message.Text); 18 | 19 | return Task.CompletedTask; 20 | } 21 | } -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/consumer/shared/rewind-modal/rewind-modal.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, Input, OnInit } from '@angular/core'; 2 | import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'; 3 | 4 | @Component({ 5 | selector: 'app-rewind-modal', 6 | templateUrl: './rewind-modal.component.html' 7 | }) 8 | export class RewindModalComponent { 9 | public rewindDate: Date | undefined; 10 | @Input() public groupId: string | undefined; 11 | @Input() public consumerName: string | undefined; 12 | @Input() public topic: string | undefined; 13 | 14 | constructor(public activeModal: NgbActiveModal) { } 15 | 16 | save = () => { 17 | this.activeModal.close(this.rewindDate); 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/ITelemetryStorage.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using KafkaFlow.Admin.Messages; 3 | 4 | namespace KafkaFlow.Admin; 5 | 6 | /// 7 | /// Used to implement a telemetry data storage provider 8 | /// 9 | public interface ITelemetryStorage 10 | { 11 | /// 12 | /// Gets all the consumer telemetry metrics 13 | /// 14 | /// The list of consumer metrics 15 | IEnumerable Get(); 16 | 17 | /// 18 | /// Store the metric provided 19 | /// 20 | /// The consumer telemetry metric 21 | void Put(ConsumerTelemetryMetric telemetryMetric); 22 | } 23 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/Messages/PauseConsumersByGroup.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Runtime.Serialization; 3 | 4 | namespace KafkaFlow.Admin.Messages; 5 | 6 | /// 7 | /// The message that pauses an entire consumer group 8 | /// 9 | [DataContract] 10 | public class PauseConsumersByGroup : IAdminMessage 11 | { 12 | /// 13 | /// Gets or sets the consumer group id 14 | /// 15 | [DataMember(Order = 1)] 16 | public string GroupId { get; set; } 17 | 18 | /// 19 | /// Gets or sets the topics that will be paused 20 | /// 21 | [DataMember(Order = 2)] 22 | public IList Topics { get; set; } = new List(); 23 | } 24 | -------------------------------------------------------------------------------- /tests/KafkaFlow.IntegrationTests/Core/Middlewares/GzipMiddleware.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics; 2 | using System.Threading.Tasks; 3 | using KafkaFlow.IntegrationTests.Core.Handlers; 4 | using KafkaFlow.OpenTelemetry; 5 | 6 | namespace KafkaFlow.IntegrationTests.Core.Middlewares; 7 | 8 | internal class GzipMiddleware : IMessageMiddleware 9 | { 10 | public async Task Invoke(IMessageContext context, MiddlewareDelegate next) 11 | { 12 | var source = new ActivitySource(KafkaFlowInstrumentation.ActivitySourceName); 13 | using var activity = source.StartActivity("integration-test", ActivityKind.Internal); 14 | 15 | MessageStorage.Add((byte[])context.Message.Value); 16 | await next(context); 17 | } 18 | } -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/Handlers/RestartConsumerByNameHandler.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using KafkaFlow.Admin.Messages; 3 | using KafkaFlow.Consumers; 4 | 5 | namespace KafkaFlow.Admin.Handlers; 6 | 7 | internal class RestartConsumerByNameHandler : IMessageHandler 8 | { 9 | private readonly IConsumerAccessor _consumerAccessor; 10 | 11 | public RestartConsumerByNameHandler(IConsumerAccessor consumerAccessor) => _consumerAccessor = consumerAccessor; 12 | 13 | public Task Handle(IMessageContext context, RestartConsumerByName message) 14 | { 15 | var consumer = _consumerAccessor[message.ConsumerName]; 16 | 17 | return consumer?.RestartAsync() ?? Task.CompletedTask; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/Messages/PauseConsumerByName.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Runtime.Serialization; 3 | 4 | namespace KafkaFlow.Admin.Messages; 5 | 6 | /// 7 | /// A message that pauses a consumer 8 | /// 9 | [DataContract] 10 | public class PauseConsumerByName : IAdminMessage 11 | { 12 | /// 13 | /// Gets or sets the consumer name that will be paused 14 | /// 15 | [DataMember(Order = 1)] 16 | public string ConsumerName { get; set; } 17 | 18 | /// 19 | /// Gets or sets the topics that will be paused 20 | /// 21 | [DataMember(Order = 2)] 22 | public IList Topics { get; set; } = new List(); 23 | } 24 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/Messages/ResumeConsumersByGroup.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Runtime.Serialization; 3 | 4 | namespace KafkaFlow.Admin.Messages; 5 | 6 | /// 7 | /// The message that resume a paused consumer group 8 | /// 9 | [DataContract] 10 | public class ResumeConsumersByGroup : IAdminMessage 11 | { 12 | /// 13 | /// Gets or sets the consumer group id 14 | /// 15 | [DataMember(Order = 1)] 16 | public string GroupId { get; set; } 17 | 18 | /// 19 | /// Gets or sets the topics that will be resumed 20 | /// 21 | [DataMember(Order = 2)] 22 | public IList Topics { get; set; } = new List(); 23 | } 24 | -------------------------------------------------------------------------------- /src/KafkaFlow.Compressor.Gzip/GzipMessageDecompressor.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | using System.IO.Compression; 3 | 4 | namespace KafkaFlow.Compressor.Gzip; 5 | 6 | /// 7 | /// A GZIP message decompressor 8 | /// 9 | public class GzipMessageDecompressor : IDecompressor 10 | { 11 | /// 12 | public byte[] Decompress(byte[] message) 13 | { 14 | using var outputStream = new MemoryStream(); 15 | using var inputStream = new MemoryStream(message); 16 | 17 | using (var gzipStream = new GZipStream(inputStream, CompressionMode.Decompress)) 18 | { 19 | gzipStream.CopyTo(outputStream); 20 | } 21 | 22 | return outputStream.ToArray(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/KafkaFlow.SchemaRegistry/KafkaFlow.SchemaRegistry.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | KafkaFlow.SchemaRegistry 6 | Provides extensions methods to use Schema Registry 7 | KafkaFlow 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /tests/KafkaFlow.UnitTests/ConfigurationBuilders/KafkaConfigurationBuilderTests.cs: -------------------------------------------------------------------------------- 1 | using KafkaFlow.Configuration; 2 | using Microsoft.VisualStudio.TestTools.UnitTesting; 3 | using Moq; 4 | 5 | namespace KafkaFlow.UnitTests.ConfigurationBuilders; 6 | 7 | [TestClass] 8 | public class KafkaConfigurationBuilderTests 9 | { 10 | [TestMethod] 11 | public void ExtensionMethod_UseMicrosoftLog_ConfigureMicrosoftLogHandler() 12 | { 13 | // Arrange 14 | var builder = new Mock(); 15 | 16 | // Act 17 | ExtensionMethods.UseMicrosoftLog(builder.Object); 18 | 19 | // Assert 20 | builder.Verify(x => x.UseLogHandler(), Times.Once); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/Messages/ResumeConsumerByName.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Runtime.Serialization; 3 | 4 | namespace KafkaFlow.Admin.Messages; 5 | 6 | /// 7 | /// The message that resume a paused consumer 8 | /// 9 | [DataContract] 10 | public class ResumeConsumerByName : IAdminMessage 11 | { 12 | /// 13 | /// Gets or sets the consumer name that will be resumed 14 | /// 15 | [DataMember(Order = 1)] 16 | public string ConsumerName { get; set; } 17 | 18 | /// 19 | /// Gets or sets the topics that will be resumed 20 | /// 21 | [DataMember(Order = 2)] 22 | public IList Topics { get; set; } = new List(); 23 | } 24 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/consumer/shared/workers-count-modal/workers-count-modal.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, Input } from '@angular/core'; 2 | import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'; 3 | 4 | @Component({ 5 | selector: 'app-workers-count-modal', 6 | templateUrl: './workers-count-modal.component.html' 7 | }) 8 | export class WorkersCountModalComponent { 9 | @Input() public workersCount: number | undefined; 10 | @Input() public groupId: string | undefined; 11 | @Input() public consumerName: string | undefined; 12 | public oldWorkersCount: number | undefined; 13 | 14 | constructor(public activeModal: NgbActiveModal) { } 15 | 16 | save = () => { 17 | this.activeModal.close(this.workersCount); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/KafkaFlow/Clusters/ClusterManagerAccessor.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | 4 | namespace KafkaFlow.Clusters; 5 | 6 | internal class ClusterManagerAccessor : IClusterManagerAccessor 7 | { 8 | private readonly Dictionary _managers; 9 | 10 | public ClusterManagerAccessor(IEnumerable managers) 11 | { 12 | _managers = managers.ToDictionary(manager => manager.ClusterName); 13 | } 14 | 15 | public IEnumerable All => _managers.Values; 16 | 17 | public IClusterManager this[string name] => this.GetManager(name); 18 | 19 | public IClusterManager GetManager(string name) => 20 | _managers.TryGetValue(name, out var manager) ? manager : null; 21 | } 22 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample/README.md: -------------------------------------------------------------------------------- 1 | # KafkaFlow.Sample 2 | 3 | This is a simple sample that shows how to produce and consume messages. 4 | 5 | ## How to run 6 | 7 | ### Requirements 8 | 9 | - [.NET 6.0 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/6.0) 10 | - [Docker Desktop](https://www.docker.com/products/docker-desktop/) 11 | 12 | ### Start the cluster 13 | 14 | Using your terminal of choice, start the cluster. 15 | You can find a docker-compose file at the root of this repository. 16 | Position the terminal in that folder and run the following command. 17 | 18 | ```bash 19 | docker-compose up -d 20 | ``` 21 | 22 | ### Run the Sample 23 | 24 | Using your terminal of choice, start the sample for the sample folder. 25 | 26 | ```bash 27 | dotnet run 28 | ``` 29 | -------------------------------------------------------------------------------- /src/KafkaFlow.LogHandler.Microsoft/KafkaFlow.LogHandler.Microsoft.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | KafkaFlow 6 | KafkaFlow.LogHandler.Microsoft 7 | LogHandler implementation leveraging Microsoft Logging framework. 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/IMessageHeaders.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace KafkaFlow; 4 | 5 | /// 6 | /// Represents a collection of message headers 7 | /// 8 | public interface IMessageHeaders : IEnumerable> 9 | { 10 | /// 11 | /// Gets or sets the header with a specified key 12 | /// 13 | /// The zero-based index of the element to get 14 | byte[] this[string key] { get; set; } 15 | 16 | /// 17 | /// Adds a new header to the enumeration 18 | /// 19 | /// The header key. 20 | /// The header value (possibly null) 21 | void Add(string key, byte[] value); 22 | } 23 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/Messages/ResetConsumerOffset.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Runtime.Serialization; 3 | 4 | namespace KafkaFlow.Admin.Messages; 5 | 6 | /// 7 | /// The message that rewind the offset of all partitions/topics of a consumer to the beginning 8 | /// 9 | [DataContract] 10 | public class ResetConsumerOffset : IAdminMessage 11 | { 12 | /// 13 | /// Gets or sets the consumer name that will be reset 14 | /// 15 | [DataMember(Order = 1)] 16 | public string ConsumerName { get; set; } 17 | 18 | /// 19 | /// Gets or sets the topics that will be reset 20 | /// 21 | [DataMember(Order = 2)] 22 | public IList Topics { get; set; } = new List(); 23 | } 24 | -------------------------------------------------------------------------------- /src/KafkaFlow.Extensions.Hosting/KafkaFlow.Extensions.Hosting.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | KafkaFlow 6 | KafkaFlow.Extensions.Hosting 7 | Helper to run KafkaFlow as a Hosted Service 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.BatchOperations/README.md: -------------------------------------------------------------------------------- 1 | # KafkaFlow.Sample.BatchOperations 2 | 3 | This is a sample that shows batch processing using KafkaFlow. 4 | 5 | ## How to run 6 | 7 | ### Requirements 8 | 9 | - [.NET 6.0 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/6.0) 10 | - [Docker Desktop](https://www.docker.com/products/docker-desktop/) 11 | 12 | ### Start the cluster 13 | 14 | Using your terminal of choice, start the cluster. 15 | You can find a docker-compose file at the root of this repository. 16 | Position the terminal in that folder and run the following command. 17 | 18 | ```bash 19 | docker-compose up -d 20 | ``` 21 | 22 | ### Run the Sample 23 | 24 | Using your terminal of choice, start the sample for the sample folder. 25 | 26 | ```bash 27 | dotnet run 28 | ``` 29 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.CooperativeSticky/README.md: -------------------------------------------------------------------------------- 1 | # KafkaFlow.Sample 2 | 3 | This is a simple sample that shows how to produce and consume messages. 4 | 5 | ## How to run 6 | 7 | ### Requirements 8 | 9 | - [.NET 6.0 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/6.0) 10 | - [Docker Desktop](https://www.docker.com/products/docker-desktop/) 11 | 12 | ### Start the cluster 13 | 14 | Using your terminal of choice, start the cluster. 15 | You can find a docker-compose file at the root of this repository. 16 | Position the terminal in that folder and run the following command. 17 | 18 | ```bash 19 | docker-compose up -d 20 | ``` 21 | 22 | ### Run the Sample 23 | 24 | Using your terminal of choice, start the sample for the sample folder. 25 | 26 | ```bash 27 | dotnet run 28 | ``` 29 | -------------------------------------------------------------------------------- /src/KafkaFlow/Authentication/OAuthBearerAuthenticator.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using Confluent.Kafka; 3 | 4 | namespace KafkaFlow.Authentication; 5 | 6 | internal readonly struct OAuthBearerAuthenticator : IOAuthBearerAuthenticator 7 | { 8 | private readonly IClient _client; 9 | 10 | public OAuthBearerAuthenticator(IClient client) 11 | { 12 | _client = client; 13 | } 14 | 15 | public void SetToken(string tokenValue, long lifetimeMs, string principalName, IDictionary extensions = null) 16 | { 17 | _client.OAuthBearerSetToken(tokenValue, lifetimeMs, principalName, extensions); 18 | } 19 | 20 | public void SetTokenFailure(string error) 21 | { 22 | _client.OAuthBearerSetTokenFailure(error); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/sort.pipe.ts: -------------------------------------------------------------------------------- 1 | import { Pipe, PipeTransform } from '@angular/core'; 2 | 3 | @Pipe({ 4 | name: 'sort' 5 | }) 6 | export class SortPipe implements PipeTransform { 7 | transform(array: any, field: string, order: string = 'asc'): any[] { 8 | order = order.toLowerCase(); 9 | if (order !== 'asc' && order !== 'desc') { 10 | return array; 11 | } 12 | if (!Array.isArray(array)) { 13 | return null as any; 14 | } 15 | array.sort((a: any, b: any) => { 16 | if (a[field] < b[field]) { 17 | return (order === 'asc') ? -1 : 1; 18 | } else if (a[field] > b[field]) { 19 | return (order === 'asc') ? 1 : -1; 20 | } else { 21 | return 0; 22 | } 23 | }); 24 | return array; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/DashboardConfiguration.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.AspNetCore.Builder; 3 | using Microsoft.AspNetCore.Http; 4 | 5 | namespace KafkaFlow.Admin.Dashboard; 6 | 7 | internal class DashboardConfiguration 8 | { 9 | public DashboardConfiguration( 10 | PathString basePath, 11 | Action requestHandler, 12 | Action endpointHandler) 13 | { 14 | this.BasePath = basePath; 15 | this.RequestHandler = requestHandler; 16 | this.EndpointHandler = endpointHandler; 17 | } 18 | 19 | public PathString BasePath { get; } 20 | 21 | public Action RequestHandler { get; } 22 | 23 | public Action EndpointHandler { get; } 24 | } 25 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.SchemaRegistry/README.md: -------------------------------------------------------------------------------- 1 | # KafkaFlow.Sample.SchemaRegistry 2 | 3 | This is a sample that shows how to use Schema Registry with KafkaFlow. 4 | 5 | ## How to run 6 | 7 | ### Requirements 8 | 9 | - [.NET 6.0 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/6.0) 10 | - [Docker Desktop](https://www.docker.com/products/docker-desktop/) 11 | 12 | ### Start the cluster 13 | 14 | Using your terminal of choice, start the cluster. 15 | You can find a docker-compose file at the root of this repository. 16 | Position the terminal in that folder and run the following command. 17 | 18 | ```bash 19 | docker-compose up -d 20 | ``` 21 | 22 | ### Run the Sample 23 | 24 | Using your terminal of choice, start the sample for the sample folder. 25 | 26 | ```bash 27 | dotnet run 28 | ``` 29 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/Message.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow; 2 | 3 | /// 4 | /// Represents a Kafka message 5 | /// 6 | public readonly struct Message 7 | { 8 | /// 9 | /// Initializes a new instance of the struct. 10 | /// 11 | /// 12 | /// 13 | public Message(object key, object value) 14 | { 15 | this.Key = key; 16 | this.Value = value; 17 | } 18 | 19 | /// 20 | /// Gets the message key 21 | /// 22 | public object Key { get; } 23 | 24 | /// 25 | /// Gets the message value 26 | /// 27 | public object Value { get; } 28 | } 29 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.Dashboard/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "iisSettings": { 3 | "windowsAuthentication": false, 4 | "anonymousAuthentication": true, 5 | "iisExpress": { 6 | "applicationUrl": "http://localhost:63128", 7 | "sslPort": 0 8 | } 9 | }, 10 | "profiles": { 11 | "IIS Express": { 12 | "commandName": "IISExpress", 13 | "launchBrowser": true, 14 | "environmentVariables": { 15 | "ASPNETCORE_ENVIRONMENT": "Development" 16 | } 17 | }, 18 | "KafkaFlow.Sample.Dashboard": { 19 | "commandName": "Project", 20 | "launchBrowser": true, 21 | "applicationUrl": "http://localhost:5002", 22 | "environmentVariables": { 23 | "ASPNETCORE_ENVIRONMENT": "Development" 24 | } 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/environments/environment.ts: -------------------------------------------------------------------------------- 1 | // This file can be replaced during build by using the `fileReplacements` array. 2 | // `ng build --prod` replaces `environment.ts` with `environment.prod.ts`. 3 | // The list of file replacements can be found in `angular.json`. 4 | 5 | export const environment = { 6 | production: false, 7 | apiUrl: '/kafkaflow' 8 | }; 9 | 10 | /* 11 | * For easier debugging in development mode, you can import the following file 12 | * to ignore zone related error stack frames such as `zone.run`, `zoneDelegate.invokeTask`. 13 | * 14 | * This import should be commented out in production mode because it will have a negative impact 15 | * on performance if an error is thrown. 16 | */ 17 | // import 'zone.js/dist/zone-error'; // Included with Angular CLI. 18 | -------------------------------------------------------------------------------- /src/KafkaFlow.Microsoft.DependencyInjection/KafkaFlow.Microsoft.DependencyInjection.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | KafkaFlow.Microsoft.DependencyInjection 6 | Adapts KafkaFlow to use Microsoft Dependency Injection 7 | KafkaFlow 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/.browserslistrc: -------------------------------------------------------------------------------- 1 | # This file is used by the build system to adjust CSS and JS output to support the specified browsers below. 2 | # For additional information regarding the format and rule options, please see: 3 | # https://github.com/browserslist/browserslist#queries 4 | 5 | # For the full list of supported browsers by the Angular framework, please see: 6 | # https://angular.io/guide/browser-support 7 | 8 | # You can see what browsers were selected by your queries by running: 9 | # npx browserslist 10 | 11 | last 1 Chrome version 12 | last 1 Firefox version 13 | last 2 Edge major versions 14 | last 2 Safari major versions 15 | last 2 iOS major versions 16 | Firefox ESR 17 | not IE 11 # Angular supports IE 11 only as an opt-in. To opt-in, remove the 'not' prefix on this line. 18 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.WebApi/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "iisSettings": { 3 | "windowsAuthentication": false, 4 | "anonymousAuthentication": true, 5 | "iisExpress": { 6 | "applicationUrl": "http://localhost:58660/", 7 | "sslPort": 44317 8 | } 9 | }, 10 | "profiles": { 11 | "IIS Express": { 12 | "commandName": "IISExpress", 13 | "launchBrowser": true, 14 | "environmentVariables": { 15 | "ASPNETCORE_ENVIRONMENT": "Development" 16 | } 17 | }, 18 | "KafkaFlow.Admin.WebApi": { 19 | "commandName": "Project", 20 | "launchBrowser": true, 21 | "environmentVariables": { 22 | "ASPNETCORE_ENVIRONMENT": "Development" 23 | }, 24 | "applicationUrl": "https://localhost:5001;http://localhost:5000" 25 | } 26 | } 27 | } -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.SchemaRegistry/MessageTypes/Avro/SchemaRegistry/LogLevel.cs: -------------------------------------------------------------------------------- 1 | // ------------------------------------------------------------------------------ 2 | // 3 | // Generated by avrogen, version 1.11.0.0 4 | // Changes to this file may cause incorrect behavior and will be lost if code 5 | // is regenerated 6 | // 7 | // ------------------------------------------------------------------------------ 8 | namespace SchemaRegistry 9 | { 10 | using System; 11 | using System.Collections.Generic; 12 | using System.Text; 13 | using Avro; 14 | using Avro.Specific; 15 | 16 | /// 17 | /// Enumerates the set of allowable log levels. 18 | /// 19 | public enum LogLevel 20 | { 21 | None, 22 | Verbose, 23 | Info, 24 | Warning, 25 | Error, 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/KafkaFlow/Configuration/KafkaConfiguration.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace KafkaFlow.Configuration; 4 | 5 | /// 6 | /// Represents the kafka configuration values 7 | /// 8 | public class KafkaConfiguration 9 | { 10 | private readonly List _clusters = new(); 11 | 12 | /// 13 | /// Gets the cluster configuration list 14 | /// 15 | public IReadOnlyCollection Clusters => _clusters; 16 | 17 | /// 18 | /// Adds a list of cluster configurations 19 | /// 20 | /// A list of cluster configurations 21 | public void AddClusters(IEnumerable configurations) => 22 | _clusters.AddRange(configurations); 23 | } 24 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/Handlers/ChangeConsumerWorkersCountHandler.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using KafkaFlow.Admin.Messages; 3 | using KafkaFlow.Consumers; 4 | 5 | namespace KafkaFlow.Admin.Handlers; 6 | 7 | internal class ChangeConsumerWorkersCountHandler : IMessageHandler 8 | { 9 | private readonly IConsumerAccessor _consumerAccessor; 10 | 11 | public ChangeConsumerWorkersCountHandler(IConsumerAccessor consumerAccessor) => _consumerAccessor = consumerAccessor; 12 | 13 | public Task Handle(IMessageContext context, ChangeConsumerWorkersCount message) 14 | { 15 | var consumer = _consumerAccessor[message.ConsumerName]; 16 | 17 | return 18 | consumer?.ChangeWorkersCountAndRestartAsync(message.WorkersCount) ?? 19 | Task.CompletedTask; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.WebApi/Adapters/ConsumerResponseAdapter.cs: -------------------------------------------------------------------------------- 1 | using KafkaFlow.Admin.WebApi.Contracts; 2 | using KafkaFlow.Consumers; 3 | 4 | namespace KafkaFlow.Admin.WebApi.Adapters; 5 | 6 | internal static class ConsumerResponseAdapter 7 | { 8 | internal static ConsumerResponse Adapt(this IMessageConsumer consumer) 9 | { 10 | return new() 11 | { 12 | Subscription = consumer.Subscription, 13 | ConsumerName = consumer.ConsumerName, 14 | GroupId = consumer.GroupId, 15 | Status = consumer.Status.ToString(), 16 | MemberId = consumer.MemberId, 17 | WorkersCount = consumer.WorkersCount, 18 | ClientInstanceName = consumer.ClientInstanceName, 19 | ManagementDisabled = consumer.ManagementDisabled, 20 | }; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/KafkaFlow/Extensions/ChannelExtensions.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Runtime.CompilerServices; 3 | using System.Threading; 4 | using System.Threading.Channels; 5 | 6 | namespace KafkaFlow.Extensions; 7 | 8 | internal static class ChannelExtensions 9 | { 10 | public static async IAsyncEnumerable ReadAllItemsAsync( 11 | this ChannelReader reader, 12 | [EnumeratorCancellation] CancellationToken cancellationToken = default) 13 | { 14 | while (await reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false)) 15 | { 16 | while (reader.TryRead(out var item)) 17 | { 18 | cancellationToken.ThrowIfCancellationRequested(); 19 | yield return item; 20 | } 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/KafkaFlow/Middlewares/ConsumerThrottling/Configuration/IConsumerThrottlingActionConfigurationBuilder.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Middlewares.ConsumerThrottling.Configuration; 2 | 3 | /// 4 | /// Provides a builder interface for setting up actions in the KafkaFlow consumer throttling feature. 5 | /// 6 | public interface IConsumerThrottlingActionConfigurationBuilder 7 | { 8 | /// 9 | /// Defines the threshold at which the throttling action should be triggered. 10 | /// 11 | /// The threshold value for the action. 12 | /// An instance of to further configure the action. 13 | IConsumerThrottlingThresholdActionConfigurationBuilder AboveThreshold(long threshold); 14 | } 15 | -------------------------------------------------------------------------------- /tests/KafkaFlow.UnitTests/Factories/WorkerFactory.cs: -------------------------------------------------------------------------------- 1 | using Moq; 2 | 3 | namespace KafkaFlow.UnitTests.Factories; 4 | 5 | internal static class WorkerFactory 6 | { 7 | /// 8 | /// Generate workers for testing. The Worker ID's are sequencial and zero-based. 9 | /// 10 | /// Number of workers to generate 11 | /// A list of workers 12 | public static IWorker[] CreateWorkers(int workerCount) 13 | { 14 | var workers = new IWorker[workerCount]; 15 | 16 | for (var i = 0; i < workerCount; i++) 17 | { 18 | var workerMock = new Mock(); 19 | workerMock.Setup(x => x.Id).Returns(i); 20 | 21 | workers[i] = workerMock.Object; 22 | } 23 | 24 | return workers; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.FlowControl/README.md: -------------------------------------------------------------------------------- 1 | # KafkaFlow.Sample.FlowControl 2 | 3 | This is a sample that shows how to control the state of a consumer (Starting, Pausing, Stopping, etc.) programmatically. 4 | 5 | ## How to run 6 | 7 | ### Requirements 8 | 9 | - [.NET 6.0 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/6.0) 10 | - [Docker Desktop](https://www.docker.com/products/docker-desktop/) 11 | 12 | ### Start the cluster 13 | 14 | Using your terminal of choice, start the cluster. 15 | You can find a docker-compose file at the root of this repository. 16 | Position the terminal in that folder and run the following command. 17 | 18 | ```bash 19 | docker-compose up -d 20 | ``` 21 | 22 | ### Run the Sample 23 | 24 | Using your terminal of choice, start the sample for the sample folder. 25 | 26 | ```bash 27 | dotnet run 28 | ``` 29 | -------------------------------------------------------------------------------- /src/KafkaFlow/Middlewares/ConsumerThrottling/Configuration/IConsumerThrottlingActionsConfigurationBuilder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace KafkaFlow.Middlewares.ConsumerThrottling.Configuration; 4 | 5 | /// 6 | /// Provides methods to configure throttling actions for KafkaFlow consumers. 7 | /// 8 | public interface IConsumerThrottlingActionsConfigurationBuilder 9 | { 10 | /// 11 | /// Adds a throttling action to the configuration. 12 | /// 13 | /// An action that configures the throttling action. 14 | /// The same instance of the for method chaining. 15 | IConsumerThrottlingActionsConfigurationBuilder AddAction(Action action); 16 | } 17 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # Description 2 | 3 | Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change. 4 | 5 | Fixes # (issue) 6 | 7 | ## How Has This Been Tested? 8 | 9 | Please describe the tests that you ran to verify your changes. 10 | 11 | ## Checklist 12 | 13 | - [ ] My code follows the style guidelines of this project 14 | - [ ] I have performed a self-review of my own code 15 | - [ ] I have added tests to cover my changes 16 | - [ ] I have made corresponding changes to the documentation 17 | 18 | ### Disclaimer 19 | 20 | By sending us your contributions, you are agreeing that your contribution is made subject to the terms of our [Contributor Ownership Statement](https://github.com/Farfetch/.github/blob/master/COS.md) 21 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.WebApi/README.md: -------------------------------------------------------------------------------- 1 | # KafkaFlow.Sample.WebApi 2 | 3 | This sample shows how to host an administration Web API for administrative operations. 4 | 5 | ## How to run 6 | 7 | ### Requirements 8 | 9 | - [.NET 6.0 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/6.0) 10 | - [Docker Desktop](https://www.docker.com/products/docker-desktop/) 11 | 12 | ### Start the cluster 13 | 14 | Using your terminal of choice, start the cluster. 15 | You can find a docker-compose file at the root of this repository. 16 | Position the terminal in that folder and run the following command. 17 | 18 | ```bash 19 | docker-compose up -d 20 | ``` 21 | 22 | ### Run the Sample 23 | 24 | Using your terminal of choice, start the sample for the sample folder. 25 | 26 | ```bash 27 | dotnet run 28 | ``` 29 | 30 | Swagger UI will be accessed on `/swagger/`. 31 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/MessageErrorEventContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace KafkaFlow; 4 | 5 | /// 6 | /// Represents the errors in message context used in the events 7 | /// 8 | public class MessageErrorEventContext : MessageEventContext 9 | { 10 | /// 11 | /// Initializes a new instance of the class. 12 | /// 13 | /// The message context 14 | /// The event exception 15 | public MessageErrorEventContext(IMessageContext messageContext, Exception exception) 16 | : base(messageContext) 17 | { 18 | this.Exception = exception; 19 | } 20 | 21 | /// 22 | /// Gets the exception 23 | /// 24 | public Exception Exception { get; } 25 | } 26 | -------------------------------------------------------------------------------- /src/KafkaFlow.Microsoft.DependencyInjection/ServiceProviderExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using global::Microsoft.Extensions.DependencyInjection; 3 | using KafkaFlow.Configuration; 4 | 5 | namespace KafkaFlow; 6 | 7 | /// 8 | /// Extension methods over IServiceProvider 9 | /// 10 | public static class ServiceProviderExtensions 11 | { 12 | /// 13 | /// Creates a KafkaFlow bus 14 | /// 15 | /// Instance of 16 | /// A KafkaFlow bus 17 | public static IKafkaBus CreateKafkaBus(this IServiceProvider provider) 18 | { 19 | var resolver = provider.GetRequiredService(); 20 | return provider.GetRequiredService().CreateBus(resolver); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/SerializerContext.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow; 2 | 3 | /// 4 | public class SerializerContext : ISerializerContext 5 | { 6 | /// 7 | /// The static instance of the 8 | /// 9 | public static readonly ISerializerContext Empty = new SerializerContext(); 10 | 11 | /// 12 | /// Initializes a new instance of the class. 13 | /// 14 | /// 15 | public SerializerContext(string topic) 16 | { 17 | this.Topic = topic; 18 | } 19 | 20 | private SerializerContext() 21 | { 22 | } 23 | 24 | /// 25 | /// Gets the topic associated with the message 26 | /// 27 | public string Topic { get; } 28 | } 29 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/consumer/shared/stop-modal/stop-modal.component.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | 6 |
7 |
8 |
9 | Stop the consumer '{{consumerName}}' from group id '{{groupId}}' 10 |
Are you really sure about it?
11 |
12 |
13 |
14 | 15 | 16 |
17 | -------------------------------------------------------------------------------- /src/KafkaFlow/Middlewares/ConsumerThrottling/Configuration/ConsumerThrottlingConfiguration.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace KafkaFlow.Middlewares.ConsumerThrottling.Configuration; 5 | 6 | internal class ConsumerThrottlingConfiguration 7 | { 8 | public ConsumerThrottlingConfiguration( 9 | TimeSpan evaluationInterval, 10 | IReadOnlyList metrics, 11 | IReadOnlyList thresholds) 12 | { 13 | this.EvaluationInterval = evaluationInterval; 14 | this.Metrics = metrics; 15 | this.Thresholds = thresholds; 16 | } 17 | 18 | public TimeSpan EvaluationInterval { get; } 19 | 20 | public IReadOnlyList Metrics { get; } 21 | 22 | public IReadOnlyList Thresholds { get; } 23 | } 24 | -------------------------------------------------------------------------------- /src/KafkaFlow/Middlewares/ConsumerThrottling/ConsumerThrottlingThreshold.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace KafkaFlow.Middlewares.ConsumerThrottling; 4 | 5 | internal class ConsumerThrottlingThreshold : IConsumerThrottlingThreshold 6 | { 7 | private readonly IConsumerThrottlingAction _action; 8 | 9 | public ConsumerThrottlingThreshold(long thresholdValue, IConsumerThrottlingAction action) 10 | { 11 | _action = action; 12 | this.ThresholdValue = thresholdValue; 13 | } 14 | 15 | public long ThresholdValue { get; } 16 | 17 | public async Task TryExecuteActionAsync(long metricValue) 18 | { 19 | if (this.ThresholdValue > metricValue) 20 | { 21 | return false; 22 | } 23 | 24 | await _action.ExecuteAsync().ConfigureAwait(false); 25 | 26 | return true; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/IMessageHandler.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace KafkaFlow; 4 | 5 | /// 6 | /// Used to create a message handler 7 | /// 8 | /// The message type 9 | public interface IMessageHandler : IMessageHandler 10 | { 11 | /// 12 | /// the method that will be called when a arrives 13 | /// 14 | /// Instance of 15 | /// The message type to be processed 16 | /// 17 | Task Handle(IMessageContext context, TMessage message); 18 | } 19 | 20 | /// 21 | /// Used to create a message handler 22 | /// 23 | public interface IMessageHandler 24 | { 25 | } 26 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/consumer/shared/start-modal/start-modal.component.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | 6 |
7 |
8 |
9 | Start the consumer '{{consumerName}}' from group id '{{groupId}}' 10 |
Are you really sure about it?
11 |
12 |
13 |
14 | 15 | 16 |
17 | -------------------------------------------------------------------------------- /src/KafkaFlow/Clusters/IClusterManagerAccessor.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace KafkaFlow.Clusters; 4 | 5 | /// 6 | /// Provides access to the configured cluster manager 7 | /// 8 | internal interface IClusterManagerAccessor 9 | { 10 | /// 11 | /// Gets all configured cluster managers 12 | /// 13 | IEnumerable All { get; } 14 | 15 | /// 16 | /// Gets a cluster manager by its name 17 | /// 18 | /// cluster name 19 | IClusterManager this[string name] { get; } 20 | 21 | /// 22 | /// Gets a cluster manager by its name 23 | /// 24 | /// The name defined in the cluster configuration 25 | /// 26 | IClusterManager GetManager(string name); 27 | } 28 | -------------------------------------------------------------------------------- /src/KafkaFlow/Middlewares/ConsumerThrottling/Configuration/ConsumerThrottlingActionConfigurationBuilder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace KafkaFlow.Middlewares.ConsumerThrottling.Configuration; 4 | 5 | internal class ConsumerThrottlingActionConfigurationBuilder 6 | : IConsumerThrottlingActionConfigurationBuilder, 7 | IConsumerThrottlingThresholdActionConfigurationBuilder 8 | { 9 | public long Threshold { get; private set; } 10 | 11 | public Func Factory { get; private set; } 12 | 13 | public IConsumerThrottlingThresholdActionConfigurationBuilder AboveThreshold(long threshold) 14 | { 15 | this.Threshold = threshold; 16 | return this; 17 | } 18 | 19 | public void Apply(Func factory) 20 | { 21 | this.Factory = factory; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/.gitignore: -------------------------------------------------------------------------------- 1 | # See http://help.github.com/ignore-files/ for more about ignoring files. 2 | 3 | # compiled output 4 | /dist 5 | /tmp 6 | /out-tsc 7 | # Only exists if Bazel was run 8 | /bazel-out 9 | 10 | # dependencies 11 | /node_modules 12 | /.angular 13 | 14 | # profiling files 15 | chrome-profiler-events*.json 16 | speed-measure-plugin*.json 17 | 18 | # IDEs and editors 19 | /.idea 20 | .project 21 | .classpath 22 | .c9/ 23 | *.launch 24 | .settings/ 25 | *.sublime-workspace 26 | 27 | # IDE - VSCode 28 | .vscode/* 29 | !.vscode/settings.json 30 | !.vscode/tasks.json 31 | !.vscode/launch.json 32 | !.vscode/extensions.json 33 | .history/* 34 | 35 | # misc 36 | /.sass-cache 37 | /connect.lock 38 | /coverage 39 | /libpeerconnection.log 40 | npm-debug.log 41 | yarn-error.log 42 | testem.log 43 | /typings 44 | 45 | # System Files 46 | .DS_Store 47 | Thumbs.db 48 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/home/home.component.html: -------------------------------------------------------------------------------- 1 | 16 | 17 | -------------------------------------------------------------------------------- /src/KafkaFlow/Producers/BatchProduceException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace KafkaFlow.Producers; 5 | 6 | /// 7 | /// Exception thrown by 8 | /// 9 | public class BatchProduceException : Exception 10 | { 11 | /// 12 | /// Initializes a new instance of the class. 13 | /// 14 | /// The items contained in the batch 15 | public BatchProduceException(IReadOnlyCollection items) 16 | { 17 | this.Items = items; 18 | } 19 | 20 | /// 21 | /// Gets the requested items to produce with filled 22 | /// 23 | public IReadOnlyCollection Items { get; } 24 | } 25 | -------------------------------------------------------------------------------- /website/docs/guides/dependency-injection.md: -------------------------------------------------------------------------------- 1 | --- 2 | sidebar_position: 11 3 | --- 4 | 5 | # Dependency Injection 6 | 7 | KafkaFlow Dependency Injection framework support is extensible. 8 | 9 | [Microsoft .NET DI](https://learn.microsoft.com/en-us/dotnet/core/extensions/dependency-injection/) and [Unity 5](http://unitycontainer.org/articles/quickstart.html) are natively supported. You can see [here](configuration) how to use them. 10 | 11 | ## Add support for a new Dependency Injection container 12 | 13 | Other DI frameworks can be supported by implementing a set of interfaces: 14 | 15 | - `IDependencyConfigurator` 16 | - `IDependencyResolver` 17 | - `IDependencyResolverScope` 18 | 19 | 20 | You can find an example [here](https://github.com/Farfetch/kafkaflow/tree/master/src/KafkaFlow.Unity). 21 | 22 | Once the interfaces are implemented, use them the same way you use Unity ([see here](configuration)). -------------------------------------------------------------------------------- /src/KafkaFlow/Consumers/IConsumerAccessor.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace KafkaFlow.Consumers; 4 | 5 | /// 6 | /// Provides access to the configured consumers 7 | /// 8 | public interface IConsumerAccessor 9 | { 10 | /// 11 | /// Gets all configured consumers 12 | /// 13 | IEnumerable All { get; } 14 | 15 | /// 16 | /// Gets a consumer by its name 17 | /// 18 | /// consumer name 19 | IMessageConsumer this[string name] { get; } 20 | 21 | /// 22 | /// Gets a consumer by its name 23 | /// 24 | /// The name defined in the consumer configuration 25 | /// 26 | IMessageConsumer GetConsumer(string name); 27 | 28 | internal void Add(IMessageConsumer consumer); 29 | } 30 | -------------------------------------------------------------------------------- /src/KafkaFlow/Producers/ProducerAccessor.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | 4 | namespace KafkaFlow.Producers; 5 | 6 | internal class ProducerAccessor : IProducerAccessor 7 | { 8 | private readonly Dictionary _producers; 9 | 10 | public ProducerAccessor(IEnumerable producers) 11 | { 12 | _producers = producers.ToDictionary(x => x.ProducerName); 13 | } 14 | 15 | public IEnumerable All => _producers.Values; 16 | 17 | public IMessageProducer this[string name] => this.GetProducer(name); 18 | 19 | public IMessageProducer GetProducer(string name) => 20 | _producers.TryGetValue(name, out var consumer) ? consumer : null; 21 | 22 | public IMessageProducer GetProducer() => 23 | _producers.TryGetValue(typeof(TProducer).FullName!, out var consumer) ? consumer : null; 24 | } 25 | -------------------------------------------------------------------------------- /website/versioned_docs/version-2.x/guides/dependency-injection.md: -------------------------------------------------------------------------------- 1 | --- 2 | sidebar_position: 20 3 | --- 4 | 5 | # Dependency Injection 6 | 7 | KafkaFlow Dependency Injection framework support is extensible. 8 | 9 | [Microsoft .NET DI](https://learn.microsoft.com/en-us/dotnet/core/extensions/dependency-injection/) and [Unity 5](http://unitycontainer.org/articles/quickstart.html) are natively supported. You can see [here](configuration) how to use them. 10 | 11 | ## Add support for a new Dependency Injection container 12 | 13 | Other DI frameworks can be supported by implementing a set of interfaces: 14 | 15 | - `IDependencyConfigurator` 16 | - `IDependencyResolver` 17 | - `IDependencyResolverScope` 18 | 19 | 20 | You can find an example [here](https://github.com/Farfetch/kafkaflow/tree/master/src/KafkaFlow.Unity). 21 | 22 | Once the interfaces are implemented, use them the same way you use Unity ([see here](configuration)). -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/NullLogHandler.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace KafkaFlow; 4 | 5 | /// 6 | /// A log handler that does nothing 7 | /// 8 | public class NullLogHandler : ILogHandler 9 | { 10 | /// 11 | public void Error(string message, Exception ex, object data) 12 | { 13 | // Do nothing 14 | } 15 | 16 | /// 17 | public void Warning(string message, object data) 18 | { 19 | // Do nothing 20 | } 21 | 22 | /// 23 | public void Warning(string message, Exception ex, object data) 24 | { 25 | // Do nothing 26 | } 27 | 28 | /// 29 | public void Info(string message, object data) 30 | { 31 | // Do nothing 32 | } 33 | 34 | /// 35 | public void Verbose(string message, object data) 36 | { 37 | // Do nothing 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/KafkaFlow.Extensions.Hosting/ServiceCollectionExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using global::Microsoft.Extensions.DependencyInjection; 3 | using KafkaFlow.Configuration; 4 | 5 | namespace KafkaFlow; 6 | 7 | /// 8 | /// No needed 9 | /// 10 | public static class ServiceCollectionExtensions 11 | { 12 | /// 13 | /// Configures KafkaFlow to run as a Hosted Service 14 | /// 15 | /// Instance of 16 | /// A handler to configure KafkaFlow 17 | /// 18 | public static IServiceCollection AddKafkaFlowHostedService( 19 | this IServiceCollection services, 20 | Action kafka) 21 | { 22 | return services 23 | .AddHostedService() 24 | .AddKafka(kafka); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/KafkaFlow.Serializer.SchemaRegistry.ConfluentJson/KafkaFlow.Serializer.SchemaRegistry.ConfluentJson.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | KafkaFlow.Serializer.SchemaRegistry.ConfluentJson 6 | Json implementation for KafkaFlow serializer middleware using Confluent.SchemaRegistry.Serdes.Json package 7 | KafkaFlow.Serializer.SchemaRegistry 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /src/KafkaFlow/Consumers/IConsumerFlowManager.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace KafkaFlow.Consumers; 4 | 5 | /// 6 | /// The consumer flow manager 7 | /// 8 | public interface IConsumerFlowManager 9 | { 10 | /// 11 | /// Gets a list of the consumer paused partitions 12 | /// 13 | IReadOnlyList PausedPartitions { get; } 14 | 15 | /// 16 | /// Pauses a set of partitions 17 | /// 18 | /// A list of partitions 19 | void Pause(IReadOnlyCollection topicPartitions); 20 | 21 | /// 22 | /// Resumes a set of partitions 23 | /// 24 | /// A list of partitions 25 | void Resume(IReadOnlyCollection topicPartitions); 26 | } 27 | -------------------------------------------------------------------------------- /website/sidebars.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creating a sidebar enables you to: 3 | - create an ordered group of docs 4 | - render a sidebar for each doc of that group 5 | - provide next/previous navigation 6 | 7 | The sidebars can be generated from the filesystem, or explicitly defined here. 8 | 9 | Create as many sidebars as you want. 10 | */ 11 | 12 | // @ts-check 13 | 14 | /** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */ 15 | const sidebars = { 16 | // By default, Docusaurus generates a sidebar from the docs folder structure 17 | tutorialSidebar: [{type: 'autogenerated', dirName: '.'}], 18 | 19 | // But you can create a sidebar manually 20 | /* 21 | tutorialSidebar: [ 22 | 'intro', 23 | 'hello', 24 | { 25 | type: 'category', 26 | label: 'Tutorial', 27 | items: ['tutorial-basics/create-a-document'], 28 | }, 29 | ], 30 | */ 31 | }; 32 | 33 | module.exports = sidebars; 34 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/Consumers/IWorker.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow; 2 | 3 | /// 4 | /// Represents the interface of a internal worker 5 | /// 6 | public interface IWorker 7 | { 8 | /// 9 | /// Gets worker's id 10 | /// 11 | int Id { get; } 12 | 13 | /// 14 | /// Gets the subject for worker stopping events where observers can subscribe to receive notifications. 15 | /// 16 | IEvent WorkerStopping { get; } 17 | 18 | /// 19 | /// Gets the subject for worker stopped events where observers can subscribe to receive notifications. 20 | /// 21 | IEvent WorkerStopped { get; } 22 | 23 | /// 24 | /// Gets the subject for worker consumption completed events where observers can subscribe to receive notifications. 25 | /// 26 | IEvent WorkerProcessingEnded { get; } 27 | } 28 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/TopicPartition.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow; 2 | 3 | /// 4 | /// Represents a Kafka topic and partition pair. 5 | /// 6 | public readonly struct TopicPartition 7 | { 8 | /// 9 | /// Initializes a new instance of the struct with the specified topic name, partition number, and offset value. 10 | /// 11 | /// The name of the topic. 12 | /// The id of the partition. 13 | public TopicPartition(string topic, int partition) 14 | { 15 | Topic = topic; 16 | Partition = partition; 17 | } 18 | 19 | /// 20 | /// Gets the name of the topic. 21 | /// 22 | public string Topic { get; } 23 | 24 | /// 25 | /// Gets the id of the partition. 26 | /// 27 | public int Partition { get; } 28 | } 29 | -------------------------------------------------------------------------------- /src/KafkaFlow.Microsoft.DependencyInjection/MicrosoftDependencyResolver.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using global::Microsoft.Extensions.DependencyInjection; 4 | 5 | namespace KafkaFlow; 6 | 7 | internal class MicrosoftDependencyResolver : IDependencyResolver 8 | { 9 | private readonly IServiceProvider _serviceProvider; 10 | 11 | public MicrosoftDependencyResolver(IServiceProvider serviceProvider) 12 | { 13 | _serviceProvider = serviceProvider; 14 | } 15 | 16 | public object Resolve(Type type) 17 | { 18 | return _serviceProvider.GetService(type); 19 | } 20 | 21 | public IEnumerable ResolveAll(Type type) 22 | { 23 | return _serviceProvider.GetServices(type); 24 | } 25 | 26 | public IDependencyResolverScope CreateScope() 27 | { 28 | return new MicrosoftDependencyResolverScope(_serviceProvider.CreateScope()); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/KafkaFlow/Consumers/OffsetsWatermark.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Confluent.Kafka; 3 | 4 | namespace KafkaFlow.Consumers; 5 | 6 | internal readonly struct OffsetsWatermark : IOffsetsWatermark, IEquatable 7 | { 8 | private readonly WatermarkOffsets _watermark; 9 | 10 | public OffsetsWatermark(WatermarkOffsets watermark) 11 | { 12 | _watermark = watermark; 13 | } 14 | 15 | public long High => _watermark.High.Value; 16 | 17 | public long Low => _watermark.Low.Value; 18 | 19 | public bool Equals(OffsetsWatermark other) 20 | { 21 | return Equals(_watermark, other._watermark); 22 | } 23 | 24 | public override bool Equals(object obj) 25 | { 26 | return obj is OffsetsWatermark other && this.Equals(other); 27 | } 28 | 29 | public override int GetHashCode() 30 | { 31 | return _watermark != null ? _watermark.GetHashCode() : 0; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/KafkaFlow/Delegates.cs: -------------------------------------------------------------------------------- 1 | using Confluent.Kafka; 2 | using KafkaFlow.Consumers; 3 | 4 | namespace KafkaFlow; 5 | 6 | /// 7 | /// A factory to decorates the consumer created by KafkaFlow 8 | /// 9 | /// The consumer created by KafkaFlow 10 | /// The to get registered services 11 | public delegate IConsumer ConsumerCustomFactory( 12 | IConsumer consumer, 13 | IDependencyResolver resolver); 14 | 15 | /// 16 | /// A factory to decorates the producer created by KafkaFlow 17 | /// 18 | /// The producer created by KafkaFlow 19 | /// The to get registered services 20 | public delegate IProducer ProducerCustomFactory( 21 | IProducer producer, 22 | IDependencyResolver resolver); 23 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/Handlers/PauseConsumerByNameHandler.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using System.Threading.Tasks; 3 | using KafkaFlow.Admin.Extensions; 4 | using KafkaFlow.Admin.Messages; 5 | using KafkaFlow.Consumers; 6 | 7 | namespace KafkaFlow.Admin.Handlers; 8 | 9 | internal class PauseConsumerByNameHandler : IMessageHandler 10 | { 11 | private readonly IConsumerAccessor _consumerAccessor; 12 | 13 | public PauseConsumerByNameHandler(IConsumerAccessor consumerAccessor) => _consumerAccessor = consumerAccessor; 14 | 15 | public Task Handle(IMessageContext context, PauseConsumerByName message) 16 | { 17 | var consumer = _consumerAccessor[message.ConsumerName]; 18 | 19 | var assignment = consumer.FilterAssigment(message.Topics); 20 | 21 | if (assignment.Any()) 22 | { 23 | consumer?.Pause(assignment); 24 | } 25 | 26 | return Task.CompletedTask; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/consumer/shared/restart-modal/restart-modal.component.html: -------------------------------------------------------------------------------- 1 | 2 |
3 | 4 | 7 |
8 |
9 |
10 | Restart the consumers '{{consumerName}}' from group id '{{groupId}}' will can generate a temporary instability in your system. 11 |
Are you really sure about it?
12 |
13 |
14 |
15 | 16 | 17 |
18 | -------------------------------------------------------------------------------- /website/README.md: -------------------------------------------------------------------------------- 1 | # Website 2 | 3 | This website is built using [Docusaurus 2](https://docusaurus.io/), a modern static website generator. 4 | 5 | ### Installation 6 | 7 | ``` 8 | $ yarn 9 | ``` 10 | 11 | ### Local Development 12 | 13 | ``` 14 | $ yarn start 15 | ``` 16 | 17 | This command starts a local development server and opens up a browser window. Most changes are reflected live without having to restart the server. 18 | 19 | ### Build 20 | 21 | ``` 22 | $ yarn build 23 | ``` 24 | 25 | This command generates static content into the `build` directory and can be served using any static contents hosting service. 26 | 27 | ### Deployment 28 | 29 | Using SSH: 30 | 31 | ``` 32 | $ USE_SSH=true yarn deploy 33 | ``` 34 | 35 | Not using SSH: 36 | 37 | ``` 38 | $ GIT_USER= yarn deploy 39 | ``` 40 | 41 | If you are using GitHub pages for hosting, this command is a convenient way to build the website and push to the `gh-pages` branch. 42 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/group-by.pipe.ts: -------------------------------------------------------------------------------- 1 | import { Pipe, PipeTransform } from '@angular/core'; 2 | 3 | @Pipe({ 4 | name: 'groupBy' 5 | }) 6 | export class GroupByPipe implements PipeTransform { 7 | transform(collection: any[], property: string): any[] { 8 | // prevents the application from breaking if the array of objects doesn't exist yet 9 | if (!collection) { 10 | return null as any; 11 | } 12 | 13 | const groupedCollection = collection.reduce((previous, current) => { 14 | if (! previous[current[property]]) { 15 | previous[current[property]] = [current]; 16 | } else { 17 | previous[current[property]].push(current); 18 | } 19 | 20 | return previous; 21 | }, {}); 22 | 23 | // this will return an array of objects, each object containing a group of objects 24 | return Object.keys(groupedCollection).map(key => ({ key, value: groupedCollection[key] })); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compileOnSave": false, 3 | "compilerOptions": { 4 | "baseUrl": "./", 5 | "outDir": "./dist/out-tsc", 6 | "forceConsistentCasingInFileNames": true, 7 | "strict": true, 8 | "noImplicitOverride": true, 9 | "noPropertyAccessFromIndexSignature": true, 10 | "noImplicitReturns": true, 11 | "noFallthroughCasesInSwitch": true, 12 | "sourceMap": true, 13 | "declaration": false, 14 | "downlevelIteration": true, 15 | "experimentalDecorators": true, 16 | "moduleResolution": "node", 17 | "importHelpers": true, 18 | "target": "es2020", 19 | "module": "es2020", 20 | "lib": [ 21 | "es2020", 22 | "dom" 23 | ] 24 | }, 25 | "angularCompilerOptions": { 26 | "enableI18nLegacyMessageIdFormat": false, 27 | "strictInjectionParameters": true, 28 | "strictInputAccessModifiers": true, 29 | "strictTemplates": true 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/Handlers/ResumeConsumerByNameHandler.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using System.Threading.Tasks; 3 | using KafkaFlow.Admin.Extensions; 4 | using KafkaFlow.Admin.Messages; 5 | using KafkaFlow.Consumers; 6 | 7 | namespace KafkaFlow.Admin.Handlers; 8 | 9 | internal class ResumeConsumerByNameHandler : IMessageHandler 10 | { 11 | private readonly IConsumerAccessor _consumerAccessor; 12 | 13 | public ResumeConsumerByNameHandler(IConsumerAccessor consumerAccessor) => _consumerAccessor = consumerAccessor; 14 | 15 | public Task Handle(IMessageContext context, ResumeConsumerByName message) 16 | { 17 | var consumer = _consumerAccessor[message.ConsumerName]; 18 | 19 | var assignment = consumer.FilterAssigment(message.Topics); 20 | 21 | if (assignment.Any()) 22 | { 23 | consumer?.Resume(assignment); 24 | } 25 | 26 | return Task.CompletedTask; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin/Messages/RewindConsumerOffsetToDateTime.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Runtime.Serialization; 4 | 5 | namespace KafkaFlow.Admin.Messages; 6 | 7 | /// 8 | /// The message that rewind a consumer to a point in time 9 | /// 10 | [DataContract] 11 | public class RewindConsumerOffsetToDateTime : IAdminMessage 12 | { 13 | /// 14 | /// Gets or sets the consumer name that will be rewind 15 | /// 16 | [DataMember(Order = 1)] 17 | public string ConsumerName { get; set; } 18 | 19 | /// 20 | /// Gets or sets the point in time 21 | /// 22 | [DataMember(Order = 2)] 23 | public DateTime DateTime { get; set; } 24 | 25 | /// 26 | /// Gets or sets the topics that will be rewind 27 | /// 28 | [DataMember(Order = 3)] 29 | public IList Topics { get; set; } = new List(); 30 | } 31 | -------------------------------------------------------------------------------- /src/KafkaFlow/Middlewares/Serializer/Resolvers/IMessageTypeResolver.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | 4 | namespace KafkaFlow.Middlewares.Serializer.Resolvers; 5 | 6 | /// 7 | /// Used by the serializer middleware to resolve the type when consuming and store it when producing 8 | /// 9 | public interface IMessageTypeResolver 10 | { 11 | /// 12 | /// Returns the message type when consuming 13 | /// 14 | /// The containing the message and the metadata 15 | /// 16 | ValueTask OnConsumeAsync(IMessageContext context); 17 | 18 | /// 19 | /// Stores the message type somewhere when producing 20 | /// 21 | /// The containing the message and the metadata 22 | ValueTask OnProduceAsync(IMessageContext context); 23 | } 24 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/consumer/shared/reset-modal/reset-modal.component.html: -------------------------------------------------------------------------------- 1 | 2 |
3 | 4 | 7 |
8 |
9 |
10 | Reset the partition-offsets can generate a huge lag at the topic '{{topic}}' to consumers '{{consumerName}}' with group id '{{groupId}}'. 11 |
Are you really sure about it?
12 |
13 |
14 |
15 | 16 | 17 |
18 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/IProducerContext.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow; 2 | 3 | /// 4 | /// Some producer metadata 5 | /// 6 | public interface IProducerContext 7 | { 8 | /// 9 | /// Gets the topic associated with the message 10 | /// 11 | string Topic { get; } 12 | 13 | /// 14 | /// Gets the partition associated with the message 15 | /// 16 | int? Partition { get; } 17 | 18 | /// 19 | /// Gets the partition offset associated with the message 20 | /// 21 | long? Offset { get; } 22 | 23 | /// 24 | /// Gets an instance of IDependencyResolver which provides methods to resolve dependencies. 25 | /// This instance is tied to the producer scope, meaning it is capable of resolving dependencies 26 | /// that are scoped to the lifecycle of a single producer. 27 | /// 28 | IDependencyResolver DependencyResolver { get; } 29 | } 30 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/consumer/shared/resume-modal/resume-modal.component.html: -------------------------------------------------------------------------------- 1 | 2 |
3 | 4 | 7 |
8 |
9 |
10 | Resume all the partitions of the topic '{{topic}}' from consumers '{{consumerName}}' with group id '{{groupId}}' will restart to process the messages. 11 |
Are you really sure about it?
12 |
13 |
14 |
15 | 16 | 17 |
18 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.Dashboard/KafkaFlow.Sample.Dashboard.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net8.0 5 | false 6 | true 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /src/KafkaFlow.Serializer.SchemaRegistry.ConfluentAvro/ConfluentAvroTypeNameResolver.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using Confluent.SchemaRegistry; 3 | using Newtonsoft.Json; 4 | 5 | namespace KafkaFlow.Serializer.SchemaRegistry; 6 | 7 | internal class ConfluentAvroTypeNameResolver : ISchemaRegistryTypeNameResolver 8 | { 9 | private readonly ISchemaRegistryClient _client; 10 | 11 | public ConfluentAvroTypeNameResolver(ISchemaRegistryClient client) 12 | { 13 | _client = client; 14 | } 15 | 16 | public async Task ResolveAsync(int id) 17 | { 18 | var schema = await _client.GetSchemaAsync(id); 19 | 20 | var avroFields = JsonConvert.DeserializeObject(schema.SchemaString); 21 | return $"{avroFields.Namespace}.{avroFields.Name}"; 22 | } 23 | 24 | private class AvroSchemaFields 25 | { 26 | public string Name { get; set; } 27 | 28 | public string Namespace { get; set; } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/IWorkerDistributionStrategy.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Threading.Tasks; 3 | 4 | namespace KafkaFlow; 5 | 6 | /// 7 | /// An interface used to create a distribution strategy 8 | /// 9 | public interface IWorkerDistributionStrategy 10 | { 11 | /// 12 | /// Initializes the distribution strategy, this method is called when a consumer is started 13 | /// 14 | /// List of workers to be initialized 15 | void Initialize(IReadOnlyList workers); 16 | 17 | /// 18 | /// Retrieves an available worker based on the provided distribution strategy context. 19 | /// 20 | /// The distribution strategy context containing message and consumer details. 21 | /// The selected instance. 22 | ValueTask GetWorkerAsync(WorkerDistributionContext context); 23 | } 24 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/consumer/shared/pause-modal/pause-modal.component.html: -------------------------------------------------------------------------------- 1 | 2 |
3 | 4 | 7 |
8 |
9 |
10 | Pause all the partitions of the topic '{{topic}}' from consumers '{{consumerName}}' with group id '{{groupId}}' will interrupt the kafka data processing and, probably, generate lag. 11 |
Are you really sure about it?
12 |
13 |
14 |
15 | 16 | 17 |
18 | -------------------------------------------------------------------------------- /src/KafkaFlow.Microsoft.DependencyInjection/ServiceCollectionExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using global::Microsoft.Extensions.DependencyInjection; 3 | using KafkaFlow.Configuration; 4 | 5 | namespace KafkaFlow; 6 | 7 | /// 8 | /// Extension methods over IServiceCollection 9 | /// 10 | public static class ServiceCollectionExtensions 11 | { 12 | /// 13 | /// Configures KafkaFlow 14 | /// 15 | /// Instance of 16 | /// A handler to configure KafkaFlow 17 | /// 18 | public static IServiceCollection AddKafka( 19 | this IServiceCollection services, 20 | Action kafka) 21 | { 22 | var configurator = new KafkaFlowConfigurator( 23 | new MicrosoftDependencyConfigurator(services), 24 | kafka); 25 | 26 | return services.AddSingleton(configurator); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/KafkaFlow/KafkaFlow.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | KafkaFlow 6 | KafkaFlow main package 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | <_Parameter1>$(AssemblyName).UnitTests 23 | 24 | 25 | -------------------------------------------------------------------------------- /src/KafkaFlow/Middlewares/ConsumerThrottling/Configuration/IConsumerThrottlingThresholdActionConfigurationBuilder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace KafkaFlow.Middlewares.ConsumerThrottling.Configuration; 4 | 5 | /// 6 | /// An interface to configure the actions applied when a throttling threshold is met. 7 | /// 8 | public interface IConsumerThrottlingThresholdActionConfigurationBuilder 9 | { 10 | /// 11 | /// Configures the action to apply when the throttling threshold is met. 12 | /// The factory function takes an instance of , allowing you to resolve dependencies 13 | /// required to create an instance of . 14 | /// 15 | /// 16 | /// A factory function that accepts an and returns an . 17 | /// 18 | void Apply(Func factory); 19 | } 20 | -------------------------------------------------------------------------------- /src/KafkaFlow/Middlewares/ConsumerThrottling/ConsumerThrottlingKafkaLagMetric.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using System.Threading.Tasks; 4 | using KafkaFlow.Consumers; 5 | 6 | namespace KafkaFlow.Middlewares.ConsumerThrottling; 7 | 8 | internal class ConsumerThrottlingKafkaLagMetric : IConsumerThrottlingMetric 9 | { 10 | private readonly IReadOnlyList _consumers; 11 | 12 | public ConsumerThrottlingKafkaLagMetric(IConsumerAccessor consumerAccessor, IEnumerable consumersNames) 13 | { 14 | _consumers = consumerAccessor.All 15 | .Where(consumer => consumersNames.Contains(consumer.ConsumerName)) 16 | .ToList() 17 | .AsReadOnly(); 18 | } 19 | 20 | public Task GetValueAsync() 21 | { 22 | var lag = _consumers 23 | .SelectMany(x => x.GetTopicPartitionsLag()) 24 | .Select(x => x.Lag) 25 | .Sum(); 26 | 27 | return Task.FromResult(lag); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: init_broker shutdown_broker 2 | 3 | init_broker: 4 | @echo command | date 5 | @echo Initializing Kafka broker 6 | docker compose -f docker-compose.yml up -d 7 | 8 | shutdown_broker: 9 | @echo command | date 10 | @echo Shutting down kafka broker 11 | docker compose -f docker-compose.yml down 12 | 13 | restore: 14 | dotnet restore KafkaFlow.sln 15 | 16 | build: 17 | dotnet build KafkaFlow.sln 18 | 19 | unit_tests: 20 | @echo command | date 21 | @echo Running unit tests 22 | dotnet test tests/KafkaFlow.UnitTests/KafkaFlow.UnitTests.csproj --framework netcoreapp2.1 --logger "console;verbosity=detailed" 23 | dotnet test tests/KafkaFlow.UnitTests/KafkaFlow.UnitTests.csproj --framework netcoreapp3.1 --logger "console;verbosity=detailed" 24 | 25 | integration_tests: 26 | @echo command | date 27 | make init_broker 28 | @echo Running integration tests 29 | dotnet test tests/KafkaFlow.IntegrationTests/KafkaFlow.IntegrationTests.csproj -c Release --logger "console;verbosity=detailed" 30 | make shutdown_broker 31 | -------------------------------------------------------------------------------- /src/KafkaFlow.Serializer.SchemaRegistry.ConfluentProtobuf/ConfluentProtobufDeserializer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Threading.Tasks; 5 | using Confluent.SchemaRegistry.Serdes; 6 | 7 | namespace KafkaFlow.Serializer.SchemaRegistry; 8 | 9 | /// 10 | /// A protobuf message serializer integrated with the confluent schema registry 11 | /// 12 | public class ConfluentProtobufDeserializer : IDeserializer 13 | { 14 | /// 15 | public Task DeserializeAsync(Stream input, Type type, ISerializerContext context) 16 | { 17 | return ConfluentDeserializerWrapper 18 | .GetOrCreateDeserializer( 19 | type, 20 | () => Activator 21 | .CreateInstance( 22 | typeof(ProtobufDeserializer<>).MakeGenericType(type), 23 | (IEnumerable>)null)) 24 | .DeserializeAsync(input, context); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/KafkaFlow.Abstractions/IEvent.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | 4 | namespace KafkaFlow; 5 | 6 | /// 7 | /// Represents an Event to be subscribed. 8 | /// 9 | public interface IEvent 10 | { 11 | /// 12 | /// Subscribes to the event. 13 | /// 14 | /// The handler to be called when the event is fired. 15 | /// Event subscription reference 16 | IEventSubscription Subscribe(Func handler); 17 | } 18 | 19 | /// 20 | /// Represents an Event to be subscribed. 21 | /// 22 | /// The argument expected by the event. 23 | public interface IEvent 24 | { 25 | /// 26 | /// Subscribes to the event. 27 | /// 28 | /// The handler to be called when the event is fired. 29 | /// Event subscription reference 30 | IEventSubscription Subscribe(Func handler); 31 | } 32 | -------------------------------------------------------------------------------- /src/KafkaFlow.SchemaRegistry/ClusterConfigurationBuilderExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Confluent.SchemaRegistry; 3 | using KafkaFlow.Configuration; 4 | 5 | namespace KafkaFlow; 6 | 7 | /// 8 | /// No needed 9 | /// 10 | public static class ClusterConfigurationBuilderExtensions 11 | { 12 | /// 13 | /// Configures schema registry to the cluster 14 | /// 15 | /// Instance of 16 | /// A handler to set the configuration values 17 | /// 18 | public static IClusterConfigurationBuilder WithSchemaRegistry( 19 | this IClusterConfigurationBuilder cluster, 20 | Action handler) 21 | { 22 | var config = new SchemaRegistryConfig(); 23 | handler(config); 24 | cluster.DependencyConfigurator.AddSingleton(_ => new CachedSchemaRegistryClient(config)); 25 | return cluster; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/KafkaFlow/Configuration/MiddlewareInstanceContainer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace KafkaFlow.Configuration; 4 | 5 | internal class MiddlewareInstanceContainer : IMiddlewareInstanceContainer 6 | { 7 | private readonly object _sync = new(); 8 | private readonly Factory _factory; 9 | 10 | private IMessageMiddleware _instance; 11 | 12 | public MiddlewareInstanceContainer(Guid id, Factory factory) 13 | { 14 | this.Id = id; 15 | _factory = factory; 16 | } 17 | 18 | public Guid Id { get; } 19 | 20 | public IMessageMiddleware GetInstance(IDependencyResolver resolver) 21 | { 22 | if (_instance is not null) 23 | { 24 | return _instance; 25 | } 26 | 27 | lock (_sync) 28 | { 29 | if (_instance is not null) 30 | { 31 | return _instance; 32 | } 33 | 34 | _instance = _factory(resolver); 35 | } 36 | 37 | return _instance; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/KafkaFlow/IKafkaBus.cs: -------------------------------------------------------------------------------- 1 | using System.Threading; 2 | using System.Threading.Tasks; 3 | using KafkaFlow.Consumers; 4 | using KafkaFlow.Producers; 5 | 6 | namespace KafkaFlow; 7 | 8 | /// 9 | /// Provides access to the kafka bus operations 10 | /// 11 | public interface IKafkaBus 12 | { 13 | /// 14 | /// Gets all configured consumers 15 | /// 16 | IConsumerAccessor Consumers { get; } 17 | 18 | /// 19 | /// Gets all configured producers 20 | /// 21 | IProducerAccessor Producers { get; } 22 | 23 | /// 24 | /// Starts all consumers 25 | /// 26 | /// A used to stop the operation. 27 | /// 28 | Task StartAsync(CancellationToken stopCancellationToken = default); 29 | 30 | /// 31 | /// Stops all consumers 32 | /// 33 | /// 34 | Task StopAsync(); 35 | } 36 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/ClientApp/src/app/consumer/shared/workers-count-modal/workers-count-modal.component.html: -------------------------------------------------------------------------------- 1 | 2 |
3 | 4 | 7 |
8 |
9 |
10 | 11 | 12 |
13 | 14 |
15 |
16 |
17 |
18 | 19 | 20 |
21 | -------------------------------------------------------------------------------- /src/KafkaFlow/Middlewares/TypedHandler/HandlerTypeMapping.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace KafkaFlow.Middlewares.TypedHandler; 5 | 6 | internal class HandlerTypeMapping 7 | { 8 | private static readonly IReadOnlyList s_emptyList = new List().AsReadOnly(); 9 | 10 | private readonly Dictionary> _mapping = new(); 11 | 12 | public void AddMapping(Type messageType, Type handlerType) 13 | { 14 | if (!_mapping.TryGetValue(messageType, out var handlers)) 15 | { 16 | handlers = new List(); 17 | _mapping.Add(messageType, handlers); 18 | } 19 | 20 | handlers.Add(handlerType); 21 | } 22 | 23 | public IReadOnlyList GetHandlersTypes(Type messageType) 24 | { 25 | if (messageType is null) 26 | { 27 | return s_emptyList; 28 | } 29 | 30 | return _mapping.TryGetValue(messageType, out var handlerType) ? 31 | handlerType : 32 | s_emptyList; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/KafkaFlow.Serializer.SchemaRegistry.ConfluentJson/ConsumerConfigurationBuilderExtensions.cs: -------------------------------------------------------------------------------- 1 | using KafkaFlow.Configuration; 2 | using KafkaFlow.Middlewares.Serializer.Resolvers; 3 | using KafkaFlow.Serializer.SchemaRegistry; 4 | 5 | namespace KafkaFlow; 6 | 7 | /// 8 | /// No needed 9 | /// 10 | public static class ConsumerConfigurationBuilderExtensions 11 | { 12 | /// 13 | /// Registers a middleware to deserialize json messages using schema registry 14 | /// 15 | /// The middleware configuration builder 16 | /// The message type 17 | /// 18 | public static IConsumerMiddlewareConfigurationBuilder AddSchemaRegistryJsonSerializer( 19 | this IConsumerMiddlewareConfigurationBuilder middlewares) 20 | { 21 | return middlewares.AddDeserializer( 22 | resolver => new ConfluentJsonDeserializer(), 23 | _ => new SingleMessageTypeResolver(typeof(TMessage))); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/KafkaFlow/Consumers/TopicPartitionLag.cs: -------------------------------------------------------------------------------- 1 | namespace KafkaFlow.Consumers; 2 | 3 | /// 4 | /// Represents the lag in a specific topic and partition 5 | /// 6 | public class TopicPartitionLag 7 | { 8 | /// 9 | /// Initializes a new instance of the class. 10 | /// 11 | /// The topic name 12 | /// The partition value 13 | /// The lag value 14 | public TopicPartitionLag(string topic, int partition, long lag) 15 | { 16 | this.Topic = topic; 17 | this.Partition = partition; 18 | this.Lag = lag; 19 | } 20 | 21 | /// 22 | /// Gets the topic name 23 | /// 24 | public string Topic { get; } 25 | 26 | /// 27 | /// Gets the partition value 28 | /// 29 | public int Partition { get; } 30 | 31 | /// 32 | /// Gets the lag 33 | /// 34 | public long Lag { get; } 35 | } 36 | -------------------------------------------------------------------------------- /src/KafkaFlow/Middlewares/TypedHandler/HandlerExecutor.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Threading.Tasks; 4 | 5 | namespace KafkaFlow.Middlewares.TypedHandler; 6 | 7 | internal abstract class HandlerExecutor 8 | { 9 | private static readonly ConcurrentDictionary s_executors = new(); 10 | 11 | public static HandlerExecutor GetExecutor(Type messageType) 12 | { 13 | return s_executors.SafeGetOrAdd( 14 | messageType, 15 | _ => (HandlerExecutor)Activator.CreateInstance(typeof(InnerHandlerExecutor<>).MakeGenericType(messageType))); 16 | } 17 | 18 | public abstract Task Execute(object handler, IMessageContext context, object message); 19 | 20 | private class InnerHandlerExecutor : HandlerExecutor 21 | { 22 | public override Task Execute(object handler, IMessageContext context, object message) 23 | { 24 | var h = (IMessageHandler)handler; 25 | 26 | return h.Handle(context, (T)message); 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /samples/KafkaFlow.Sample.Dashboard/README.md: -------------------------------------------------------------------------------- 1 | # KafkaFlow.Sample.Dashboard 2 | 3 | This sample shows how to use KafkaFlow to expose an administration Dashboard. 4 | 5 | ## How to run 6 | 7 | ### Requirements 8 | 9 | - [.NET 6.0 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/6.0) 10 | - [Docker Desktop](https://www.docker.com/products/docker-desktop/) 11 | 12 | ### Start the cluster 13 | 14 | Using your terminal of choice, start the cluster. 15 | You can find a docker-compose file at the root of this repository. 16 | Position the terminal in that folder and run the following command. 17 | 18 | ```bash 19 | docker-compose up -d 20 | ``` 21 | 22 | ### Build Dashboard UI 23 | 24 | Using your terminal of choice, navigate to `kafkaflow\src\KafkaFlow.Admin.Dashboard\ClientApp` folder and run the following command: 25 | 26 | ```bash 27 | ng build 28 | ``` 29 | 30 | ### Run the Sample 31 | 32 | Using your terminal of choice, start the sample for the sample folder. 33 | 34 | ```bash 35 | dotnet run 36 | ``` 37 | 38 | The dashboard UI will be available at `/kafkaflow`. 39 | -------------------------------------------------------------------------------- /src/KafkaFlow.Admin.Dashboard/DashboardConfigurationBuilder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.AspNetCore.Builder; 3 | using Microsoft.AspNetCore.Http; 4 | 5 | namespace KafkaFlow.Admin.Dashboard; 6 | 7 | internal class DashboardConfigurationBuilder : IDashboardConfigurationBuilder 8 | { 9 | private readonly PathString _basePath = "/kafkaflow"; 10 | 11 | private Action _requestHandler = _ => { }; 12 | private Action _endpointHandler = _ => { }; 13 | 14 | public IDashboardConfigurationBuilder ConfigureRequestPipeline(Action requestHandler) 15 | { 16 | _requestHandler = requestHandler; 17 | return this; 18 | } 19 | 20 | public IDashboardConfigurationBuilder ConfigureEndpoint(Action endpointHandler) 21 | { 22 | _endpointHandler = endpointHandler; 23 | return this; 24 | } 25 | 26 | public DashboardConfiguration Build() 27 | { 28 | return new(_basePath, _requestHandler, _endpointHandler); 29 | } 30 | } 31 | --------------------------------------------------------------------------------