├── samples ├── basic │ ├── .dockerignore │ ├── deploy │ │ ├── service_account.yaml │ │ ├── crds │ │ │ ├── cr.yaml │ │ │ └── crd.yaml │ │ ├── role_binding.yaml │ │ ├── operator.yaml │ │ └── role.yaml │ ├── Dockerfile │ ├── k8s.Operators.Samples.Basic.csproj │ ├── MyResource.cs │ ├── MyResourceController.cs │ ├── README.md │ └── Program.cs └── dynamic │ ├── .dockerignore │ ├── deploy │ ├── service_account.yaml │ ├── crds │ │ ├── cr.yaml │ │ └── crd.yaml │ ├── role_binding.yaml │ ├── operator.yaml │ └── role.yaml │ ├── Dockerfile │ ├── MyDynamicResource.cs │ ├── k8s.Operators.Samples.Dynamic.csproj │ ├── README.md │ ├── MyDynamicResourceController.cs │ └── Program.cs ├── docs └── writing-csharp-operator.md ├── src └── k8s.Operators │ ├── k8s.Operators.snk │ ├── Models │ ├── IStatus.cs │ ├── DynamicCustomResource.cs │ ├── CustomResourceList.cs │ ├── RetryPolicy.cs │ ├── CustomResourceDefinitionAttribute.cs │ ├── CustomResourceEvent.cs │ ├── Disposable.cs │ ├── OperatorConfiguration.cs │ └── CustomResource.cs │ ├── Logging │ ├── SilentLogger.cs │ └── ConsoleTracingInterceptor.cs │ ├── IController.cs │ ├── k8s.Operators.csproj │ ├── IOperator.cs │ ├── EventWatcher.cs │ ├── ResourceChangeTracker.cs │ ├── EventManager.cs │ ├── Operator.cs │ └── Controller.cs ├── tests └── k8s.Operators.Tests │ ├── TestableDynamicController.cs │ ├── TestableDynamicCustomResource.cs │ ├── TestableCustomResource.cs │ ├── k8s.Operators.Tests.csproj │ ├── TestableOperator.cs │ ├── TestableController.cs │ ├── BaseTests.cs │ ├── OperatorTests.cs │ └── ControllerTests.cs ├── .vscode ├── launch.json └── tasks.json ├── .github └── workflows │ └── dotnet-core.yml ├── README.md ├── csharp-operator-sdk.sln ├── .gitignore └── LICENSE /samples/basic/.dockerignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | obj/ -------------------------------------------------------------------------------- /samples/dynamic/.dockerignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | obj/ -------------------------------------------------------------------------------- /docs/writing-csharp-operator.md: -------------------------------------------------------------------------------- 1 | # Writing a Kubernetes Operator in C# 2 | 3 | TODO -------------------------------------------------------------------------------- /samples/basic/deploy/service_account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: basic-operator 5 | -------------------------------------------------------------------------------- /samples/dynamic/deploy/service_account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: dynamic-operator 5 | -------------------------------------------------------------------------------- /src/k8s.Operators/k8s.Operators.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/falox/csharp-operator-sdk/HEAD/src/k8s.Operators/k8s.Operators.snk -------------------------------------------------------------------------------- /samples/basic/deploy/crds/cr.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "csharp-operator.example.com/v1" 2 | kind: MyResource 3 | metadata: 4 | name: mr1 5 | spec: 6 | desiredProperty: 1 -------------------------------------------------------------------------------- /samples/dynamic/deploy/crds/cr.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "csharp-operator.example.com/v1" 2 | kind: MyResource 3 | metadata: 4 | name: mr1 5 | spec: 6 | desiredProperty: 1 -------------------------------------------------------------------------------- /src/k8s.Operators/Models/IStatus.cs: -------------------------------------------------------------------------------- 1 | namespace k8s.Operators 2 | { 3 | /// 4 | /// Kubernetes custom resource that exposes status 5 | /// 6 | public interface IStatus 7 | { 8 | object Status { get; set; } 9 | } 10 | } -------------------------------------------------------------------------------- /samples/basic/deploy/role_binding.yaml: -------------------------------------------------------------------------------- 1 | kind: RoleBinding 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | metadata: 4 | name: basic-operator 5 | subjects: 6 | - kind: ServiceAccount 7 | name: basic-operator 8 | roleRef: 9 | kind: Role 10 | name: basic-operator 11 | apiGroup: rbac.authorization.k8s.io 12 | -------------------------------------------------------------------------------- /samples/dynamic/deploy/role_binding.yaml: -------------------------------------------------------------------------------- 1 | kind: RoleBinding 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | metadata: 4 | name: dynamic-operator 5 | subjects: 6 | - kind: ServiceAccount 7 | name: dynamic-operator 8 | roleRef: 9 | kind: Role 10 | name: dynamic-operator 11 | apiGroup: rbac.authorization.k8s.io 12 | -------------------------------------------------------------------------------- /src/k8s.Operators/Models/DynamicCustomResource.cs: -------------------------------------------------------------------------------- 1 | using System.Dynamic; 2 | 3 | namespace k8s.Operators 4 | { 5 | /// 6 | /// Represents a Kubernetes custom resource with dynamic typed spec and status 7 | /// 8 | public abstract class DynamicCustomResource : CustomResource 9 | { 10 | } 11 | } -------------------------------------------------------------------------------- /samples/basic/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/dotnet/core/sdk:3.1 AS build-env 2 | WORKDIR /app 3 | COPY . ./ 4 | 5 | RUN dotnet restore 6 | RUN dotnet publish samples/basic/k8s.Operators.Samples.Basic.csproj -c Release -o out 7 | 8 | FROM mcr.microsoft.com/dotnet/core/aspnet:3.1 9 | WORKDIR /app 10 | COPY --from=build-env /app/out . 11 | ENTRYPOINT ["dotnet", "k8s.Operators.Samples.Basic.dll", "--debug"] 12 | -------------------------------------------------------------------------------- /samples/dynamic/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/dotnet/core/sdk:3.1 AS build-env 2 | WORKDIR /app 3 | COPY . ./ 4 | 5 | RUN dotnet restore 6 | RUN dotnet publish samples/dynamic/k8s.Operators.Samples.Dynamic.csproj -c Release -o out 7 | 8 | FROM mcr.microsoft.com/dotnet/core/aspnet:3.1 9 | WORKDIR /app 10 | COPY --from=build-env /app/out . 11 | ENTRYPOINT ["dotnet", "k8s.Operators.Samples.Dynamic.dll", "--debug"] 12 | -------------------------------------------------------------------------------- /samples/dynamic/MyDynamicResource.cs: -------------------------------------------------------------------------------- 1 | using k8s.Operators; 2 | using Newtonsoft.Json; 3 | using System.Collections.Generic; 4 | 5 | namespace k8s.Operators.Samples.Dynamic 6 | { 7 | [CustomResourceDefinition("csharp-operator.example.com", "v1", "myresources")] 8 | public class MyDynamicResource : DynamicCustomResource 9 | { 10 | public override string ToString() 11 | { 12 | return $"{Metadata.NamespaceProperty}/{Metadata.Name} (gen: {Metadata.Generation}), Spec: {JsonConvert.SerializeObject(Spec)} Status: {JsonConvert.SerializeObject(Status ?? new object())}"; 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/k8s.Operators/Models/CustomResourceList.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Diagnostics.CodeAnalysis; 3 | using k8s.Models; 4 | using Newtonsoft.Json; 5 | 6 | namespace k8s.Operators 7 | { 8 | /// 9 | /// Represents a Kubernetes list of custom resources of type T 10 | /// 11 | [ExcludeFromCodeCoverage] 12 | public abstract class CustomResourceList : KubernetesObject where T : CustomResource 13 | { 14 | [JsonProperty("metadata")] 15 | public V1ListMeta Metadata { get; set; } 16 | 17 | [JsonProperty("items")] 18 | public List Items { get; set; } 19 | } 20 | } -------------------------------------------------------------------------------- /tests/k8s.Operators.Tests/TestableDynamicController.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using System.Threading; 3 | 4 | namespace k8s.Operators.Tests 5 | { 6 | public class TestableDynamicController : Controller 7 | { 8 | public TestableDynamicController() : base(OperatorConfiguration.Default, null, null) 9 | { 10 | } 11 | 12 | protected override Task AddOrModifyAsync(TestableDynamicCustomResource resource, CancellationToken cancellationToken) 13 | { 14 | resource.Status.property = resource.Spec.property; 15 | return Task.CompletedTask; 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /samples/basic/k8s.Operators.Samples.Basic.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp3.1 6 | 7 | false 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /samples/dynamic/k8s.Operators.Samples.Dynamic.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp3.1 6 | 7 | false 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /src/k8s.Operators/Logging/SilentLogger.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics.CodeAnalysis; 3 | using Microsoft.Extensions.Logging; 4 | 5 | namespace k8s.Operators.Logging 6 | { 7 | /// 8 | /// Empty ILogger that doesn't log, used as fallback when no logger is passed to the library. 9 | /// 10 | [ExcludeFromCodeCoverage] 11 | internal class SilentLogger : Disposable, ILogger 12 | { 13 | public static ILogger Instance = new SilentLogger(); 14 | 15 | public IDisposable BeginScope(TState state) => this; 16 | 17 | public bool IsEnabled(LogLevel logLevel) => false; 18 | 19 | public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) 20 | { 21 | } 22 | } 23 | } -------------------------------------------------------------------------------- /tests/k8s.Operators.Tests/TestableDynamicCustomResource.cs: -------------------------------------------------------------------------------- 1 | using System.Dynamic; 2 | using k8s.Models; 3 | using k8s.Operators; 4 | 5 | namespace k8s.Operators.Tests 6 | { 7 | [CustomResourceDefinition("group", "v1", "resources")] 8 | public class TestableDynamicCustomResource : Operators.DynamicCustomResource 9 | { 10 | public TestableDynamicCustomResource() : base() 11 | { 12 | Metadata = new Models.V1ObjectMeta(); 13 | Metadata.EnsureFinalizers().Add(CustomResourceDefinitionAttribute.DEFAULT_FINALIZER); 14 | Metadata.NamespaceProperty = "ns1"; 15 | Metadata.Name = "resource1"; 16 | Metadata.Generation = 1; 17 | Metadata.Uid = "id1"; 18 | 19 | Spec = new ExpandoObject(); 20 | Status = new ExpandoObject(); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /samples/basic/MyResource.cs: -------------------------------------------------------------------------------- 1 | using Newtonsoft.Json; 2 | 3 | namespace k8s.Operators.Samples.Basic 4 | { 5 | [CustomResourceDefinition("csharp-operator.example.com", "v1", "myresources")] 6 | public class MyResource : CustomResource 7 | { 8 | public class MyResourceSpec 9 | { 10 | [JsonProperty("desiredProperty")] 11 | public int Desired { get; set; } 12 | } 13 | 14 | public class MyResourceStatus 15 | { 16 | [JsonProperty("actualProperty")] 17 | public int Actual { get; set; } 18 | } 19 | 20 | public override string ToString() 21 | { 22 | return $"{Metadata.NamespaceProperty}/{Metadata.Name} (gen: {Metadata.Generation}), Spec: {Spec.Desired} Status: {Status?.Actual}"; 23 | } 24 | } 25 | } -------------------------------------------------------------------------------- /tests/k8s.Operators.Tests/TestableCustomResource.cs: -------------------------------------------------------------------------------- 1 | using k8s.Operators; 2 | 3 | namespace k8s.Operators.Tests 4 | { 5 | [CustomResourceDefinition("group", "v1", "resources")] 6 | public class TestableCustomResource : Operators.CustomResource 7 | { 8 | public TestableCustomResource() 9 | { 10 | Metadata = new Models.V1ObjectMeta(); 11 | Metadata.NamespaceProperty = "ns1"; 12 | Metadata.Name = "resource1"; 13 | Metadata.Generation = 1; 14 | Metadata.Uid = "id1"; 15 | } 16 | 17 | public class TestableSpec 18 | { 19 | public string Property { get; set; } 20 | } 21 | 22 | public class TestableStatus 23 | { 24 | public string Property { get; set; } 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /samples/basic/deploy/crds/crd.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apiextensions.k8s.io/v1 2 | kind: CustomResourceDefinition 3 | metadata: 4 | name: myresources.csharp-operator.example.com 5 | spec: 6 | group: csharp-operator.example.com 7 | versions: 8 | - name: v1 9 | served: true 10 | storage: true 11 | schema: 12 | openAPIV3Schema: 13 | type: object 14 | properties: 15 | spec: 16 | type: object 17 | properties: 18 | desiredProperty: 19 | type: integer 20 | status: 21 | type: object 22 | properties: 23 | actualProperty: 24 | type: integer 25 | subresources: 26 | status: {} 27 | scope: Namespaced 28 | names: 29 | plural: myresources 30 | singular: myresource 31 | kind: MyResource 32 | shortNames: 33 | - mr -------------------------------------------------------------------------------- /samples/dynamic/deploy/crds/crd.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apiextensions.k8s.io/v1 2 | kind: CustomResourceDefinition 3 | metadata: 4 | name: myresources.csharp-operator.example.com 5 | spec: 6 | group: csharp-operator.example.com 7 | versions: 8 | - name: v1 9 | served: true 10 | storage: true 11 | schema: 12 | openAPIV3Schema: 13 | type: object 14 | properties: 15 | spec: 16 | type: object 17 | properties: 18 | desiredProperty: 19 | type: integer 20 | status: 21 | type: object 22 | properties: 23 | actualProperty: 24 | type: integer 25 | subresources: 26 | status: {} 27 | scope: Namespaced 28 | names: 29 | plural: myresources 30 | singular: myresource 31 | kind: MyResource 32 | shortNames: 33 | - mr -------------------------------------------------------------------------------- /src/k8s.Operators/Models/RetryPolicy.cs: -------------------------------------------------------------------------------- 1 | namespace k8s.Operators 2 | { 3 | /// 4 | /// Represents a retry policy for a custom resource controller 5 | /// 6 | public class RetryPolicy 7 | { 8 | /// 9 | /// Max number of attempts 10 | /// 11 | public int MaxAttempts { get; set; } = 3; 12 | 13 | /// 14 | /// Initial time delay (in milliseconds) before to process again the event. 15 | /// After an attempt, the delay is incresead by multiplying it by DelayMultiplier 16 | /// 17 | public int InitialDelay { get; set; } = 5000; 18 | 19 | /// 20 | /// The multiplier applied to the delay after each attempt. 21 | /// DelayMultiplier = 1 keeps the delay constant 22 | /// 23 | public double DelayMultiplier { get; set; } = 1.5; 24 | } 25 | } -------------------------------------------------------------------------------- /src/k8s.Operators/Models/CustomResourceDefinitionAttribute.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace k8s.Operators 4 | { 5 | /// 6 | /// Describe the essential custom resource definition attributes used by the Controller 7 | /// 8 | [AttributeUsage(AttributeTargets.Class)] 9 | public class CustomResourceDefinitionAttribute : Attribute 10 | { 11 | public const string DEFAULT_FINALIZER = "operator.default.finalizer"; 12 | 13 | public CustomResourceDefinitionAttribute(string group, string version, string plural) 14 | { 15 | Group = group; 16 | Version = version; 17 | Plural = plural; 18 | } 19 | 20 | public string Group { get; private set; } 21 | public string Version { get; private set; } 22 | public string Plural { get; private set; } 23 | public string Finalizer { get; set; } = DEFAULT_FINALIZER; 24 | } 25 | } -------------------------------------------------------------------------------- /samples/basic/deploy/operator.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: basic-operator 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | name: basic-operator 10 | template: 11 | metadata: 12 | labels: 13 | name: basic-operator 14 | spec: 15 | serviceAccountName: basic-operator 16 | containers: 17 | - name: basic-operator 18 | image: csharp-basic-operator 19 | imagePullPolicy: IfNotPresent 20 | env: 21 | - name: WATCH_NAMESPACE 22 | valueFrom: 23 | fieldRef: 24 | fieldPath: metadata.namespace 25 | - name: LOG_LEVEL 26 | value: "information" 27 | - name: RETRY_MAX_ATTEMPTS 28 | value: "3" 29 | - name: RETRY_INITIAL_DELAY 30 | value: "5000" 31 | - name: RETRY_DELAY_MULTIPLIER 32 | value: "1.5" -------------------------------------------------------------------------------- /samples/dynamic/deploy/operator.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: dynamic-operator 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | name: dynamic-operator 10 | template: 11 | metadata: 12 | labels: 13 | name: dynamic-operator 14 | spec: 15 | serviceAccountName: dynamic-operator 16 | containers: 17 | - name: dynamic-operator 18 | image: csharp-dynamic-operator 19 | imagePullPolicy: IfNotPresent 20 | env: 21 | - name: WATCH_NAMESPACE 22 | valueFrom: 23 | fieldRef: 24 | fieldPath: metadata.namespace 25 | - name: LOG_LEVEL 26 | value: "information" 27 | - name: RETRY_MAX_ATTEMPTS 28 | value: "3" 29 | - name: RETRY_INITIAL_DELAY 30 | value: "5000" 31 | - name: RETRY_DELAY_MULTIPLIER 32 | value: "1.5" -------------------------------------------------------------------------------- /src/k8s.Operators/IController.cs: -------------------------------------------------------------------------------- 1 | using System.Threading; 2 | using System.Threading.Tasks; 3 | 4 | namespace k8s.Operators 5 | { 6 | /// 7 | /// Controller of a custom resource 8 | /// 9 | public interface IController 10 | { 11 | /// 12 | /// Processes a custom resource event 13 | /// 14 | /// The event to handle 15 | /// Signals if the current execution has been canceled 16 | Task ProcessEventAsync(CustomResourceEvent resourceEvent, CancellationToken cancellationToken); 17 | 18 | /// 19 | /// Retry policy for the controller 20 | /// 21 | RetryPolicy RetryPolicy { get; } 22 | } 23 | 24 | /// 25 | /// Controller of a custom resource of type T 26 | /// 27 | public interface IController : IController where T : CustomResource 28 | { 29 | } 30 | } -------------------------------------------------------------------------------- /tests/k8s.Operators.Tests/k8s.Operators.Tests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp3.1 5 | 6 | false 7 | 8 | 9 | 10 | 11 | runtime; build; native; contentfiles; analyzers; buildtransitive 12 | all 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /samples/dynamic/README.md: -------------------------------------------------------------------------------- 1 | # C# Dynamic Kubernetes Operator 2 | 3 | The *Dynamic Operator* is a variant of the [*Basic Operator*](../basic/README.md). 4 | 5 | The `MyResource` class of the Basic Operator has been replaced with the `MyDynamicResource` class: 6 | 7 | ```csharp 8 | [CustomResourceDefinition("csharp-operator.example.com", "v1", "myresources")] 9 | public class MyDynamicResource : DynamicCustomResource 10 | { 11 | } 12 | ``` 13 | 14 | A `DynamicCustomResource` doesn't force you to strongly define the schema of `Spec` and `Status` in advance ([pros and cons](https://docs.microsoft.com/en-us/archive/msdn-magazine/2011/february/msdn-magazine-dynamic-net-understanding-the-dynamic-keyword-in-csharp-4)), and you can read and write any property without errors at compile time: 15 | 16 | ```csharp 17 | string x = resource.Spec.foo; 18 | resource.Status.bar = 123; 19 | ``` 20 | 21 | You can run and deploy the Dynamic Operator by following the same [instructions of the Basic Operator](../basic/README.md). Just replace `basic` with `dynamic` in the paths and commands. -------------------------------------------------------------------------------- /src/k8s.Operators/Models/CustomResourceEvent.cs: -------------------------------------------------------------------------------- 1 | namespace k8s.Operators 2 | { 3 | /// 4 | /// Represents a custom resource event 5 | /// 6 | public class CustomResourceEvent 7 | { 8 | public CustomResourceEvent(WatchEventType type, CustomResource resource) 9 | { 10 | Type = type; 11 | Resource = resource; 12 | } 13 | 14 | /// 15 | /// The type of the event 16 | /// 17 | /// 18 | public WatchEventType Type { get; } 19 | 20 | /// 21 | /// The watched custom resource 22 | /// 23 | /// 24 | public CustomResource Resource { get; } 25 | 26 | /// 27 | /// Returns the Uid of the custom resource 28 | /// 29 | public string ResourceUid => Resource.Metadata.Uid; 30 | 31 | public override string ToString() 32 | { 33 | return $"{Type} {Resource}"; 34 | } 35 | } 36 | } -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to find out which attributes exist for C# debugging 3 | // Use hover for the description of the existing attributes 4 | // For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": ".NET Core Launch (console)", 9 | "type": "coreclr", 10 | "request": "launch", 11 | "preLaunchTask": "build", 12 | // If you have changed target frameworks, make sure to update the program path. 13 | "program": "${workspaceFolder}/samples/basic/bin/Debug/netcoreapp3.1/k8s.Operators.Samples.Basic.dll", 14 | "args": [], 15 | "cwd": "${workspaceFolder}/samples/basic", 16 | // For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console 17 | "console": "internalConsole", 18 | "stopAtEntry": false 19 | }, 20 | { 21 | "name": ".NET Core Attach", 22 | "type": "coreclr", 23 | "request": "attach", 24 | "processId": "${command:pickProcess}" 25 | } 26 | ] 27 | } -------------------------------------------------------------------------------- /src/k8s.Operators/Models/Disposable.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading; 3 | 4 | namespace k8s.Operators 5 | { 6 | /// 7 | /// Represents a disposable object 8 | /// 9 | public abstract class Disposable : IDisposable 10 | { 11 | private volatile int _barrier; 12 | private volatile bool _disposing; 13 | private volatile bool _disposed; 14 | 15 | public bool IsDisposed => _disposed; 16 | public bool IsDisposing => _disposing; 17 | 18 | public void Dispose() 19 | { 20 | Dispose(true); 21 | GC.SuppressFinalize(this); 22 | } 23 | 24 | protected void Dispose(bool disposing) 25 | { 26 | if (Interlocked.CompareExchange(ref _barrier, 1, 0) == 0) 27 | { 28 | // This block can be executed only once 29 | 30 | _disposing = true; 31 | 32 | if (disposing) 33 | { 34 | DisposeInternal(); 35 | } 36 | 37 | _disposing = false; 38 | _disposed = true; 39 | } 40 | } 41 | 42 | protected virtual void DisposeInternal() 43 | { 44 | } 45 | } 46 | } -------------------------------------------------------------------------------- /src/k8s.Operators/Models/OperatorConfiguration.cs: -------------------------------------------------------------------------------- 1 | namespace k8s.Operators 2 | { 3 | /// 4 | /// Represents the operator configuration. 5 | /// 6 | public class OperatorConfiguration 7 | { 8 | /// 9 | /// Returns the default configuration. 10 | /// 11 | public static OperatorConfiguration Default = new OperatorConfiguration(); // TODO: make readonly 12 | 13 | /// 14 | /// The namespace to watch. Set to empty string to watch all namespaces. 15 | /// 16 | public string WatchNamespace { get; set; } = ""; 17 | 18 | /// 19 | /// The label selector to filter events. Set to null to not filter. 20 | /// 21 | public string WatchLabelSelector { get; set; } = null; 22 | 23 | /// 24 | /// The retry policy for the event handling. 25 | /// 26 | public RetryPolicy RetryPolicy { get; set; } = new RetryPolicy(); 27 | 28 | /// 29 | /// If true, discards the event whose spec generation has already been received and processed 30 | /// 31 | public bool DiscardDuplicateSpecGenerations { get; set; } = true; 32 | } 33 | } -------------------------------------------------------------------------------- /src/k8s.Operators/Models/CustomResource.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using k8s.Models; 3 | using Newtonsoft.Json; 4 | 5 | namespace k8s.Operators 6 | { 7 | /// 8 | /// Represents a Kubernetes custom resource 9 | /// 10 | public abstract class CustomResource : KubernetesObject, IKubernetesObject 11 | { 12 | [JsonProperty("metadata")] 13 | public V1ObjectMeta Metadata { get; set; } 14 | 15 | public override string ToString() 16 | { 17 | return $"{Metadata.NamespaceProperty}/{Metadata.Name} (gen: {Metadata.Generation}, uid: {Metadata.Uid})"; 18 | } 19 | } 20 | 21 | /// 22 | /// Represents a Kubernetes custom resource that has a spec 23 | /// 24 | public abstract class CustomResource : CustomResource, ISpec 25 | { 26 | [JsonProperty("spec")] 27 | public TSpec Spec { get; set; } 28 | } 29 | 30 | /// 31 | /// Represents a Kubernetes custom resource that has a spec and status 32 | /// 33 | public abstract class CustomResource : CustomResource, IStatus, IStatus 34 | { 35 | [JsonProperty("status")] 36 | public TStatus Status { get; set; } 37 | 38 | object IStatus.Status { get => Status; set => Status = (TStatus) value; } 39 | } 40 | } -------------------------------------------------------------------------------- /samples/basic/deploy/role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: Role 3 | metadata: 4 | creationTimestamp: null 5 | name: basic-operator 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - pods 11 | - services 12 | - services/finalizers 13 | - endpoints 14 | - persistentvolumeclaims 15 | - events 16 | - configmaps 17 | - secrets 18 | verbs: 19 | - create 20 | - delete 21 | - get 22 | - list 23 | - patch 24 | - update 25 | - watch 26 | - apiGroups: 27 | - apps 28 | resources: 29 | - deployments 30 | - daemonsets 31 | - replicasets 32 | - statefulsets 33 | verbs: 34 | - create 35 | - delete 36 | - get 37 | - list 38 | - patch 39 | - update 40 | - watch 41 | - apiGroups: 42 | - monitoring.coreos.com 43 | resources: 44 | - servicemonitors 45 | verbs: 46 | - get 47 | - create 48 | - apiGroups: 49 | - apps 50 | resourceNames: 51 | - basic-operator 52 | resources: 53 | - deployments/finalizers 54 | verbs: 55 | - update 56 | - apiGroups: 57 | - "" 58 | resources: 59 | - pods 60 | verbs: 61 | - get 62 | - apiGroups: 63 | - apps 64 | resources: 65 | - replicasets 66 | - deployments 67 | verbs: 68 | - get 69 | - apiGroups: 70 | - csharp-operator.example.com 71 | resources: 72 | - '*' 73 | verbs: 74 | - create 75 | - delete 76 | - get 77 | - list 78 | - patch 79 | - update 80 | - watch -------------------------------------------------------------------------------- /samples/dynamic/deploy/role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: Role 3 | metadata: 4 | creationTimestamp: null 5 | name: dynamic-operator 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - pods 11 | - services 12 | - services/finalizers 13 | - endpoints 14 | - persistentvolumeclaims 15 | - events 16 | - configmaps 17 | - secrets 18 | verbs: 19 | - create 20 | - delete 21 | - get 22 | - list 23 | - patch 24 | - update 25 | - watch 26 | - apiGroups: 27 | - apps 28 | resources: 29 | - deployments 30 | - daemonsets 31 | - replicasets 32 | - statefulsets 33 | verbs: 34 | - create 35 | - delete 36 | - get 37 | - list 38 | - patch 39 | - update 40 | - watch 41 | - apiGroups: 42 | - monitoring.coreos.com 43 | resources: 44 | - servicemonitors 45 | verbs: 46 | - get 47 | - create 48 | - apiGroups: 49 | - apps 50 | resourceNames: 51 | - dynamic-operator 52 | resources: 53 | - deployments/finalizers 54 | verbs: 55 | - update 56 | - apiGroups: 57 | - "" 58 | resources: 59 | - pods 60 | verbs: 61 | - get 62 | - apiGroups: 63 | - apps 64 | resources: 65 | - replicasets 66 | - deployments 67 | verbs: 68 | - get 69 | - apiGroups: 70 | - csharp-operator.example.com 71 | resources: 72 | - '*' 73 | verbs: 74 | - create 75 | - delete 76 | - get 77 | - list 78 | - patch 79 | - update 80 | - watch -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "label": "build", 6 | "command": "dotnet", 7 | "type": "process", 8 | "args": [ 9 | "build", 10 | "${workspaceFolder}/samples/basic/k8s.Operators.Samples.Basic.csproj", 11 | "/property:GenerateFullPaths=true", 12 | "/consoleloggerparameters:NoSummary" 13 | ], 14 | "problemMatcher": "$msCompile" 15 | }, 16 | { 17 | "label": "publish", 18 | "command": "dotnet", 19 | "type": "process", 20 | "args": [ 21 | "publish", 22 | "${workspaceFolder}/samples/basic/k8s.Operators.Samples.Basic.csproj", 23 | "/property:GenerateFullPaths=true", 24 | "/consoleloggerparameters:NoSummary" 25 | ], 26 | "problemMatcher": "$msCompile" 27 | }, 28 | { 29 | "label": "watch", 30 | "command": "dotnet", 31 | "type": "process", 32 | "args": [ 33 | "watch", 34 | "run", 35 | "${workspaceFolder}/samples/basic/k8s.Operators.Samples.Basic.csproj", 36 | "/property:GenerateFullPaths=true", 37 | "/consoleloggerparameters:NoSummary" 38 | ], 39 | "problemMatcher": "$msCompile" 40 | } 41 | ] 42 | } -------------------------------------------------------------------------------- /src/k8s.Operators/k8s.Operators.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp3.1 5 | k8s.Operators 6 | 1.1.0-beta1 7 | true 8 | k8s.Operators.snk 9 | 10 | 11 | 12 | True 13 | k8s.Operators 14 | C# Operator SDK 15 | Build Kubernetes operators with C# and .NET Core 16 | Alberto Falossi 17 | kubernetes;operator;operators;c#;.net 18 | https://github.com/falox/csharp-operator-sdk 19 | https://github.com/falox/csharp-operator-sdk 20 | Apache-2.0 21 | true 22 | snupkg 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /tests/k8s.Operators.Tests/TestableOperator.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.Logging; 2 | using System; 3 | using System.Linq; 4 | 5 | namespace k8s.Operators.Tests 6 | { 7 | public class TestableOperator : Operator 8 | { 9 | public TestableOperator(OperatorConfiguration configuration, IKubernetes client, ILoggerFactory loggerFactory = null) : base(configuration, client, loggerFactory) 10 | { 11 | } 12 | 13 | /// 14 | /// Simulates an incoming event for a given controller 15 | /// 16 | public void SimulateEvent(IController controller, WatchEventType eventType, CustomResource resource) 17 | { 18 | _watchers.Single(x => x.Controller == controller).OnIncomingEvent(eventType, resource); 19 | } 20 | 21 | /// 22 | /// Protected method exposed as Public 23 | /// 24 | public void Exposed_OnWatchError(Exception exception) => OnWatcherError(exception); 25 | 26 | protected override void OnWatcherClose() 27 | { 28 | // HACK: Any watcher will fail and close during tests, since the external Watcher class is not mocked at the moment. 29 | // This override will ignore the close event and prevent the operator to be stopped prematurely 30 | } 31 | 32 | public int DisposeInvocationCount { get; private set; } 33 | 34 | protected override void DisposeInternal() 35 | { 36 | base.DisposeInternal(); 37 | 38 | DisposeInvocationCount++; 39 | } 40 | } 41 | } -------------------------------------------------------------------------------- /src/k8s.Operators/Logging/ConsoleTracingInterceptor.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.Diagnostics.CodeAnalysis; 5 | using System.Net.Http; 6 | using Microsoft.Rest; 7 | 8 | namespace k8s.Operators.Logging 9 | { 10 | /// 11 | /// A console tracer for the ServiceClientTracing service used by the Kubernetes C# client 12 | /// 13 | /// 14 | [ExcludeFromCodeCoverage] 15 | public class ConsoleTracingInterceptor : IServiceClientTracingInterceptor 16 | { 17 | public void Information(string message) 18 | { 19 | Console.WriteLine(message); 20 | } 21 | 22 | public void TraceError(string invocationId, Exception exception) 23 | { 24 | Console.WriteLine($"invocationId: {invocationId}, exception: {exception}"); 25 | } 26 | 27 | public void ReceiveResponse(string invocationId, HttpResponseMessage response) 28 | { 29 | Console.WriteLine($"invocationId: {invocationId}\r\nresponse: {(response == null ? string.Empty : response.AsFormattedString())}"); 30 | } 31 | 32 | public void SendRequest(string invocationId, HttpRequestMessage request) 33 | { 34 | Console.WriteLine($"invocationId: {invocationId}\r\nrequest: {(request == null ? string.Empty : request.AsFormattedString())}"); 35 | } 36 | 37 | public void Configuration(string source, string name, string value) 38 | { 39 | Console.WriteLine($"Configuration: source={source}, name={name}, value={value}"); 40 | } 41 | 42 | public void EnterMethod(string invocationId, object instance, string method, IDictionary parameters) 43 | { 44 | Console.WriteLine($"invocationId: {invocationId}\r\ninstance: {instance}\r\nmethod: {method}\r\nparameters: {parameters.AsFormattedString()}"); 45 | } 46 | 47 | public void ExitMethod(string invocationId, object returnValue) 48 | { 49 | Console.WriteLine($"invocationId: {invocationId}, return value: {(returnValue == null ? string.Empty : returnValue.ToString())}"); 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /src/k8s.Operators/IOperator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | 4 | namespace k8s.Operators 5 | { 6 | /// 7 | /// Represents a Kubernetes operator 8 | /// 9 | public interface IOperator : IDisposable 10 | { 11 | /// 12 | /// Adds a controller to handle the events of the custom resource R 13 | /// 14 | /// The controller for the custom resource 15 | /// The watched namespace. Set to null to watch all namespaces 16 | /// The label selector to filter the sets of events returned/> 17 | /// The type of the custom resource 18 | IOperator AddController(IController controller, string watchNamespace = "default", string labelSelector = null) where R : CustomResource; 19 | 20 | /// 21 | /// Adds a new instance of a controller of type C to handle the events of the custom resource 22 | /// 23 | /// The type of the controller. C must implement IController and expose a constructor that accepts (OperatorConfiguration, IKubernetes, ILoggerFactory) 24 | /// The instance of the controller 25 | IController AddControllerOfType() where C : IController; 26 | 27 | /// 28 | /// Starts watching and handling events 29 | /// 30 | Task StartAsync(); 31 | 32 | /// 33 | /// Stops the operator and release the resources. Once stopped, an operator cannot be restarted. Stop() is an alias for Dispose() 34 | /// 35 | void Stop(); 36 | 37 | /// 38 | /// Returns true if StartAsync has been called and the operator is running 39 | /// 40 | bool IsRunning { get; } 41 | 42 | /// 43 | /// Returns true if Stop/Dispose has been called and not completed 44 | /// 45 | /// 46 | bool IsDisposing { get; } 47 | } 48 | } -------------------------------------------------------------------------------- /src/k8s.Operators/EventWatcher.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading; 3 | using Microsoft.Extensions.Logging; 4 | 5 | namespace k8s.Operators 6 | { 7 | /// 8 | /// Implements the watch callback method for a given 9 | /// 10 | public class EventWatcher 11 | { 12 | private readonly ILogger _logger; 13 | private readonly CancellationToken _cancellationToken; 14 | 15 | public Type ResourceType { get; private set; } 16 | public CustomResourceDefinitionAttribute CRD { get; private set; } 17 | public string Namespace { get; private set; } 18 | public string LabelSelector { get; private set; } 19 | public IController Controller { get; private set; } 20 | 21 | public EventWatcher(Type resourceType, string @namespace, string labelSelector, IController controller, ILogger logger, CancellationToken cancellationToken) 22 | { 23 | this.ResourceType = resourceType; 24 | this.Namespace = @namespace; 25 | this.LabelSelector = labelSelector; 26 | this.Controller = controller; 27 | this._logger = logger; 28 | this._cancellationToken = cancellationToken; 29 | 30 | // Retrieve the CRD associated to the CR 31 | var crd = (CustomResourceDefinitionAttribute)Attribute.GetCustomAttribute(resourceType, typeof(CustomResourceDefinitionAttribute)); 32 | this.CRD = crd; 33 | } 34 | 35 | /// 36 | /// Dispatches an incoming event to the controller 37 | /// 38 | public void OnIncomingEvent(WatchEventType eventType, CustomResource resource) 39 | { 40 | var resourceEvent = new CustomResourceEvent(eventType, resource); 41 | 42 | _logger.LogDebug($"Received event {resourceEvent}"); 43 | 44 | Controller.ProcessEventAsync(resourceEvent, _cancellationToken) 45 | .ContinueWith(t => 46 | { 47 | if (t.IsFaulted) 48 | { 49 | var exception = t.Exception.Flatten().InnerException; 50 | _logger.LogError(exception, $"Error processing {resourceEvent}"); 51 | } 52 | }); 53 | } 54 | } 55 | } -------------------------------------------------------------------------------- /src/k8s.Operators/ResourceChangeTracker.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using k8s.Operators.Logging; 3 | using Microsoft.Extensions.Logging; 4 | 5 | namespace k8s.Operators 6 | { 7 | /// 8 | /// Keeps track of the resource changes to avoid unnecessary updates 9 | /// 10 | public class ResourceChangeTracker 11 | { 12 | private readonly ILogger _logger; 13 | 14 | // Last generation number successfully processed, for each resource 15 | private readonly Dictionary _lastResourceGenerationProcessed; 16 | private readonly bool _discardDuplicates; 17 | 18 | public ResourceChangeTracker(OperatorConfiguration configuration, ILoggerFactory loggerFactory) 19 | { 20 | this._logger = loggerFactory?.CreateLogger() ?? SilentLogger.Instance; 21 | this._lastResourceGenerationProcessed = new Dictionary(); 22 | this._discardDuplicates = configuration.DiscardDuplicateSpecGenerations; 23 | } 24 | 25 | /// 26 | /// Returns true if the same resource/generation has already been handled 27 | /// 28 | public bool IsResourceGenerationAlreadyHandled(CustomResource resource) 29 | { 30 | if (_discardDuplicates) 31 | { 32 | bool processedInPast = _lastResourceGenerationProcessed.TryGetValue(resource.Metadata.Uid, out long resourceGeneration); 33 | 34 | return processedInPast 35 | && resource.Metadata.Generation != null 36 | && resourceGeneration >= resource.Metadata.Generation.Value; 37 | } 38 | else 39 | { 40 | return false; 41 | } 42 | } 43 | 44 | /// 45 | /// Mark a resource generation as successfully handled 46 | /// 47 | public void TrackResourceGenerationAsHandled(CustomResource resource) 48 | { 49 | if (resource.Metadata.Generation != null) 50 | { 51 | _lastResourceGenerationProcessed[resource.Metadata.Uid] = resource.Metadata.Generation.Value; 52 | } 53 | } 54 | 55 | /// 56 | /// Mark a resource generation as successfully deleted 57 | /// 58 | public void TrackResourceGenerationAsDeleted(CustomResource resource) 59 | { 60 | _lastResourceGenerationProcessed.Remove(resource.Metadata.Uid); 61 | } 62 | } 63 | } -------------------------------------------------------------------------------- /samples/basic/MyResourceController.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading; 3 | using System.Threading.Tasks; 4 | using k8s.Models; 5 | using Microsoft.Extensions.Logging; 6 | 7 | namespace k8s.Operators.Samples.Basic 8 | { 9 | public class MyResourceController : Controller 10 | { 11 | public MyResourceController(OperatorConfiguration configuration, IKubernetes client, ILoggerFactory loggerFactory = null) : base(configuration, client, loggerFactory) 12 | { 13 | } 14 | 15 | protected override async Task AddOrModifyAsync(MyResource resource, CancellationToken cancellationToken) 16 | { 17 | _logger.LogInformation($"Begin AddOrModify {resource}"); 18 | 19 | try 20 | { 21 | // Simulate event handling 22 | await Task.Delay(5000, cancellationToken); 23 | 24 | // Update the resource 25 | resource.Metadata.EnsureAnnotations()["custom-key"] = DateTime.UtcNow.ToString("s"); 26 | await UpdateResourceAsync(resource, cancellationToken); 27 | 28 | // Update the status 29 | if (resource.Status == null) 30 | { 31 | resource.Status = new MyResource.MyResourceStatus(); 32 | } 33 | if (resource.Status.Actual != resource.Spec.Desired) 34 | { 35 | resource.Status.Actual = resource.Spec.Desired; 36 | await UpdateStatusAsync(resource, cancellationToken); 37 | } 38 | } 39 | catch (OperationCanceledException) 40 | { 41 | _logger.LogInformation($"Interrupted! Trying to shutdown gracefully..."); 42 | 43 | // Simulate a blocking operation 44 | Task.Delay(3000).Wait(); 45 | } 46 | 47 | _logger.LogInformation($"End AddOrModify {resource}"); 48 | } 49 | 50 | protected override async Task DeleteAsync(MyResource resource, CancellationToken cancellationToken) 51 | { 52 | _logger.LogInformation($"Begin Delete {resource}"); 53 | 54 | try 55 | { 56 | // Simulate event handling 57 | await Task.Delay(5000, cancellationToken); 58 | } 59 | catch (OperationCanceledException) 60 | { 61 | _logger.LogInformation($"Interrupted! Trying to shutdown gracefully..."); 62 | 63 | // Simulate a blocking operation 64 | Task.Delay(3000).Wait(); 65 | } 66 | 67 | _logger.LogInformation($"End Delete {resource}"); 68 | } 69 | } 70 | } -------------------------------------------------------------------------------- /.github/workflows/dotnet-core.yml: -------------------------------------------------------------------------------- 1 | name: .NET Core 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | release: 9 | types: [ published ] 10 | 11 | env: 12 | DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true 13 | DOTNET_CLI_TELEMETRY_OPTOUT: true 14 | # GITHUB_FEED: https://nuget.pkg.github.com/falox/ 15 | # GITHUB_USER: falox 16 | # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 17 | NUGET_FEED: https://api.nuget.org/v3/index.json 18 | NUGET_KEY: ${{ secrets.NUGET_KEY }} 19 | 20 | jobs: 21 | build: 22 | 23 | runs-on: ubuntu-latest 24 | 25 | steps: 26 | - uses: actions/checkout@v2 27 | - name: Setup .NET Core 28 | uses: actions/setup-dotnet@v1 29 | with: 30 | dotnet-version: 3.1.301 31 | - name: Install dependencies 32 | run: dotnet restore 33 | - name: Build 34 | run: dotnet build --configuration Release --no-restore 35 | - name: Test 36 | run: dotnet test --no-restore --verbosity normal 37 | - name: Generate coverage report 38 | run: | 39 | cd ./tests/k8s.Operators.Tests/ 40 | dotnet test /p:CollectCoverage=true /p:CoverletOutput=TestResults/ /p:CoverletOutputFormat=lcov 41 | - name: Publish coverage report to coveralls.io 42 | uses: coverallsapp/github-action@master 43 | with: 44 | github-token: ${{ secrets.GITHUB_TOKEN }} 45 | path-to-lcov: ./tests/k8s.Operators.Tests/TestResults/coverage.info 46 | - name: Pack 47 | run: dotnet pack --configuration Release --no-restore --verbosity normal 48 | - name: Upload Artifact 49 | uses: actions/upload-artifact@v2 50 | with: 51 | name: nupkg 52 | path: ./src/k8s.Operators/bin/Release/*.nupkg 53 | 54 | deploy: 55 | needs: build 56 | if: github.event_name == 'release' 57 | 58 | runs-on: ubuntu-latest 59 | 60 | steps: 61 | - uses: actions/checkout@v2 62 | - name: Setup .NET Core 63 | uses: actions/setup-dotnet@v1 64 | with: 65 | dotnet-version: 3.1.301 66 | - name: Create Release NuGet package 67 | run: | 68 | arrTag=(${GITHUB_REF//\// }) 69 | VERSION="${arrTag[2]}" 70 | echo Version: $VERSION 71 | VERSION="${VERSION//v}" 72 | echo Clean Version: $VERSION 73 | dotnet pack -v normal -c Release -p:PackageVersion=$VERSION -o nupkg src/k8s.Operators/k8s.Operators.*proj 74 | #- name: Push to GitHub Feed 75 | # run: | 76 | # for f in ./nupkg/*.nupkg 77 | # do 78 | # curl -vX PUT -u "$GITHUB_USER:$GITHUB_TOKEN" -F package=@$f $GITHUB_FEED 79 | # done 80 | - name: Push to NuGet Feed 81 | run: dotnet nuget push ./nupkg/*.nupkg --source $NUGET_FEED --skip-duplicate --api-key $NUGET_KEY 82 | -------------------------------------------------------------------------------- /samples/dynamic/MyDynamicResourceController.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading; 3 | using System.Threading.Tasks; 4 | using System.Dynamic; 5 | using k8s.Models; 6 | using Microsoft.Extensions.Logging; 7 | 8 | namespace k8s.Operators.Samples.Dynamic 9 | { 10 | public class MyDynamicResourceController : Controller 11 | { 12 | public MyDynamicResourceController(OperatorConfiguration configuration, IKubernetes client, ILoggerFactory loggerFactory = null) : base(configuration, client, loggerFactory) 13 | { 14 | } 15 | 16 | protected override async Task AddOrModifyAsync(MyDynamicResource resource, CancellationToken cancellationToken) 17 | { 18 | _logger.LogInformation($"Begin AddOrModify {resource}"); 19 | 20 | try 21 | { 22 | // Simulate event handling 23 | await Task.Delay(5000, cancellationToken); 24 | 25 | // Update the resource 26 | resource.Metadata.EnsureAnnotations()["custom-key"] = DateTime.UtcNow.ToString("s"); 27 | await UpdateResourceAsync(resource, cancellationToken); 28 | 29 | // Update the status 30 | if (resource.Status?.actualProperty != resource.Spec.desiredProperty) 31 | { 32 | if (resource.Status == null) 33 | { 34 | resource.Status = new ExpandoObject(); 35 | } 36 | resource.Status.actualProperty = resource.Spec.desiredProperty; 37 | await UpdateStatusAsync(resource, cancellationToken); 38 | } 39 | } 40 | catch (OperationCanceledException) 41 | { 42 | _logger.LogInformation($"Interrupted! Trying to shutdown gracefully..."); 43 | 44 | // Simulate a blocking operation 45 | Task.Delay(3000).Wait(); 46 | } 47 | 48 | _logger.LogInformation($"End AddOrModify {resource}"); 49 | } 50 | 51 | protected override async Task DeleteAsync(MyDynamicResource resource, CancellationToken cancellationToken) 52 | { 53 | _logger.LogInformation($"Begin Delete {resource}"); 54 | 55 | try 56 | { 57 | // Simulate event handling 58 | await Task.Delay(5000, cancellationToken); 59 | } 60 | catch (OperationCanceledException) 61 | { 62 | _logger.LogInformation($"Interrupted! Trying to shutdown gracefully..."); 63 | 64 | // Simulate a blocking operation 65 | Task.Delay(3000).Wait(); 66 | } 67 | 68 | _logger.LogInformation($"End Delete {resource}"); 69 | } 70 | } 71 | } -------------------------------------------------------------------------------- /samples/basic/README.md: -------------------------------------------------------------------------------- 1 | # C# Basic Kubernetes Operator 2 | 3 | The *Basic Operator* handles the `MyResource` custom resource. The operator simulates the interaction with an external service and can be used as a template for real-world operators. 4 | 5 | Once the operator detects an added/modified event, it waits for 5 seconds and: 6 | 7 | - Adds a custom annotation `custom-key` in the object's `metadata.annotation` 8 | - Updates the `status.actualProperty` to match the `spec.desiredProperty` 9 | 10 | See the implementation of `MyResourceController.cs` for more details. 11 | 12 | ## Prerequisites 13 | 14 | - [.NET Core 3.1 SDK](https://dotnet.microsoft.com/download/dotnet-core/3.1) 15 | - [Kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) 16 | - [Kubernetes](https://kubernetes.io/docs/setup/) 17 | 18 | ## Running locally 19 | 20 | 1. Add the custom resource definition to your Kubernetes cluster: 21 | 22 | ```bash 23 | $ cd csharp-operator-sdk/samples/basic 24 | $ kubectl apply -f ./deploy/crds/crd.yaml 25 | ``` 26 | 27 | 2. Compile and run the operator: 28 | ```bash 29 | $ dotnet build 30 | $ dotnet run 31 | ``` 32 | 33 | The operator will connect to the Kubernetes cluster and will start watching for events. You'll see something similar to: 34 | 35 | ``` 36 | <6>k8s.Operators.Operator[0] Start operator 37 | ``` 38 | 39 | 3. In another terminal, create a new `MyResource` object: 40 | 41 | ```bash 42 | $ kubectl apply -f ./deploy/crds/cr.yaml 43 | ``` 44 | 45 | The operator will detect the new object and after 5 seconds will update the *status* to match the desired *spec*: 46 | 47 | ``` 48 | <6>k8s.Operators.Controller[0] Begin AddOrModify default/mr1 (gen: 1), Spec: 1 Status: 49 | <6>k8s.Operators.Controller[0] End AddOrModify default/mr1 (gen: 1), Spec: 1 Status: 1 50 | ``` 51 | 52 | 4. Edit the resource and change the `spec.desiredProperty` to `2`: 53 | 54 | ```bash 55 | $ kubectl edit myresources mr1 56 | ``` 57 | 58 | ```yaml 59 | apiVersion: csharp-operator.example.com/v1 60 | kind: MyResource 61 | : 62 | spec: 63 | desiredProperty: 2 64 | : 65 | ``` 66 | 67 | The operator will detect the change and will align again the *status*: 68 | 69 | ``` 70 | <6>k8s.Operators.Controller[0] Begin AddOrModify default/mr1 (gen: 2), Spec: 2 Status: 1 71 | <6>k8s.Operators.Controller[0] End AddOrModify default/mr1 (gen: 2), Spec: 2 Status: 2 72 | ``` 73 | 74 | 5. Delete the resource: 75 | 76 | ```bash 77 | $ kubectl delete myresources mr1 78 | ``` 79 | 80 | The operator will simulate the deletion of the resource: 81 | 82 | ``` 83 | <6>k8s.Operators.Controller[0] Begin Delete default/mr1 (gen: 3), Spec: 2 Status: 2 84 | <6>k8s.Operators.Controller[0] End Delete default/mr1 (gen: 3), Spec: 2 Status: 2 85 | ``` 86 | 87 | 6. Shutdown the operator with CTRL+C or by sending a SIGTERM signal with: 88 | 89 | ```bash 90 | kill $(ps aux | grep '[k]8s.Operators.Samples.Basic' | awk '{print $2}') 91 | ``` 92 | 93 | The operator will gracefully shutdown: 94 | 95 | ``` 96 | <6>k8s.Operators.Operator[0] Stop operator 97 | <6>k8s.Operators.Operator[0] Disposing operator 98 | ``` 99 | 100 | ## Deploy the operator in Kubernetes 101 | 102 | 1. If you are running [Minikube](https://kubernetes.io/docs/setup/learning-environment/minikube/), point your shell to minikube's docker-daemon: 103 | 104 | ```bash 105 | $ eval $(minikube -p minikube docker-env) 106 | ``` 107 | 108 | 2. Create the docker image: 109 | 110 | ```bash 111 | $ cd csharp-operator-sdk 112 | $ docker build -t csharp-basic-operator -f samples/basic/Dockerfile . 113 | ``` 114 | 115 | 3. Add the custom resource definition to your Kubernetes cluster: 116 | 117 | ```bash 118 | cd samples/basic 119 | kubectl apply -f ./deploy/crds/crd.yaml 120 | ``` 121 | 122 | 4. Deploy the operator: 123 | 124 | ```bash 125 | kubectl apply -f ./deploy/service_account.yaml 126 | kubectl apply -f ./deploy/role.yaml 127 | kubectl apply -f ./deploy/role_binding.yaml 128 | kubectl apply -f ./deploy/operator.yaml 129 | ``` -------------------------------------------------------------------------------- /src/k8s.Operators/EventManager.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using k8s.Operators.Logging; 4 | using Microsoft.Extensions.Logging; 5 | 6 | namespace k8s.Operators 7 | { 8 | /// 9 | /// Manages the event queues for the watched resources 10 | /// 11 | public class EventManager 12 | { 13 | private ILogger _logger; 14 | 15 | // Next event to handle, for each resource. 16 | // A real queue is not used since intermediate events are discarded and only the queue head is stored. 17 | private Dictionary _queuesByResource; 18 | 19 | // Events that are currently being handled 20 | private Dictionary _handling; 21 | 22 | public EventManager(ILoggerFactory loggerFactory) 23 | { 24 | this._logger = loggerFactory?.CreateLogger() ?? SilentLogger.Instance; 25 | this._queuesByResource = new Dictionary(); 26 | this._handling = new Dictionary(); 27 | } 28 | 29 | /// 30 | /// Enqueue the event 31 | /// 32 | public void Enqueue(CustomResourceEvent resourceEvent) 33 | { 34 | lock (this) 35 | { 36 | _logger.LogTrace($"Enqueue {resourceEvent}"); 37 | // Insert or update the next event for the resource 38 | _queuesByResource[resourceEvent.ResourceUid] = resourceEvent; 39 | } 40 | } 41 | 42 | /// 43 | /// Returns the next event to process, without dequeuing it 44 | /// 45 | public CustomResourceEvent Peek(string resourceUid) 46 | { 47 | lock (this) 48 | { 49 | if (_queuesByResource.TryGetValue(resourceUid, out CustomResourceEvent result)) 50 | { 51 | _logger.LogTrace($"Peek {result}"); 52 | } 53 | return result; 54 | } 55 | } 56 | 57 | /// 58 | /// Pops and returns the next event to process, if any 59 | /// 60 | public CustomResourceEvent Dequeue(string resourceUid) 61 | { 62 | lock (this) 63 | { 64 | if (IsHandling(resourceUid, out var handlingEvent)) 65 | { 66 | _logger.LogDebug($"Postponed Dequeue, handling {handlingEvent}"); 67 | return null; 68 | } 69 | else 70 | { 71 | if (_queuesByResource.TryGetValue(resourceUid, out CustomResourceEvent result)) 72 | { 73 | _queuesByResource.Remove(resourceUid); 74 | _logger.LogTrace($"Dequeue {result}"); 75 | } 76 | return result; 77 | } 78 | } 79 | } 80 | 81 | /// 82 | /// Track the begin of an event handling 83 | /// 84 | public void BeginHandleEvent(CustomResourceEvent resourceEvent) 85 | { 86 | lock (this) 87 | { 88 | _logger.LogTrace($"BeginHandleEvent {resourceEvent}"); 89 | _handling[resourceEvent.ResourceUid] = resourceEvent; 90 | } 91 | } 92 | 93 | /// 94 | /// Track the end of an event handling 95 | /// 96 | public void EndHandleEvent(CustomResourceEvent resourceEvent) 97 | { 98 | lock (this) 99 | { 100 | _logger.LogTrace($"EndHandleEvent {resourceEvent}"); 101 | _handling.Remove(resourceEvent.ResourceUid); 102 | } 103 | } 104 | 105 | /// 106 | /// Returns true if there is an event being handled 107 | /// 108 | private bool IsHandling(string resourceUid, out CustomResourceEvent handlingEvent) 109 | { 110 | return _handling.TryGetValue(resourceUid, out handlingEvent); 111 | } 112 | } 113 | } -------------------------------------------------------------------------------- /tests/k8s.Operators.Tests/TestableController.cs: -------------------------------------------------------------------------------- 1 | using k8s.Operators; 2 | using Microsoft.Extensions.Logging; 3 | using System.Threading.Tasks; 4 | using System.Collections.Generic; 5 | using System.Threading; 6 | using System; 7 | 8 | namespace k8s.Operators.Tests 9 | { 10 | public class TestableController : Controller 11 | { 12 | public TestableController() : base(OperatorConfiguration.Default, null, null) 13 | { 14 | } 15 | 16 | public TestableController(IKubernetes client, ILoggerFactory loggerFactory = null) : base(OperatorConfiguration.Default, client, loggerFactory) 17 | { 18 | } 19 | 20 | public TestableController(OperatorConfiguration configuration, IKubernetes client, ILoggerFactory loggerFactory = null) : base(configuration, client, loggerFactory) 21 | { 22 | } 23 | 24 | public List Invocations_AddOrModify = new List(); 25 | public List Invocations_Delete = new List(); 26 | public List<(TestableCustomResource resource, bool deleteEvent)> Invocations = new List<(TestableCustomResource resource, bool deleteEvent)>(); 27 | public List<(TestableCustomResource resource, bool deleteEvent)> CompletedEvents = new List<(TestableCustomResource resource, bool deleteEvent)>(); 28 | 29 | private Queue> _signals = new Queue>(); 30 | 31 | protected override async Task AddOrModifyAsync(TestableCustomResource resource, CancellationToken cancellationToken) 32 | { 33 | Invocations_AddOrModify.Add(resource); 34 | Invocations.Add((resource, false)); 35 | 36 | if (_signals.TryDequeue(out var signal)) 37 | { 38 | // Wait for UnblockEvent() 39 | await signal?.Task; 40 | } 41 | 42 | if (_exceptionsToThrow > 0) 43 | { 44 | _exceptionsToThrow--; 45 | throw new Exception(); 46 | } 47 | 48 | CompletedEvents.Add((resource, deleteEvent: false)); 49 | } 50 | 51 | protected override async Task DeleteAsync(TestableCustomResource resource, CancellationToken cancellationToken) 52 | { 53 | Invocations_Delete.Add(resource); 54 | Invocations.Add((resource, true)); 55 | 56 | if (_signals.TryDequeue(out var signal)) 57 | { 58 | // Wait for UnblockEvent() 59 | await signal?.Task; 60 | } 61 | 62 | if (_exceptionsToThrow > 0) 63 | { 64 | _exceptionsToThrow--; 65 | throw new Exception(); 66 | } 67 | 68 | CompletedEvents.Add((resource, deleteEvent: true)); 69 | } 70 | 71 | /// 72 | /// Protected method exposed as Public 73 | /// 74 | public Task Exposed_UpdateResourceAsync(TestableCustomResource resource, CancellationToken cancellationToken) => UpdateResourceAsync(resource, cancellationToken); 75 | 76 | /// 77 | /// Protected method exposed as Public 78 | /// 79 | public Task Exposed_UpdateStatusAsync(TestableCustomResource resource, CancellationToken cancellationToken) => UpdateStatusAsync(resource, cancellationToken); 80 | 81 | /// 82 | /// Throws an exception in the next calls to AddOrModifyAsync or DeleteAsync 83 | /// 84 | /// The number of the events to make fail 85 | public void ThrowExceptionOnNextEvents(int count) 86 | { 87 | _exceptionsToThrow = count; 88 | } 89 | 90 | private int _exceptionsToThrow = 0; 91 | 92 | /// 93 | /// Block the next call to AddOrModifyAsync or DeleteAsync 94 | /// 95 | public TaskCompletionSource BlockNextEvent() 96 | { 97 | var signal = new TaskCompletionSource(); 98 | _signals.Enqueue(signal); 99 | return signal; 100 | } 101 | 102 | /// 103 | /// Unblock the next call to AddOrModifyAsync or DeleteAsync 104 | /// 105 | public void UnblockEvent(TaskCompletionSource signal) 106 | { 107 | signal.SetResult(true); 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![.NET Core](https://github.com/falox/csharp-operator-sdk/workflows/.NET%20Core/badge.svg?branch=master)](https://github.com/falox/csharp-operator-sdk/actions?query=workflow%3A%22.NET+Core%22) 2 | [![Nuget (with prereleases)](https://img.shields.io/nuget/vpre/k8s.Operators)](https://www.nuget.org/packages/k8s.Operators) 3 | [![Coverage Status](https://coveralls.io/repos/github/falox/csharp-operator-sdk/badge.svg?branch=master)](https://coveralls.io/github/falox/csharp-operator-sdk?branch=master) 4 | 5 | # C# Operator SDK 6 | 7 | The C# Operator SDK is a framework to build [Kubernetes operators](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) with C# and .NET Core. 8 | 9 | ## Features 10 | 11 | - Easy custom resource and controller definition as C# classes 12 | - Custom resource event watchers at namespace and cluster scope 13 | - [Configurable retry-on-failure](https://github.com/falox/csharp-operator-sdk/blob/f989ab3ad5fdf322f681c863052338c982680bc5/samples/basic/deploy/operator.yaml#L27) policy 14 | - Smart concurrent event queues (inspired by [Container Solution](https://blog.container-solutions.com/a-deep-dive-into-the-java-operator-sdk)'s article) 15 | - Kubernetes [graceful termination policy](https://github.com/falox/csharp-operator-sdk/blob/f989ab3ad5fdf322f681c863052338c982680bc5/samples/basic/Program.cs#L89) support 16 | 17 | ## Usage 18 | 19 | Setup a new [.NET Core 3.1](https://dotnet.microsoft.com/download/dotnet-core/3.1) project and add the [C# Operator SDK package](https://www.nuget.org/packages/k8s.Operators): 20 | 21 | ```bash 22 | dotnet new console 23 | dotnet add package k8s.Operators 24 | ``` 25 | 26 | Assuming that you have already added a custom resource definition for `MyResource` in your Kubernetes cluster, define a class deriving from `CustomResource` for the custom resource schema: 27 | 28 | ```csharp 29 | // Set the CRD attributes 30 | [CustomResourceDefinition("example.com", "v1", "myresources")] 31 | public class MyResource : CustomResource 32 | { 33 | // Define spec 34 | public class MyResourceSpec 35 | { 36 | public int property1 { get; set; } 37 | // : 38 | } 39 | 40 | // Define status 41 | public class MyResourceStatus 42 | { 43 | public int property2 { get; set; } 44 | // : 45 | } 46 | } 47 | ``` 48 | 49 | Define a class deriving from `Controller` for the controller logic: 50 | 51 | ```csharp 52 | public class MyResourceController : Controller 53 | { 54 | public MyResourceController(OperatorConfiguration configuration, IKubernetes client, ILoggerFactory loggerFactory = null) 55 | : base(configuration, client, loggerFactory) 56 | { 57 | } 58 | 59 | protected override async Task AddOrModifyAsync(MyResource resource, CancellationToken cancellationToken) 60 | { 61 | Console.WriteLine($"Add/Modify {resource}"); 62 | // : 63 | // Handle Add/Modify event 64 | } 65 | 66 | protected override async Task DeleteAsync(MyResource resource, CancellationToken cancellationToken) 67 | { 68 | Console.WriteLine($"Delete {resource}"); 69 | // : 70 | // Handle Delete event 71 | } 72 | } 73 | ``` 74 | 75 | Setup the operator in `Main()`: 76 | 77 | ```csharp 78 | static async Task Main(string[] args) 79 | { 80 | // Create the Kubernetes client 81 | using var client = new Kubernetes(KubernetesClientConfiguration.BuildConfigFromConfigFile()); 82 | 83 | // Setup the operator 84 | var @operator = new Operator(OperatorConfiguration.Default, client); 85 | @operator.AddControllerOfType(); 86 | 87 | // Start the operator 88 | return await @operator.StartAsync(); 89 | } 90 | ``` 91 | 92 | > Curiosity: Since `operator` is a reserved keyword in C#, it has been escaped with `@operator`. 93 | 94 | Start the operator with: 95 | 96 | ```bash 97 | dotnet run 98 | ``` 99 | 100 | In the `/samples/basic` directory you find a [sample operator](./samples/basic/README.md) that simulates the interaction with an external service and can be used as a template for real-world operators. 101 | 102 | [Follow the instructions](./samples/basic/README.md) to run it locally and deploy it to Kubernetes. 103 | 104 | ## Compiling the source code 105 | 106 | ```bash 107 | git clone https://github.com/falox/csharp-operator-sdk.git 108 | cd csharp-operator-sdk 109 | dotnet restore 110 | dotnet build 111 | ``` 112 | 113 | Running the tests: 114 | 115 | ```bash 116 | dotnet test 117 | ``` 118 | 119 | ## References 120 | 121 | - Jason Dobies and Joshua Wood, [Kubernetes Operators](https://www.oreilly.com/library/view/kubernetes-operators/9781492048039/), O'Reilly, 2020 122 | - Radu Matei, [Writing controllers for Kubernetes CRDs with C#](https://radu-matei.com/blog/kubernetes-controller-csharp/), 2019 123 | -------------------------------------------------------------------------------- /tests/k8s.Operators.Tests/BaseTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Threading; 4 | using System.Threading.Tasks; 5 | using k8s.Models; 6 | using Microsoft.Extensions.Logging; 7 | using Microsoft.Extensions.Logging.Console; 8 | using Microsoft.Rest; 9 | using Moq; 10 | using Xunit; 11 | 12 | namespace k8s.Operators.Tests 13 | { 14 | public class BaseTests 15 | { 16 | protected CancellationToken DUMMY_TOKEN = default(CancellationToken); 17 | 18 | protected Mock _clientMock; 19 | protected IKubernetes _client => _clientMock.Object; 20 | protected ILoggerFactory _loggerFactory; 21 | 22 | public BaseTests() 23 | { 24 | // _loggerFactory = LoggerFactory.Create(builder => builder 25 | // .AddConsole(options => options.Format=ConsoleLoggerFormat.Systemd) 26 | // .SetMinimumLevel(LogLevel.Debug) 27 | // ); 28 | 29 | // Setup the client mock 30 | _clientMock = new Mock(); 31 | 32 | _clientMock.Setup(x => x.ListNamespacedCustomObjectWithHttpMessagesAsync( 33 | It.IsAny(), 34 | It.IsAny(), 35 | It.IsAny(), 36 | It.IsAny(), 37 | It.IsAny(), 38 | It.IsAny(), 39 | It.IsAny(), 40 | It.IsAny(), 41 | It.IsAny(), 42 | It.IsAny(), 43 | It.IsAny(), 44 | It.IsAny(), 45 | It.IsAny>>(), 46 | It.IsAny() 47 | ) 48 | ).Returns(Task.FromResult(new HttpOperationResponse())); 49 | 50 | _clientMock.Setup(x => x.ReplaceNamespacedCustomObjectWithHttpMessagesAsync( 51 | It.IsAny(), 52 | It.IsAny(), 53 | It.IsAny(), 54 | It.IsAny(), 55 | It.IsAny(), 56 | It.IsAny(), 57 | It.IsAny>>(), 58 | It.IsAny() 59 | ) 60 | ).Returns(Task.FromResult(new HttpOperationResponse())); 61 | 62 | _clientMock.Setup(x => x.PatchNamespacedCustomObjectStatusWithHttpMessagesAsync( 63 | It.IsAny(), 64 | It.IsAny(), 65 | It.IsAny(), 66 | It.IsAny(), 67 | It.IsAny(), 68 | It.IsAny(), 69 | It.IsAny>>(), 70 | It.IsAny() 71 | ) 72 | ).Returns(Task.FromResult(new HttpOperationResponse())); 73 | } 74 | 75 | protected TestableCustomResource CreateCustomResource(string uid = "1", string ns = null, long generation = 1L, bool withFinalizer = true, DateTime? deletionTimeStamp = null) 76 | { 77 | var resource = new TestableCustomResource(); 78 | 79 | if (withFinalizer) 80 | { 81 | resource.Metadata.EnsureFinalizers().Add(CustomResourceDefinitionAttribute.DEFAULT_FINALIZER); 82 | } 83 | 84 | if (ns != null) 85 | { 86 | resource.Metadata.NamespaceProperty = ns; 87 | } 88 | resource.Metadata.Uid = uid; 89 | resource.Metadata.DeletionTimestamp = deletionTimeStamp; 90 | resource.Metadata.Generation = generation; 91 | 92 | resource.Spec = new TestableCustomResource.TestableSpec(); 93 | resource.Status = new TestableCustomResource.TestableStatus(); 94 | 95 | return resource; 96 | } 97 | 98 | protected void VerifyCompletedEvents(TestableController controller, params (TestableCustomResource resource, bool deleteEvent)[] inputs) => Assert.Equal(inputs, controller.CompletedEvents); 99 | protected void VerifyCalledEvents(TestableController controller, params (TestableCustomResource resource, bool deleted)[] inputs) => Assert.Equal(inputs, controller.Invocations); 100 | protected void VerifyAddOrModifyIsCalledWith(TestableController controller,params TestableCustomResource[] inputs) => Assert.Equal(inputs, controller.Invocations_AddOrModify); 101 | protected void VerifyDeleteIsCalledWith(TestableController controller,params TestableCustomResource[] inputs) => Assert.Equal(inputs, controller.Invocations_Delete); 102 | protected void VerifyAddOrModifyIsNotCalled(TestableController controller) => Assert.Equal(new TestableCustomResource[] { }, controller.Invocations_AddOrModify); 103 | protected void VerifyDeleteIsNotCalled(TestableController controller) => Assert.Equal(new TestableCustomResource[] { }, controller.Invocations_Delete); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /samples/basic/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Threading.Tasks; 4 | using Microsoft.Extensions.Logging; 5 | using Microsoft.Extensions.Logging.Console; 6 | using Microsoft.Rest; 7 | using k8s.Operators.Logging; 8 | using Newtonsoft.Json; 9 | 10 | namespace k8s.Operators.Samples.Basic 11 | { 12 | class Program 13 | { 14 | static async Task Main(string[] args) 15 | { 16 | IOperator basicOperator = null; 17 | 18 | // Setup logging 19 | using var loggerFactory = SetupLogging(args); 20 | var logger = loggerFactory.CreateLogger(); 21 | 22 | try 23 | { 24 | logger.LogDebug($"Environment variables: {JsonConvert.SerializeObject(Environment.GetEnvironmentVariables())}"); 25 | 26 | // Setup termination handlers 27 | SetupSignalHandlers(); 28 | 29 | // Setup the Kubernetes client 30 | using var client = SetupClient(args); 31 | 32 | // Setup the operator 33 | var configuration = GetOperatorConfiguration(); 34 | basicOperator = new Operator(configuration, client, loggerFactory); 35 | basicOperator.AddControllerOfType(); 36 | 37 | // Start the operator 38 | return await basicOperator.StartAsync(); 39 | } 40 | catch (Exception exception) 41 | { 42 | logger.LogError(exception, "Operator error"); 43 | return 1; 44 | } 45 | 46 | IKubernetes SetupClient(string[] args) 47 | { 48 | // Load the Kubernetes configuration 49 | KubernetesClientConfiguration config = null; 50 | 51 | if (KubernetesClientConfiguration.IsInCluster()) 52 | { 53 | logger.LogDebug("Loading cluster configuration"); 54 | config = KubernetesClientConfiguration.InClusterConfig(); 55 | } 56 | else 57 | { 58 | logger.LogDebug("Loading local configuration"); 59 | config = KubernetesClientConfiguration.BuildConfigFromConfigFile(); 60 | } 61 | 62 | if (logger.IsEnabled(LogLevel.Debug)) 63 | { 64 | logger.LogDebug($"Client configuration: {JsonConvert.SerializeObject(config)}"); 65 | } 66 | 67 | return new Kubernetes(config); 68 | } 69 | 70 | ILoggerFactory SetupLogging(string[] args) 71 | { 72 | if (!System.Enum.TryParse(Environment.GetEnvironmentVariable("LOG_LEVEL"), true, out LogLevel logLevel)) 73 | { 74 | logLevel = LogLevel.Debug; 75 | } 76 | 77 | var loggerFactory = LoggerFactory.Create(builder => builder 78 | .AddConsole(options => options.Format=ConsoleLoggerFormat.Systemd) 79 | .SetMinimumLevel(logLevel) 80 | ); 81 | 82 | // Enable Kubernetes client logging if level = DEBUG 83 | ServiceClientTracing.IsEnabled = logLevel <= LogLevel.Debug; 84 | ServiceClientTracing.AddTracingInterceptor(new ConsoleTracingInterceptor()); 85 | 86 | return loggerFactory; 87 | } 88 | 89 | void SetupSignalHandlers() 90 | { 91 | // SIGTERM: signal the operator to shut down gracefully 92 | AppDomain.CurrentDomain.ProcessExit += (s, e) => 93 | { 94 | logger.LogDebug("Received SIGTERM"); 95 | basicOperator?.Stop(); 96 | }; 97 | 98 | // SIGINT: try to shut down gracefully on the first attempt 99 | Console.CancelKeyPress += (s, e) => 100 | { 101 | bool isFirstSignal = !basicOperator.IsDisposing; 102 | logger.LogDebug($"Received SIGINT, first signal: {isFirstSignal}"); 103 | if (isFirstSignal) 104 | { 105 | e.Cancel = true; 106 | Environment.Exit(0); 107 | } 108 | }; 109 | } 110 | } 111 | 112 | private static OperatorConfiguration GetOperatorConfiguration() 113 | { 114 | var configuration = new OperatorConfiguration(); 115 | 116 | var retryPolicy = new RetryPolicy(); 117 | if (int.TryParse(Environment.GetEnvironmentVariable("RETRY_MAX_ATTEMPTS"), out int maxAttempts)) 118 | { 119 | retryPolicy.MaxAttempts = Math.Max(1, maxAttempts); 120 | } 121 | if (int.TryParse(Environment.GetEnvironmentVariable("RETRY_INITIAL_DELAY"), out int initialDelay)) 122 | { 123 | retryPolicy.InitialDelay = Math.Max(0, initialDelay); 124 | } 125 | if (int.TryParse(Environment.GetEnvironmentVariable("RETRY_DELAY_MULTIPLIER"), out int delayMultiplier)) 126 | { 127 | retryPolicy.DelayMultiplier = delayMultiplier; 128 | } 129 | configuration.RetryPolicy = retryPolicy; 130 | 131 | configuration.WatchNamespace = Environment.GetEnvironmentVariable("WATCH_NAMESPACE"); 132 | 133 | configuration.WatchLabelSelector = Environment.GetEnvironmentVariable("WATCH_LABEL_SELECTOR"); 134 | 135 | return configuration; 136 | } 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /samples/dynamic/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Threading.Tasks; 4 | using Microsoft.Extensions.Logging; 5 | using Microsoft.Extensions.Logging.Console; 6 | using Microsoft.Rest; 7 | using k8s.Operators.Logging; 8 | using Newtonsoft.Json; 9 | 10 | namespace k8s.Operators.Samples.Dynamic 11 | { 12 | class Program 13 | { 14 | static async Task Main(string[] args) 15 | { 16 | IOperator dynamicOperator = null; 17 | 18 | // Setup logging 19 | using var loggerFactory = SetupLogging(args); 20 | var logger = loggerFactory.CreateLogger(); 21 | 22 | try 23 | { 24 | logger.LogDebug($"Environment variables: {JsonConvert.SerializeObject(Environment.GetEnvironmentVariables())}"); 25 | 26 | // Setup termination handlers 27 | SetupSignalHandlers(); 28 | 29 | // Setup the Kubernetes client 30 | using var client = SetupClient(args); 31 | 32 | // Setup the operator 33 | var configuration = GetOperatorConfiguration(); 34 | dynamicOperator = new Operator(configuration, client, loggerFactory); 35 | dynamicOperator.AddControllerOfType(); 36 | 37 | // Start the operator 38 | return await dynamicOperator.StartAsync(); 39 | } 40 | catch (Exception exception) 41 | { 42 | logger.LogError(exception, "Operator error"); 43 | return 1; 44 | } 45 | 46 | IKubernetes SetupClient(string[] args) 47 | { 48 | // Load the Kubernetes configuration 49 | KubernetesClientConfiguration config = null; 50 | 51 | if (KubernetesClientConfiguration.IsInCluster()) 52 | { 53 | logger.LogDebug("Loading cluster configuration"); 54 | config = KubernetesClientConfiguration.InClusterConfig(); 55 | } 56 | else 57 | { 58 | logger.LogDebug("Loading local configuration"); 59 | config = KubernetesClientConfiguration.BuildConfigFromConfigFile(); 60 | } 61 | 62 | if (logger.IsEnabled(LogLevel.Debug)) 63 | { 64 | logger.LogDebug($"Client configuration: {JsonConvert.SerializeObject(config)}"); 65 | } 66 | 67 | return new Kubernetes(config); 68 | } 69 | 70 | ILoggerFactory SetupLogging(string[] args) 71 | { 72 | if (!System.Enum.TryParse(Environment.GetEnvironmentVariable("LOG_LEVEL"), true, out LogLevel logLevel)) 73 | { 74 | logLevel = LogLevel.Debug; 75 | } 76 | 77 | var loggerFactory = LoggerFactory.Create(builder => builder 78 | .AddConsole(options => options.Format=ConsoleLoggerFormat.Systemd) 79 | .SetMinimumLevel(logLevel) 80 | ); 81 | 82 | // Enable Kubernetes client logging if level = DEBUG 83 | ServiceClientTracing.IsEnabled = logLevel <= LogLevel.Debug; 84 | ServiceClientTracing.AddTracingInterceptor(new ConsoleTracingInterceptor()); 85 | 86 | return loggerFactory; 87 | } 88 | 89 | void SetupSignalHandlers() 90 | { 91 | // SIGTERM: signal the operator to shut down gracefully 92 | AppDomain.CurrentDomain.ProcessExit += (s, e) => 93 | { 94 | logger.LogDebug("Received SIGTERM"); 95 | dynamicOperator?.Stop(); 96 | }; 97 | 98 | // SIGINT: try to shut down gracefully on the first attempt 99 | Console.CancelKeyPress += (s, e) => 100 | { 101 | bool isFirstSignal = !dynamicOperator.IsDisposing; 102 | logger.LogDebug($"Received SIGINT, first signal: {isFirstSignal}"); 103 | if (isFirstSignal) 104 | { 105 | e.Cancel = true; 106 | Environment.Exit(0); 107 | } 108 | }; 109 | } 110 | } 111 | 112 | private static OperatorConfiguration GetOperatorConfiguration() 113 | { 114 | var configuration = new OperatorConfiguration(); 115 | 116 | var retryPolicy = new RetryPolicy(); 117 | if (int.TryParse(Environment.GetEnvironmentVariable("RETRY_MAX_ATTEMPTS"), out int maxAttempts)) 118 | { 119 | retryPolicy.MaxAttempts = Math.Max(1, maxAttempts); 120 | } 121 | if (int.TryParse(Environment.GetEnvironmentVariable("RETRY_INITIAL_DELAY"), out int initialDelay)) 122 | { 123 | retryPolicy.InitialDelay = Math.Max(0, initialDelay); 124 | } 125 | if (int.TryParse(Environment.GetEnvironmentVariable("RETRY_DELAY_MULTIPLIER"), out int delayMultiplier)) 126 | { 127 | retryPolicy.DelayMultiplier = delayMultiplier; 128 | } 129 | configuration.RetryPolicy = retryPolicy; 130 | 131 | configuration.WatchNamespace = Environment.GetEnvironmentVariable("WATCH_NAMESPACE"); 132 | 133 | configuration.WatchLabelSelector = Environment.GetEnvironmentVariable("WATCH_LABEL_SELECTOR"); 134 | 135 | return configuration; 136 | } 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /csharp-operator-sdk.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.26124.0 5 | MinimumVisualStudioVersion = 15.0.26124.0 6 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{474A20F0-CDE8-48DA-93D3-38FD7B9D6D60}" 7 | EndProject 8 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "k8s.Operators", "src\k8s.Operators\k8s.Operators.csproj", "{52D177C6-E107-4967-965F-39ECBF62823B}" 9 | EndProject 10 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{FCE39CFE-B106-4D00-9816-CCAABB327AC9}" 11 | EndProject 12 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "k8s.Operators.Tests", "tests\k8s.Operators.Tests\k8s.Operators.Tests.csproj", "{D3388B9E-D9FB-41D8-9E82-7FBB63A9B232}" 13 | EndProject 14 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "samples", "samples", "{760190CC-EB11-452A-B6BF-6ED305D8657F}" 15 | EndProject 16 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "k8s.Operators.Samples.Basic", "samples\basic\k8s.Operators.Samples.Basic.csproj", "{A737113F-AC8E-42E7-94A2-8C035BF11A94}" 17 | EndProject 18 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "k8s.Operators.Samples.Dynamic", "samples\dynamic\k8s.Operators.Samples.Dynamic.csproj", "{BC34DA91-5F79-49C0-A119-5F43B60E88C5}" 19 | EndProject 20 | Global 21 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 22 | Debug|Any CPU = Debug|Any CPU 23 | Debug|x64 = Debug|x64 24 | Debug|x86 = Debug|x86 25 | Release|Any CPU = Release|Any CPU 26 | Release|x64 = Release|x64 27 | Release|x86 = Release|x86 28 | EndGlobalSection 29 | GlobalSection(SolutionProperties) = preSolution 30 | HideSolutionNode = FALSE 31 | EndGlobalSection 32 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 33 | {52D177C6-E107-4967-965F-39ECBF62823B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 34 | {52D177C6-E107-4967-965F-39ECBF62823B}.Debug|Any CPU.Build.0 = Debug|Any CPU 35 | {52D177C6-E107-4967-965F-39ECBF62823B}.Debug|x64.ActiveCfg = Debug|Any CPU 36 | {52D177C6-E107-4967-965F-39ECBF62823B}.Debug|x64.Build.0 = Debug|Any CPU 37 | {52D177C6-E107-4967-965F-39ECBF62823B}.Debug|x86.ActiveCfg = Debug|Any CPU 38 | {52D177C6-E107-4967-965F-39ECBF62823B}.Debug|x86.Build.0 = Debug|Any CPU 39 | {52D177C6-E107-4967-965F-39ECBF62823B}.Release|Any CPU.ActiveCfg = Release|Any CPU 40 | {52D177C6-E107-4967-965F-39ECBF62823B}.Release|Any CPU.Build.0 = Release|Any CPU 41 | {52D177C6-E107-4967-965F-39ECBF62823B}.Release|x64.ActiveCfg = Release|Any CPU 42 | {52D177C6-E107-4967-965F-39ECBF62823B}.Release|x64.Build.0 = Release|Any CPU 43 | {52D177C6-E107-4967-965F-39ECBF62823B}.Release|x86.ActiveCfg = Release|Any CPU 44 | {52D177C6-E107-4967-965F-39ECBF62823B}.Release|x86.Build.0 = Release|Any CPU 45 | {D3388B9E-D9FB-41D8-9E82-7FBB63A9B232}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 46 | {D3388B9E-D9FB-41D8-9E82-7FBB63A9B232}.Debug|Any CPU.Build.0 = Debug|Any CPU 47 | {D3388B9E-D9FB-41D8-9E82-7FBB63A9B232}.Debug|x64.ActiveCfg = Debug|Any CPU 48 | {D3388B9E-D9FB-41D8-9E82-7FBB63A9B232}.Debug|x64.Build.0 = Debug|Any CPU 49 | {D3388B9E-D9FB-41D8-9E82-7FBB63A9B232}.Debug|x86.ActiveCfg = Debug|Any CPU 50 | {D3388B9E-D9FB-41D8-9E82-7FBB63A9B232}.Debug|x86.Build.0 = Debug|Any CPU 51 | {D3388B9E-D9FB-41D8-9E82-7FBB63A9B232}.Release|Any CPU.ActiveCfg = Release|Any CPU 52 | {D3388B9E-D9FB-41D8-9E82-7FBB63A9B232}.Release|Any CPU.Build.0 = Release|Any CPU 53 | {D3388B9E-D9FB-41D8-9E82-7FBB63A9B232}.Release|x64.ActiveCfg = Release|Any CPU 54 | {D3388B9E-D9FB-41D8-9E82-7FBB63A9B232}.Release|x64.Build.0 = Release|Any CPU 55 | {D3388B9E-D9FB-41D8-9E82-7FBB63A9B232}.Release|x86.ActiveCfg = Release|Any CPU 56 | {D3388B9E-D9FB-41D8-9E82-7FBB63A9B232}.Release|x86.Build.0 = Release|Any CPU 57 | {A737113F-AC8E-42E7-94A2-8C035BF11A94}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 58 | {A737113F-AC8E-42E7-94A2-8C035BF11A94}.Debug|Any CPU.Build.0 = Debug|Any CPU 59 | {A737113F-AC8E-42E7-94A2-8C035BF11A94}.Debug|x64.ActiveCfg = Debug|Any CPU 60 | {A737113F-AC8E-42E7-94A2-8C035BF11A94}.Debug|x64.Build.0 = Debug|Any CPU 61 | {A737113F-AC8E-42E7-94A2-8C035BF11A94}.Debug|x86.ActiveCfg = Debug|Any CPU 62 | {A737113F-AC8E-42E7-94A2-8C035BF11A94}.Debug|x86.Build.0 = Debug|Any CPU 63 | {A737113F-AC8E-42E7-94A2-8C035BF11A94}.Release|Any CPU.ActiveCfg = Release|Any CPU 64 | {A737113F-AC8E-42E7-94A2-8C035BF11A94}.Release|Any CPU.Build.0 = Release|Any CPU 65 | {A737113F-AC8E-42E7-94A2-8C035BF11A94}.Release|x64.ActiveCfg = Release|Any CPU 66 | {A737113F-AC8E-42E7-94A2-8C035BF11A94}.Release|x64.Build.0 = Release|Any CPU 67 | {A737113F-AC8E-42E7-94A2-8C035BF11A94}.Release|x86.ActiveCfg = Release|Any CPU 68 | {A737113F-AC8E-42E7-94A2-8C035BF11A94}.Release|x86.Build.0 = Release|Any CPU 69 | {BC34DA91-5F79-49C0-A119-5F43B60E88C5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 70 | {BC34DA91-5F79-49C0-A119-5F43B60E88C5}.Debug|Any CPU.Build.0 = Debug|Any CPU 71 | {BC34DA91-5F79-49C0-A119-5F43B60E88C5}.Debug|x64.ActiveCfg = Debug|Any CPU 72 | {BC34DA91-5F79-49C0-A119-5F43B60E88C5}.Debug|x64.Build.0 = Debug|Any CPU 73 | {BC34DA91-5F79-49C0-A119-5F43B60E88C5}.Debug|x86.ActiveCfg = Debug|Any CPU 74 | {BC34DA91-5F79-49C0-A119-5F43B60E88C5}.Debug|x86.Build.0 = Debug|Any CPU 75 | {BC34DA91-5F79-49C0-A119-5F43B60E88C5}.Release|Any CPU.ActiveCfg = Release|Any CPU 76 | {BC34DA91-5F79-49C0-A119-5F43B60E88C5}.Release|Any CPU.Build.0 = Release|Any CPU 77 | {BC34DA91-5F79-49C0-A119-5F43B60E88C5}.Release|x64.ActiveCfg = Release|Any CPU 78 | {BC34DA91-5F79-49C0-A119-5F43B60E88C5}.Release|x64.Build.0 = Release|Any CPU 79 | {BC34DA91-5F79-49C0-A119-5F43B60E88C5}.Release|x86.ActiveCfg = Release|Any CPU 80 | {BC34DA91-5F79-49C0-A119-5F43B60E88C5}.Release|x86.Build.0 = Release|Any CPU 81 | EndGlobalSection 82 | GlobalSection(NestedProjects) = preSolution 83 | {52D177C6-E107-4967-965F-39ECBF62823B} = {474A20F0-CDE8-48DA-93D3-38FD7B9D6D60} 84 | {D3388B9E-D9FB-41D8-9E82-7FBB63A9B232} = {FCE39CFE-B106-4D00-9816-CCAABB327AC9} 85 | {A737113F-AC8E-42E7-94A2-8C035BF11A94} = {760190CC-EB11-452A-B6BF-6ED305D8657F} 86 | {BC34DA91-5F79-49C0-A119-5F43B60E88C5} = {760190CC-EB11-452A-B6BF-6ED305D8657F} 87 | EndGlobalSection 88 | EndGlobal 89 | -------------------------------------------------------------------------------- /tests/k8s.Operators.Tests/OperatorTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using Microsoft.Rest; 4 | using Xunit; 5 | using Moq; 6 | using k8s.Operators; 7 | 8 | namespace k8s.Operators.Tests 9 | { 10 | public class OperatorTests : BaseTests 11 | { 12 | private TestableOperator _operator; 13 | 14 | public OperatorTests() 15 | { 16 | _operator = new TestableOperator(OperatorConfiguration.Default, _client, _loggerFactory); 17 | } 18 | 19 | [Fact] 20 | public void AddController_ThrowsExceptionIfControllerIsNull() 21 | { 22 | Assert.Throws(() => _operator.AddController(null)); 23 | } 24 | 25 | [Fact] 26 | public void Dispose_ExecutesOnlyOnce() 27 | { 28 | _operator.Dispose(); 29 | _operator.Dispose(); 30 | 31 | Assert.False(_operator.IsRunning); 32 | Assert.False(_operator.IsDisposing); 33 | Assert.True(_operator.IsDisposed); 34 | Assert.Equal(1, _operator.DisposeInvocationCount); 35 | } 36 | 37 | [Fact] 38 | public void Stop_ExecutesOnlyOnce() 39 | { 40 | _operator.Stop(); 41 | _operator.Stop(); 42 | 43 | Assert.False(_operator.IsRunning); 44 | Assert.False(_operator.IsDisposing); 45 | Assert.True(_operator.IsDisposed); 46 | Assert.Equal(1, _operator.DisposeInvocationCount); 47 | } 48 | 49 | [Fact] 50 | public async Task StartAsync_CallsDisposeAndStopIfNoControllersArePresent() 51 | { 52 | await _operator.StartAsync(); 53 | 54 | Assert.False(_operator.IsRunning); 55 | Assert.False(_operator.IsDisposing); 56 | Assert.True(_operator.IsDisposed); 57 | } 58 | 59 | [Fact] 60 | public void StartAsync_ThrowsExceptionIfDisposed() 61 | { 62 | _operator.AddController(new TestableController()); 63 | _operator.Dispose(); 64 | 65 | Assert.True(_operator.IsDisposed); 66 | Assert.False(_operator.IsDisposing); 67 | Assert.ThrowsAsync(() => _operator.StartAsync()); 68 | } 69 | 70 | [Fact] 71 | public void StartAsync_ThrowsExceptionIfStopped() 72 | { 73 | _operator.AddController(new TestableController()); 74 | _operator.Stop(); 75 | 76 | Assert.True(_operator.IsDisposed); 77 | Assert.False(_operator.IsDisposing); 78 | Assert.ThrowsAsync(() => _operator.StartAsync()); 79 | } 80 | 81 | [Fact] 82 | public async Task AddControllerOfType_CreatesAndAddController() 83 | { 84 | // Arrange 85 | var resource = CreateCustomResource(); 86 | 87 | // Act 88 | var controller = (TestableController) _operator.AddControllerOfType(); 89 | 90 | // Assert 91 | var task =_operator.StartAsync(); 92 | _operator.SimulateEvent(controller, WatchEventType.Added, resource); 93 | _operator.Stop(); await task; 94 | VerifyAddOrModifyIsCalledWith(controller, resource); 95 | } 96 | 97 | [Theory] 98 | [InlineData(WatchEventType.Added)] 99 | [InlineData(WatchEventType.Modified)] 100 | public async Task OnIncomingEvent_EventsAreDispatched(WatchEventType eventType) 101 | { 102 | // Arrange 103 | var resource = CreateCustomResource(ns: "default"); 104 | var controller = new TestableController(_client, _loggerFactory); 105 | _operator.AddController(controller); 106 | var task =_operator.StartAsync(); 107 | 108 | // Act 109 | _operator.SimulateEvent(controller, eventType, resource); 110 | 111 | // Assert 112 | _operator.Stop(); await task; 113 | VerifyAddOrModifyIsCalledWith(controller, resource); 114 | } 115 | 116 | [Theory] 117 | [InlineData(WatchEventType.Error)] 118 | [InlineData(WatchEventType.Deleted)] 119 | [InlineData(WatchEventType.Bookmark)] 120 | public async Task OnIncomingEvent_EventsAreDispatchedAndIgnored(WatchEventType eventType) 121 | { 122 | // Arrange 123 | var resource = CreateCustomResource(ns: "namespace1"); 124 | var controller = new TestableController(_client); 125 | _operator.AddController(controller, "namespace1"); 126 | var task =_operator.StartAsync(); 127 | 128 | // Act 129 | _operator.SimulateEvent(controller, eventType, resource); 130 | 131 | // Assert 132 | _operator.Stop(); await task; 133 | VerifyAddOrModifyIsNotCalled(controller); 134 | VerifyDeleteIsNotCalled(controller); 135 | } 136 | 137 | [Theory] 138 | [InlineData(WatchEventType.Added, "")] 139 | [InlineData(WatchEventType.Modified, "")] 140 | [InlineData(WatchEventType.Added, null)] 141 | [InlineData(WatchEventType.Modified, null)] 142 | public async Task OnIncomingEvent_EventsAreDispatchedToAssociatedControllers(WatchEventType eventType, string allNamespaceVariant) 143 | { 144 | // Arrange 145 | var resource1 = CreateCustomResource(ns: "namespace1"); 146 | var resource2 = CreateCustomResource(ns: "namespace2"); 147 | var controller1 = new TestableController(_client); 148 | var controller2 = new TestableController(_client); 149 | _operator.AddController(controller1, allNamespaceVariant); 150 | _operator.AddController(controller2, "namespace2"); 151 | var task =_operator.StartAsync(); 152 | 153 | // Act 154 | _operator.SimulateEvent(controller1, eventType, resource1); 155 | _operator.SimulateEvent(controller2, eventType, resource2); 156 | 157 | // Assert 158 | _operator.Stop(); await task; 159 | VerifyAddOrModifyIsCalledWith(controller1, resource1); 160 | VerifyAddOrModifyIsCalledWith(controller2, resource2); 161 | } 162 | } 163 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.rsuser 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # User-specific files (vscode) 17 | .vscode/settings.json 18 | 19 | # Mono auto generated files 20 | mono_crash.* 21 | 22 | # Build results 23 | [Dd]ebug/ 24 | [Dd]ebugPublic/ 25 | [Rr]elease/ 26 | [Rr]eleases/ 27 | x64/ 28 | x86/ 29 | [Aa][Rr][Mm]/ 30 | [Aa][Rr][Mm]64/ 31 | bld/ 32 | [Bb]in/ 33 | [Oo]bj/ 34 | [Ll]og/ 35 | [Ll]ogs/ 36 | [Oo]ut/ 37 | 38 | # Visual Studio 2015/2017 cache/options directory 39 | .vs/ 40 | # Uncomment if you have tasks that create the project's static files in wwwroot 41 | #wwwroot/ 42 | 43 | # Visual Studio 2017 auto generated files 44 | Generated\ Files/ 45 | 46 | # MSTest test Results 47 | [Tt]est[Rr]esult*/ 48 | [Bb]uild[Ll]og.* 49 | 50 | # NUnit 51 | *.VisualState.xml 52 | TestResult.xml 53 | nunit-*.xml 54 | 55 | # Build Results of an ATL Project 56 | [Dd]ebugPS/ 57 | [Rr]eleasePS/ 58 | dlldata.c 59 | 60 | # Benchmark Results 61 | BenchmarkDotNet.Artifacts/ 62 | 63 | # .NET Core 64 | project.lock.json 65 | project.fragment.lock.json 66 | artifacts/ 67 | 68 | # StyleCop 69 | StyleCopReport.xml 70 | 71 | # Files built by Visual Studio 72 | *_i.c 73 | *_p.c 74 | *_h.h 75 | *.ilk 76 | *.meta 77 | *.obj 78 | *.iobj 79 | *.pch 80 | *.pdb 81 | *.ipdb 82 | *.pgc 83 | *.pgd 84 | *.rsp 85 | *.sbr 86 | *.tlb 87 | *.tli 88 | *.tlh 89 | *.tmp 90 | *.tmp_proj 91 | *_wpftmp.csproj 92 | *.log 93 | *.vspscc 94 | *.vssscc 95 | .builds 96 | *.pidb 97 | *.svclog 98 | *.scc 99 | 100 | # Chutzpah Test files 101 | _Chutzpah* 102 | 103 | # Visual C++ cache files 104 | ipch/ 105 | *.aps 106 | *.ncb 107 | *.opendb 108 | *.opensdf 109 | *.sdf 110 | *.cachefile 111 | *.VC.db 112 | *.VC.VC.opendb 113 | 114 | # Visual Studio profiler 115 | *.psess 116 | *.vsp 117 | *.vspx 118 | *.sap 119 | 120 | # Visual Studio Trace Files 121 | *.e2e 122 | 123 | # TFS 2012 Local Workspace 124 | $tf/ 125 | 126 | # Guidance Automation Toolkit 127 | *.gpState 128 | 129 | # ReSharper is a .NET coding add-in 130 | _ReSharper*/ 131 | *.[Rr]e[Ss]harper 132 | *.DotSettings.user 133 | 134 | # TeamCity is a build add-in 135 | _TeamCity* 136 | 137 | # DotCover is a Code Coverage Tool 138 | *.dotCover 139 | 140 | # AxoCover is a Code Coverage Tool 141 | .axoCover/* 142 | !.axoCover/settings.json 143 | 144 | # Visual Studio code coverage results 145 | *.coverage 146 | *.coveragexml 147 | 148 | # NCrunch 149 | _NCrunch_* 150 | .*crunch*.local.xml 151 | nCrunchTemp_* 152 | 153 | # MightyMoose 154 | *.mm.* 155 | AutoTest.Net/ 156 | 157 | # Web workbench (sass) 158 | .sass-cache/ 159 | 160 | # Installshield output folder 161 | [Ee]xpress/ 162 | 163 | # DocProject is a documentation generator add-in 164 | DocProject/buildhelp/ 165 | DocProject/Help/*.HxT 166 | DocProject/Help/*.HxC 167 | DocProject/Help/*.hhc 168 | DocProject/Help/*.hhk 169 | DocProject/Help/*.hhp 170 | DocProject/Help/Html2 171 | DocProject/Help/html 172 | 173 | # Click-Once directory 174 | publish/ 175 | 176 | # Publish Web Output 177 | *.[Pp]ublish.xml 178 | *.azurePubxml 179 | # Note: Comment the next line if you want to checkin your web deploy settings, 180 | # but database connection strings (with potential passwords) will be unencrypted 181 | *.pubxml 182 | *.publishproj 183 | 184 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 185 | # checkin your Azure Web App publish settings, but sensitive information contained 186 | # in these scripts will be unencrypted 187 | PublishScripts/ 188 | 189 | # NuGet Packages 190 | *.nupkg 191 | # NuGet Symbol Packages 192 | *.snupkg 193 | # The packages folder can be ignored because of Package Restore 194 | **/[Pp]ackages/* 195 | # except build/, which is used as an MSBuild target. 196 | !**/[Pp]ackages/build/ 197 | # Uncomment if necessary however generally it will be regenerated when needed 198 | #!**/[Pp]ackages/repositories.config 199 | # NuGet v3's project.json files produces more ignorable files 200 | *.nuget.props 201 | *.nuget.targets 202 | 203 | # Microsoft Azure Build Output 204 | csx/ 205 | *.build.csdef 206 | 207 | # Microsoft Azure Emulator 208 | ecf/ 209 | rcf/ 210 | 211 | # Windows Store app package directories and files 212 | AppPackages/ 213 | BundleArtifacts/ 214 | Package.StoreAssociation.xml 215 | _pkginfo.txt 216 | *.appx 217 | *.appxbundle 218 | *.appxupload 219 | 220 | # Visual Studio cache files 221 | # files ending in .cache can be ignored 222 | *.[Cc]ache 223 | # but keep track of directories ending in .cache 224 | !?*.[Cc]ache/ 225 | 226 | # Others 227 | ClientBin/ 228 | ~$* 229 | *~ 230 | *.dbmdl 231 | *.dbproj.schemaview 232 | *.jfm 233 | *.pfx 234 | *.publishsettings 235 | orleans.codegen.cs 236 | 237 | # Including strong name files can present a security risk 238 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 239 | #*.snk 240 | 241 | # Since there are multiple workflows, uncomment next line to ignore bower_components 242 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 243 | #bower_components/ 244 | 245 | # RIA/Silverlight projects 246 | Generated_Code/ 247 | 248 | # Backup & report files from converting an old project file 249 | # to a newer Visual Studio version. Backup files are not needed, 250 | # because we have git ;-) 251 | _UpgradeReport_Files/ 252 | Backup*/ 253 | UpgradeLog*.XML 254 | UpgradeLog*.htm 255 | ServiceFabricBackup/ 256 | *.rptproj.bak 257 | 258 | # SQL Server files 259 | *.mdf 260 | *.ldf 261 | *.ndf 262 | 263 | # Business Intelligence projects 264 | *.rdl.data 265 | *.bim.layout 266 | *.bim_*.settings 267 | *.rptproj.rsuser 268 | *- [Bb]ackup.rdl 269 | *- [Bb]ackup ([0-9]).rdl 270 | *- [Bb]ackup ([0-9][0-9]).rdl 271 | 272 | # Microsoft Fakes 273 | FakesAssemblies/ 274 | 275 | # GhostDoc plugin setting file 276 | *.GhostDoc.xml 277 | 278 | # Node.js Tools for Visual Studio 279 | .ntvs_analysis.dat 280 | node_modules/ 281 | 282 | # Visual Studio 6 build log 283 | *.plg 284 | 285 | # Visual Studio 6 workspace options file 286 | *.opt 287 | 288 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 289 | *.vbw 290 | 291 | # Visual Studio LightSwitch build output 292 | **/*.HTMLClient/GeneratedArtifacts 293 | **/*.DesktopClient/GeneratedArtifacts 294 | **/*.DesktopClient/ModelManifest.xml 295 | **/*.Server/GeneratedArtifacts 296 | **/*.Server/ModelManifest.xml 297 | _Pvt_Extensions 298 | 299 | # Paket dependency manager 300 | .paket/paket.exe 301 | paket-files/ 302 | 303 | # FAKE - F# Make 304 | .fake/ 305 | 306 | # CodeRush personal settings 307 | .cr/personal 308 | 309 | # Python Tools for Visual Studio (PTVS) 310 | __pycache__/ 311 | *.pyc 312 | 313 | # Cake - Uncomment if you are using it 314 | # tools/** 315 | # !tools/packages.config 316 | 317 | # Tabs Studio 318 | *.tss 319 | 320 | # Telerik's JustMock configuration file 321 | *.jmconfig 322 | 323 | # BizTalk build output 324 | *.btp.cs 325 | *.btm.cs 326 | *.odx.cs 327 | *.xsd.cs 328 | 329 | # OpenCover UI analysis results 330 | OpenCover/ 331 | 332 | # Azure Stream Analytics local run output 333 | ASALocalRun/ 334 | 335 | # MSBuild Binary and Structured Log 336 | *.binlog 337 | 338 | # NVidia Nsight GPU debugger configuration file 339 | *.nvuser 340 | 341 | # MFractors (Xamarin productivity tool) working folder 342 | .mfractor/ 343 | 344 | # Local History for Visual Studio 345 | .localhistory/ 346 | 347 | # BeatPulse healthcheck temp database 348 | healthchecksdb 349 | 350 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 351 | MigrationBackup/ 352 | 353 | # Ionide (cross platform F# VS Code tools) working folder 354 | .ionide/ 355 | -------------------------------------------------------------------------------- /src/k8s.Operators/Operator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Reflection; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | using Microsoft.Rest; 7 | using Microsoft.Extensions.Logging; 8 | using k8s.Operators.Logging; 9 | using System.Diagnostics.CodeAnalysis; 10 | 11 | namespace k8s.Operators 12 | { 13 | /// 14 | /// Represents a Kubernetes operator 15 | /// 16 | public class Operator : Disposable, IOperator 17 | { 18 | private const string ALL_NAMESPACES = ""; 19 | 20 | private readonly ILogger _logger; 21 | private readonly OperatorConfiguration _configuration; 22 | private readonly IKubernetes _client; 23 | protected readonly List _watchers; 24 | private readonly CancellationTokenSource _cts; 25 | private readonly ILoggerFactory _loggerFactory; 26 | private bool _isStarted; 27 | private bool _unexpectedWatcherTermination; 28 | 29 | public Operator(OperatorConfiguration configuration, IKubernetes client, ILoggerFactory loggerFactory = null) 30 | { 31 | this._configuration = configuration; 32 | this._client = client; 33 | this._loggerFactory = loggerFactory; 34 | this._logger = loggerFactory?.CreateLogger() ?? SilentLogger.Instance; 35 | this._watchers = new List(); 36 | this._cts = new CancellationTokenSource(); 37 | 38 | TaskScheduler.UnobservedTaskException += (o, ev) => 39 | { 40 | _logger.LogError(ev.Exception, "Unobserved exception"); 41 | ev.SetObserved(); 42 | }; 43 | 44 | // TODO: log versions 45 | } 46 | 47 | /// 48 | /// Adds a controller to handle the events of the custom resource R 49 | /// 50 | /// The controller for the custom resource 51 | /// The watched namespace. Set to null to watch all namespaces 52 | /// The label selector to filter the sets of events returned/> 53 | /// The type of the custom resource 54 | public IOperator AddController(IController controller, string watchNamespace = "default", string labelSelector = null) where R : CustomResource 55 | { 56 | if (IsDisposed) 57 | { 58 | throw new ObjectDisposedException("Operator"); 59 | } 60 | 61 | if (controller == null) 62 | { 63 | throw new ValidationException(ValidationRules.CannotBeNull, nameof(controller)); 64 | } 65 | 66 | if (IsRunning) 67 | { 68 | throw new InvalidOperationException("A controller cannot be added once the operator has started"); 69 | } 70 | 71 | if (watchNamespace == null) 72 | { 73 | watchNamespace = ALL_NAMESPACES; 74 | } 75 | 76 | _logger.LogDebug($"Added controller {controller} on namespace {(string.IsNullOrEmpty(watchNamespace) ? "\"\"" : watchNamespace)}"); 77 | 78 | _watchers.Add(new EventWatcher(typeof(R), watchNamespace, labelSelector, controller, _logger, _cts.Token)); 79 | 80 | return this; 81 | } 82 | 83 | /// 84 | /// Adds a new instance of a controller of type C to handle the events of the custom resource 85 | /// 86 | /// The type of the controller. C must implement IController and expose a constructor that accepts (OperatorConfiguration, IKubernetes, ILoggerFactory) 87 | /// The instance of the controller 88 | public IController AddControllerOfType() where C : IController 89 | { 90 | // Use Reflection to instantiate the controller and pass it to AddController() 91 | 92 | // ASSUMPTION: C implements IController, where R is a custom resource 93 | 94 | // Retrieve the type of R 95 | var R = typeof(C).BaseType.GetGenericArguments()[0]; 96 | 97 | // Instantiate the controller implementing IController via the standard constructor (OperatorConfiguration, IKubernetes, ILoggerFactory) 98 | object controller = Activator.CreateInstance(typeof(C), _configuration, _client, _loggerFactory); 99 | 100 | // Invoke AddController() 101 | typeof(Operator) 102 | .GetMethod("AddController") 103 | .MakeGenericMethod(R) 104 | .Invoke(this, new object[] { controller, _configuration.WatchNamespace, _configuration.WatchLabelSelector }); 105 | 106 | return (IController) controller; 107 | } 108 | 109 | /// 110 | /// Starts watching and handling events 111 | /// 112 | public async Task StartAsync() 113 | { 114 | if (IsDisposed) 115 | { 116 | throw new ObjectDisposedException("Operator"); 117 | } 118 | 119 | _logger.LogInformation($"Start operator"); 120 | 121 | if (_watchers.Count == 0) 122 | { 123 | _logger.LogDebug($"No controller added, stopping operator"); 124 | Stop(); 125 | return 0; 126 | } 127 | 128 | _isStarted = true; 129 | 130 | var tasks = new List(); 131 | 132 | foreach (var entry in _watchers) 133 | { 134 | // Invoke WatchCustomResourceAsync() via reflection, since T is in a variable 135 | var watchCustomResourceAsync = typeof(Operator) 136 | .GetMethod("WatchCustomResourceAsync", BindingFlags.NonPublic | BindingFlags.Instance) 137 | .MakeGenericMethod(entry.ResourceType); 138 | 139 | // Start a watcher for each 140 | var watcher = ((Task) watchCustomResourceAsync.Invoke(this, new object[] { entry })) 141 | .ContinueWith(t => 142 | { 143 | if (t.IsFaulted) 144 | { 145 | _logger.LogError(t.Exception.Flatten().InnerException, $"Error watching {entry.Namespace}/{entry.CRD.Plural} {entry.LabelSelector}"); 146 | } 147 | }); 148 | 149 | tasks.Add(watcher); 150 | } 151 | 152 | await Task.WhenAll(tasks.ToArray()); 153 | 154 | return _unexpectedWatcherTermination ? 1 : 0; 155 | } 156 | 157 | /// 158 | /// Stops the operator and release the resources. Once stopped, an operator cannot be restarted. Stop() is an alias for Dispose() 159 | /// 160 | public void Stop() 161 | { 162 | _logger.LogInformation($"Stop operator"); 163 | Dispose(); 164 | } 165 | 166 | /// 167 | /// Returns true if StartAsync has been called and the operator is running 168 | /// 169 | public bool IsRunning => !IsDisposing && !IsDisposed && _isStarted; 170 | 171 | /// 172 | /// Watches for events for a given resource definition and namespace. If namespace is empty string, it watches all namespaces 173 | /// 174 | private async Task WatchCustomResourceAsync(EventWatcher watcher) where T : CustomResource 175 | { 176 | if (IsDisposing || IsDisposed) 177 | { 178 | return; 179 | } 180 | 181 | var response = await _client.ListNamespacedCustomObjectWithHttpMessagesAsync( 182 | watcher.CRD.Group, 183 | watcher.CRD.Version, 184 | watcher.Namespace, 185 | watcher.CRD.Plural, 186 | watch: true, 187 | labelSelector: watcher.LabelSelector, 188 | timeoutSeconds: (int)TimeSpan.FromMinutes(60).TotalSeconds, 189 | cancellationToken: _cts.Token 190 | ).ConfigureAwait(false); 191 | 192 | _logger.LogDebug($"Begin watch {watcher.Namespace}/{watcher.CRD.Plural} {watcher.LabelSelector}"); 193 | 194 | using (var _ = response.Watch(watcher.OnIncomingEvent, OnWatcherError, OnWatcherClose)) 195 | { 196 | await WaitOneAsync(_cts.Token.WaitHandle); 197 | 198 | _logger.LogDebug($"End watch {watcher.Namespace}/{watcher.CRD.Plural} {watcher.LabelSelector}"); 199 | } 200 | } 201 | 202 | [ExcludeFromCodeCoverage] 203 | protected void OnWatcherError(Exception exception) 204 | { 205 | if (IsRunning) 206 | { 207 | _logger.LogError(exception, "Watcher error"); 208 | } 209 | } 210 | 211 | [ExcludeFromCodeCoverage] 212 | protected virtual void OnWatcherClose() 213 | { 214 | _logger.LogError("Watcher closed"); 215 | 216 | if (IsRunning) 217 | { 218 | // At least one watcher stopped unexpectedly. Stop the operator, let Kubernetes restart it 219 | _unexpectedWatcherTermination = true; 220 | Stop(); 221 | } 222 | } 223 | 224 | /// 225 | /// Returns a Task wrapper for a synchronous wait on a wait handle 226 | /// 227 | /// 228 | private Task WaitOneAsync(WaitHandle waitHandle, int millisecondsTimeOutInterval = Timeout.Infinite) 229 | { 230 | if (waitHandle == null) 231 | { 232 | throw new ArgumentNullException(nameof(waitHandle)); 233 | } 234 | 235 | var tcs = new TaskCompletionSource(); 236 | 237 | var rwh = ThreadPool.RegisterWaitForSingleObject( 238 | waitHandle, 239 | callBack: (state, timedOut) => { tcs.TrySetResult(!timedOut); }, 240 | state: null, 241 | millisecondsTimeOutInterval: millisecondsTimeOutInterval, 242 | executeOnlyOnce: true 243 | ); 244 | 245 | var task = tcs.Task; 246 | 247 | task.ContinueWith(t => 248 | { 249 | rwh.Unregister(waitObject: null); 250 | try 251 | { 252 | return t.Result; 253 | } 254 | catch 255 | { 256 | return false; 257 | throw; 258 | } 259 | }); 260 | 261 | return task; 262 | } 263 | 264 | protected override void DisposeInternal() 265 | { 266 | _logger.LogInformation($"Disposing operator"); 267 | 268 | // Signal the watchers to stop 269 | _cts.Cancel(); 270 | } 271 | } 272 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2020 Alberto Falossi 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /src/k8s.Operators/Controller.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics.CodeAnalysis; 3 | using System.Threading; 4 | using System.Threading.Tasks; 5 | using k8s; 6 | using k8s.Models; 7 | using k8s.Operators.Logging; 8 | using Microsoft.AspNetCore.JsonPatch; 9 | using Microsoft.Extensions.Logging; 10 | using Microsoft.Rest; 11 | using Newtonsoft.Json; 12 | using Newtonsoft.Json.Linq; 13 | 14 | namespace k8s.Operators 15 | { 16 | /// 17 | /// Controller of a custom resource of type T 18 | /// 19 | public abstract class Controller : IController where T : CustomResource 20 | { 21 | protected readonly ILogger _logger; 22 | protected readonly IKubernetes _client; 23 | private readonly EventManager _eventManager; 24 | private readonly ResourceChangeTracker _changeTracker; 25 | private readonly CustomResourceDefinitionAttribute _crd; 26 | 27 | public Controller(OperatorConfiguration configuration, IKubernetes client, ILoggerFactory loggerFactory = null) 28 | { 29 | this._client = client; 30 | this._logger = loggerFactory?.CreateLogger>() ?? SilentLogger.Instance; 31 | this._eventManager = new EventManager(loggerFactory); 32 | this._changeTracker = new ResourceChangeTracker(configuration, loggerFactory); 33 | this._crd = (CustomResourceDefinitionAttribute) Attribute.GetCustomAttribute(typeof(T), typeof(CustomResourceDefinitionAttribute)); 34 | this.RetryPolicy = configuration.RetryPolicy; 35 | } 36 | 37 | /// 38 | /// Retry policy for the controller 39 | /// 40 | public RetryPolicy RetryPolicy { get; protected set; } 41 | 42 | /// 43 | /// Processes a custom resource event 44 | /// 45 | /// The event to handle 46 | /// Signals if the current execution has been canceled 47 | public async Task ProcessEventAsync(CustomResourceEvent resourceEvent, CancellationToken cancellationToken) 48 | { 49 | _logger.LogDebug($"Begin ProcessEvent, {resourceEvent}"); 50 | 51 | if (resourceEvent.Type == WatchEventType.Error) 52 | { 53 | _logger.LogError($"Received Error event, {resourceEvent.Resource}"); 54 | return; 55 | } 56 | 57 | if (resourceEvent.Type == WatchEventType.Deleted) 58 | { 59 | // Skip Deleted events since there is nothing else to do 60 | _logger.LogDebug($"Skip ProcessEvent, received Deleted event, {resourceEvent.Resource}"); 61 | return; 62 | } 63 | 64 | if (resourceEvent.Type == WatchEventType.Bookmark) 65 | { 66 | // Skip Bookmark events since there is nothing else to do 67 | _logger.LogDebug($"Skip ProcessEvent, received Bookmark event, {resourceEvent.Resource}"); 68 | return; 69 | } 70 | 71 | // Enqueue the event 72 | _eventManager.Enqueue(resourceEvent); 73 | 74 | while (!cancellationToken.IsCancellationRequested) 75 | { 76 | // Dequeue the next event to process for this resource, if any 77 | var nextEvent =_eventManager.Dequeue(resourceEvent.ResourceUid); 78 | if (nextEvent == null) 79 | { 80 | break; 81 | } 82 | 83 | await HandleEventAsync(nextEvent, cancellationToken); 84 | } 85 | 86 | _logger.LogDebug($"End ProcessEvent, {resourceEvent}"); 87 | } 88 | 89 | private async Task HandleEventAsync(CustomResourceEvent resourceEvent, CancellationToken cancellationToken) 90 | { 91 | if (resourceEvent == null) 92 | { 93 | _logger.LogWarning($"Skip HandleEvent, {nameof(resourceEvent)} is null"); 94 | return; 95 | } 96 | 97 | _logger.LogDebug($"Begin HandleEvent, {resourceEvent}"); 98 | 99 | _eventManager.BeginHandleEvent(resourceEvent); 100 | 101 | var attempt = 1; 102 | var delay = RetryPolicy.InitialDelay; 103 | while (true) 104 | { 105 | // Try to handle the event 106 | var handled = await TryHandleEventAsync(resourceEvent, cancellationToken); 107 | if (handled) 108 | { 109 | break; 110 | } 111 | 112 | // Something went wrong 113 | if (!CanTryAgain(resourceEvent, attempt, cancellationToken)) 114 | { 115 | break; 116 | } 117 | 118 | _logger.LogDebug($"Retrying to handle {resourceEvent} in {delay}ms (attempt #{attempt})"); 119 | 120 | // Wait 121 | await Task.Delay(delay); 122 | 123 | // Increase the delay for the next attempt 124 | attempt++; 125 | delay = (int)(delay * RetryPolicy.DelayMultiplier); 126 | } 127 | 128 | _logger.LogDebug($"End HandleEvent, {resourceEvent}"); 129 | 130 | _eventManager.EndHandleEvent(resourceEvent); 131 | } 132 | 133 | private bool CanTryAgain(CustomResourceEvent resourceEvent, int attemptNumber, CancellationToken cancellationToken) 134 | { 135 | if (cancellationToken.IsCancellationRequested) 136 | { 137 | _logger.LogDebug($"Cannot retry {resourceEvent}, processing has been canceled"); 138 | return false; 139 | } 140 | 141 | var upcoming = _eventManager.Peek(resourceEvent.ResourceUid); 142 | if (upcoming != null) 143 | { 144 | _logger.LogDebug($"Cannot retry {resourceEvent}, received {upcoming} in the meantime"); 145 | return false; 146 | } 147 | 148 | if (attemptNumber > RetryPolicy.MaxAttempts) 149 | { 150 | _logger.LogDebug($"Cannot retry {resourceEvent}, max number of attempts reached"); 151 | return false; 152 | } 153 | 154 | return true; 155 | } 156 | 157 | private async Task TryHandleEventAsync(CustomResourceEvent resourceEvent, CancellationToken cancellationToken) 158 | { 159 | bool handled = true; 160 | 161 | try 162 | { 163 | var resource = (T)resourceEvent.Resource; 164 | 165 | if (IsDeletePending(resource)) 166 | { 167 | await HandleDeletedEventAsync(resource, cancellationToken); 168 | } 169 | else 170 | { 171 | await HandleAddedOrModifiedEventAsync(resource, cancellationToken); 172 | } 173 | } 174 | catch (OperationCanceledException) 175 | { 176 | _logger.LogDebug($"Canceled HandleEvent, {resourceEvent}"); 177 | } 178 | catch (Exception exception) 179 | { 180 | if (exception is HttpOperationException httpException && httpException.Response?.StatusCode == System.Net.HttpStatusCode.Conflict) 181 | { 182 | // Conflicts happen. The next event will make the resource consistent again 183 | _logger.LogDebug(exception, $"Conflict handling {resourceEvent}"); 184 | } 185 | else 186 | { 187 | _logger.LogError(exception, $"Error handling {resourceEvent}"); 188 | handled = false; 189 | } 190 | } 191 | 192 | return handled; 193 | } 194 | 195 | private async Task HandleAddedOrModifiedEventAsync(T resource, CancellationToken cancellationToken) 196 | { 197 | _logger.LogDebug($"Handle Added/Modified, {resource}"); 198 | 199 | if (!HasFinalizer(resource)) 200 | { 201 | // Before any custom logic, add a finalizer to be used later during the deletion phase 202 | _logger.LogDebug($"Add missing finalizer"); 203 | await AddFinalizerAsync(resource, cancellationToken); 204 | return; 205 | } 206 | 207 | if (_changeTracker.IsResourceGenerationAlreadyHandled(resource)) 208 | { 209 | _logger.LogDebug($"Skip AddOrModifyAsync, {resource} already handled"); 210 | } 211 | else 212 | { 213 | _logger.LogDebug($"Begin AddOrModifyAsync, {resource}"); 214 | 215 | // Add/modify logic (implemented by the derived class) 216 | await AddOrModifyAsync(resource, cancellationToken); 217 | 218 | _changeTracker.TrackResourceGenerationAsHandled(resource); 219 | 220 | _logger.LogDebug($"End AddOrModifyAsync, {resource}"); 221 | } 222 | } 223 | 224 | private async Task HandleDeletedEventAsync(T resource, CancellationToken cancellationToken) 225 | { 226 | _logger.LogDebug($"Handle Deleted, {resource}"); 227 | 228 | if (!HasFinalizer(resource)) 229 | { 230 | // The current deletion request is not handled by this controller 231 | _logger.LogDebug($"Skip OnDeleted, {resource} has no finalizer"); 232 | return; 233 | } 234 | 235 | _logger.LogDebug($"Begin OnDeleted, {resource}"); 236 | 237 | // Delete logic (implemented by the derived class) 238 | await DeleteAsync(resource, cancellationToken); 239 | 240 | _changeTracker.TrackResourceGenerationAsDeleted(resource); 241 | 242 | if (HasFinalizer(resource)) 243 | { 244 | await RemoveFinalizerAsync(resource, cancellationToken); 245 | } 246 | 247 | _logger.LogDebug($"End OnDeleted, {resource}"); 248 | } 249 | 250 | /// 251 | /// Implements the logic to add or modify a resource 252 | /// 253 | /// Resource being added or modified 254 | /// Signals if the current execution has been canceled 255 | [ExcludeFromCodeCoverage] 256 | protected virtual Task AddOrModifyAsync(T resource, CancellationToken cancellationToken) 257 | { 258 | return Task.FromResult(0); 259 | } 260 | 261 | /// 262 | /// Implements the logic to delete a resource 263 | /// 264 | /// Resource being deleted 265 | /// Signals if the current execution has been canceled 266 | /// 267 | [ExcludeFromCodeCoverage] 268 | protected virtual Task DeleteAsync(T resource, CancellationToken cancellationToken) 269 | { 270 | return Task.FromResult(0); 271 | } 272 | 273 | /// 274 | /// Updates the status subresource 275 | /// 276 | /// 277 | protected Task UpdateStatusAsync(R resource, CancellationToken cancellationToken) where R : T, IStatus 278 | { 279 | // Build the delta JSON 280 | var patch = new JsonPatchDocument().Replace(x => x.Status, resource.Status); 281 | 282 | return PatchCustomResourceStatusAsync(resource, patch, cancellationToken); 283 | } 284 | 285 | /// 286 | /// Updates the resource (except the status) 287 | /// 288 | protected Task UpdateResourceAsync(T resource, CancellationToken cancellationToken) 289 | { 290 | return ReplaceCustomResourceAsync(resource, cancellationToken); 291 | } 292 | 293 | private bool IsDeletePending(CustomResource resource) 294 | { 295 | return resource.Metadata.DeletionTimestamp != null; 296 | } 297 | 298 | private bool HasFinalizer(CustomResource resource) 299 | { 300 | return resource.Metadata.Finalizers?.Contains(_crd.Finalizer) == true; 301 | } 302 | 303 | private Task AddFinalizerAsync(T resource, CancellationToken cancellationToken) 304 | { 305 | // Add the finalizer 306 | resource.Metadata.EnsureFinalizers().Add(_crd.Finalizer); 307 | 308 | return ReplaceCustomResourceAsync(resource, cancellationToken); 309 | } 310 | 311 | private Task RemoveFinalizerAsync(T resource, CancellationToken cancellationToken) 312 | { 313 | // Remove the finalizer 314 | resource.Metadata.Finalizers.Remove(_crd.Finalizer); 315 | 316 | return ReplaceCustomResourceAsync(resource, cancellationToken); 317 | } 318 | 319 | private async Task ReplaceCustomResourceAsync(T resource, CancellationToken cancellationToken) 320 | { 321 | _logger.LogDebug($"Replace Custom Resource, {(resource == null ? "" : JsonConvert.SerializeObject(resource))}"); 322 | 323 | // Replace the resource 324 | var result = await _client.ReplaceNamespacedCustomObjectAsync( 325 | resource, 326 | _crd.Group, 327 | _crd.Version, 328 | resource.Metadata.NamespaceProperty, 329 | _crd.Plural, 330 | resource.Metadata.Name, 331 | cancellationToken: cancellationToken 332 | ).ConfigureAwait(false); 333 | 334 | return ToCustomResource(result); 335 | } 336 | 337 | private async Task PatchCustomResourceStatusAsync(R resource, IJsonPatchDocument patch, CancellationToken cancellationToken) where R : T, IStatus 338 | { 339 | _logger.LogDebug($"Patch Status, {(patch == null ? "" : JsonConvert.SerializeObject(patch))}"); 340 | 341 | // Patch the status 342 | var result = await _client.PatchNamespacedCustomObjectStatusAsync( 343 | new V1Patch(patch), 344 | _crd.Group, 345 | _crd.Version, 346 | resource.Metadata.NamespaceProperty, 347 | _crd.Plural, 348 | resource.Metadata.Name, 349 | cancellationToken: cancellationToken 350 | ).ConfigureAwait(false); 351 | 352 | return ToCustomResource(result); 353 | } 354 | 355 | private T ToCustomResource(object input) 356 | { 357 | T result = default(T); 358 | 359 | if (input is JObject json) 360 | { 361 | result = json.ToObject(); 362 | } 363 | else 364 | { 365 | result = (T)input; 366 | } 367 | 368 | return result; 369 | } 370 | } 371 | } -------------------------------------------------------------------------------- /tests/k8s.Operators.Tests/ControllerTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading; 3 | using System.Threading.Tasks; 4 | using k8s.Models; 5 | using k8s.Operators; 6 | using Microsoft.AspNetCore.JsonPatch; 7 | using Newtonsoft.Json; 8 | using Xunit; 9 | using Moq; 10 | 11 | namespace k8s.Operators.Tests 12 | { 13 | public class ControllerTests : BaseTests 14 | { 15 | private TestableController _controller; 16 | 17 | public ControllerTests() 18 | { 19 | _controller = new TestableController(OperatorConfiguration.Default,_client); 20 | _controller.RetryPolicy.InitialDelay = 1; 21 | } 22 | 23 | [Theory] 24 | [InlineData(WatchEventType.Added)] 25 | [InlineData(WatchEventType.Modified)] 26 | public async Task ProcessEventAsync_AddOrModifyIsCalled(WatchEventType eventType) 27 | { 28 | // Arrange 29 | var resource = CreateCustomResource(); 30 | var resourceEvent = new CustomResourceEvent(eventType, resource); 31 | 32 | // Act 33 | await _controller.ProcessEventAsync(resourceEvent, DUMMY_TOKEN); 34 | 35 | // Assert 36 | VerifyAddOrModifyIsCalledWith(_controller, resource); 37 | VerifyDeleteIsNotCalled(_controller); 38 | VerifyNoOtherApiIsCalled(); 39 | } 40 | 41 | [Theory] 42 | [InlineData(WatchEventType.Error)] 43 | [InlineData(WatchEventType.Deleted)] 44 | [InlineData(WatchEventType.Bookmark)] 45 | public async Task ProcessEventAsync_AddOrModifyIsNotCalled(WatchEventType eventType) 46 | { 47 | // Arrange 48 | var resource = CreateCustomResource(); 49 | var resourceEvent = new CustomResourceEvent(eventType, resource); 50 | 51 | // Act 52 | await _controller.ProcessEventAsync(resourceEvent, DUMMY_TOKEN); 53 | 54 | // Assert 55 | VerifyAddOrModifyIsNotCalled(_controller); 56 | VerifyDeleteIsNotCalled(_controller); 57 | VerifyNoOtherApiIsCalled(); 58 | } 59 | 60 | [Theory] 61 | [InlineData(WatchEventType.Added)] 62 | [InlineData(WatchEventType.Modified)] 63 | public async Task ProcessEventAsync_DuplicateGenerationsAreDiscardedByDefault(WatchEventType eventType) 64 | { 65 | // Arrange 66 | var resource_v1 = CreateCustomResource(generation: 1); 67 | var resource_v2 = CreateCustomResource(generation: 2); 68 | 69 | // Act 70 | await _controller.ProcessEventAsync(new CustomResourceEvent(eventType, resource_v1), DUMMY_TOKEN); 71 | await _controller.ProcessEventAsync(new CustomResourceEvent(eventType, resource_v1), DUMMY_TOKEN); // duplicate generation 72 | await _controller.ProcessEventAsync(new CustomResourceEvent(eventType, resource_v2), DUMMY_TOKEN); 73 | 74 | // Assert 75 | VerifyAddOrModifyIsCalledWith(_controller, resource_v1, resource_v2); 76 | VerifyDeleteIsNotCalled(_controller); 77 | VerifyNoOtherApiIsCalled(); 78 | } 79 | 80 | [Theory] 81 | [InlineData(WatchEventType.Added)] 82 | [InlineData(WatchEventType.Modified)] 83 | public async Task ProcessEventAsync_DuplicateGenerationsAreProcessedIfForcedByConfiguration(WatchEventType eventType) 84 | { 85 | // Arrange 86 | var controller = new TestableController(new OperatorConfiguration() { DiscardDuplicateSpecGenerations = false },_client); 87 | var resource_v1 = CreateCustomResource(generation: 1); 88 | var resource_v2 = CreateCustomResource(generation: 2); 89 | 90 | // Act 91 | await controller.ProcessEventAsync(new CustomResourceEvent(eventType, resource_v1), DUMMY_TOKEN); 92 | await controller.ProcessEventAsync(new CustomResourceEvent(eventType, resource_v1), DUMMY_TOKEN); // duplicate generation 93 | await controller.ProcessEventAsync(new CustomResourceEvent(eventType, resource_v2), DUMMY_TOKEN); 94 | 95 | // Assert 96 | VerifyAddOrModifyIsCalledWith(controller, resource_v1, resource_v1, resource_v2); 97 | VerifyDeleteIsNotCalled(controller); 98 | VerifyNoOtherApiIsCalled(); 99 | } 100 | 101 | [Fact] 102 | public async Task ProcessEventAsync_DeleteIsCalled() 103 | { 104 | // Arrange 105 | var resource = CreateCustomResource(deletionTimeStamp: DateTime.Now); 106 | var resourceEvent = new CustomResourceEvent(WatchEventType.Modified, resource); 107 | 108 | // Act 109 | await _controller.ProcessEventAsync(resourceEvent, DUMMY_TOKEN); 110 | 111 | // Assert 112 | VerifyAddOrModifyIsNotCalled(_controller); 113 | VerifyDeleteIsCalledWith(_controller, resource); 114 | VerifyReplaceIsCalled(resource, out TestableCustomResource updatedResource); 115 | Assert.Equal(0, updatedResource.Metadata.Finalizers.Count); 116 | } 117 | 118 | 119 | [Fact] 120 | public async Task ProcessEventAsync_UpdateResourceCallsReplaceApi() 121 | { 122 | // Arrange 123 | var resource = CreateCustomResource(); 124 | 125 | // Act 126 | await _controller.Exposed_UpdateResourceAsync(resource, DUMMY_TOKEN); 127 | 128 | // Assert 129 | VerifyReplaceIsCalled(resource, out TestableCustomResource _); 130 | } 131 | 132 | [Fact] 133 | public async Task ProcessEventAsync_UpdateStatusCallsPatchApi() 134 | { 135 | // Arrange 136 | var resource = CreateCustomResource(); 137 | resource.Spec.Property = "before"; 138 | resource.Status.Property = "before"; 139 | 140 | // Act 141 | await _controller.Exposed_UpdateStatusAsync(resource, DUMMY_TOKEN); 142 | 143 | // Assert 144 | var patch = new JsonPatchDocument().Replace(x => x.Status, resource.Status); 145 | VerifyPatchIsCalled(patch); 146 | VerifyNoOtherApiIsCalled(); 147 | } 148 | 149 | [Theory] 150 | [InlineData(WatchEventType.Added)] 151 | [InlineData(WatchEventType.Modified)] 152 | public async Task ProcessEventAsync_MissingFinalizerIsAdded(WatchEventType eventType) 153 | { 154 | // Arrange 155 | var resource = CreateCustomResource(withFinalizer: false); 156 | var resourceEvent = new CustomResourceEvent(eventType, resource); 157 | 158 | // Act 159 | await _controller.ProcessEventAsync(resourceEvent, DUMMY_TOKEN); 160 | 161 | // Assert 162 | VerifyReplaceIsCalled(resource, out TestableCustomResource updatedResource); 163 | Assert.Equal(CustomResourceDefinitionAttribute.DEFAULT_FINALIZER, updatedResource.Metadata.Finalizers[0]); 164 | } 165 | 166 | [Theory] 167 | [InlineData(WatchEventType.Error, true)] 168 | [InlineData(WatchEventType.Error, false)] 169 | [InlineData(WatchEventType.Deleted, true)] 170 | [InlineData(WatchEventType.Deleted, false)] 171 | [InlineData(WatchEventType.Bookmark, true)] 172 | [InlineData(WatchEventType.Bookmark, false)] 173 | [InlineData(WatchEventType.Added, true)] 174 | [InlineData(WatchEventType.Modified, true)] 175 | public async Task ProcessEventAsync_FinalizerIsNotAdded(WatchEventType eventType, bool withFinalizer) 176 | { 177 | // Arrange 178 | var resource = CreateCustomResource(withFinalizer: withFinalizer); 179 | var resourceEvent = new CustomResourceEvent(eventType, resource); 180 | 181 | // Act 182 | await _controller.ProcessEventAsync(resourceEvent, DUMMY_TOKEN); 183 | 184 | // Assert 185 | VerifyNoOtherApiIsCalled(); 186 | } 187 | 188 | [Theory] 189 | [InlineData(WatchEventType.Added, false, WatchEventType.Modified, false)] 190 | [InlineData(WatchEventType.Added, true, WatchEventType.Modified, false)] 191 | [InlineData(WatchEventType.Added, false, WatchEventType.Modified, true)] 192 | [InlineData(WatchEventType.Added, true, WatchEventType.Modified, true)] 193 | 194 | [InlineData(WatchEventType.Modified, false, WatchEventType.Added, false)] 195 | [InlineData(WatchEventType.Modified, true, WatchEventType.Added, false)] 196 | [InlineData(WatchEventType.Modified, false, WatchEventType.Added, true)] 197 | [InlineData(WatchEventType.Modified, true, WatchEventType.Added, true)] 198 | 199 | [InlineData(WatchEventType.Modified, false, WatchEventType.Modified, false)] 200 | [InlineData(WatchEventType.Modified, true, WatchEventType.Modified, false)] 201 | [InlineData(WatchEventType.Modified, false, WatchEventType.Modified, true)] 202 | [InlineData(WatchEventType.Modified, true, WatchEventType.Modified, true)] 203 | public void ProcessEventAsync_EventsForSameResourceAreProcessedSerially(WatchEventType eventType1, bool delete1, WatchEventType eventType2, bool delete2) 204 | { 205 | var resource_v1 = CreateCustomResource(generation: 1, deletionTimeStamp: delete1 ? DateTime.Now : (DateTime?) null); 206 | var resource_v2 = CreateCustomResource(generation: 2, deletionTimeStamp: delete2 ? DateTime.Now : (DateTime?) null); 207 | 208 | // Send 2 updates in a row for the same resource 209 | 210 | // Update #1 211 | var token1 = _controller.BlockNextEvent(); 212 | var task1 = _controller.ProcessEventAsync(new CustomResourceEvent(eventType1, resource_v1), DUMMY_TOKEN); 213 | 214 | // Update #2 215 | var token2 = _controller.BlockNextEvent(); 216 | var task2 = _controller.ProcessEventAsync(new CustomResourceEvent(eventType2, resource_v2), DUMMY_TOKEN); 217 | 218 | // Update #1 starts, update #2 is waiting 219 | VerifyCalledEvents(_controller, (resource_v1, delete1)); 220 | 221 | // Update #1 ends, update #2 starts 222 | _controller.UnblockEvent(token1); 223 | VerifyCalledEvents(_controller, (resource_v1, delete1), (resource_v2, delete2)); 224 | 225 | // Update #2 ends 226 | _controller.UnblockEvent(token2); 227 | Task.WaitAll(task2, task1); 228 | } 229 | 230 | [Theory] 231 | [InlineData(WatchEventType.Added, false, WatchEventType.Modified, false)] 232 | [InlineData(WatchEventType.Added, true, WatchEventType.Modified, false)] 233 | [InlineData(WatchEventType.Added, false, WatchEventType.Modified, true)] 234 | [InlineData(WatchEventType.Added, true, WatchEventType.Modified, true)] 235 | 236 | [InlineData(WatchEventType.Modified, false, WatchEventType.Added, false)] 237 | [InlineData(WatchEventType.Modified, true, WatchEventType.Added, false)] 238 | [InlineData(WatchEventType.Modified, false, WatchEventType.Added, true)] 239 | [InlineData(WatchEventType.Modified, true, WatchEventType.Added, true)] 240 | 241 | [InlineData(WatchEventType.Modified, false, WatchEventType.Modified, false)] 242 | [InlineData(WatchEventType.Modified, true, WatchEventType.Modified, false)] 243 | [InlineData(WatchEventType.Modified, false, WatchEventType.Modified, true)] 244 | [InlineData(WatchEventType.Modified, true, WatchEventType.Modified, true)] 245 | public void ProcessEventAsync_EventsForDifferentResourceAreProcessedConcurrently(WatchEventType eventType1, bool delete1, WatchEventType eventType2, bool delete2) 246 | { 247 | var resource1 = CreateCustomResource(uid: "1", deletionTimeStamp: delete1 ? DateTime.Now : (DateTime?) null); 248 | var resource2 = CreateCustomResource(uid: "2", deletionTimeStamp: delete2 ? DateTime.Now : (DateTime?) null); 249 | 250 | // Send 2 updates in a row for the different resources 251 | 252 | // Update #1 253 | var token1 = _controller.BlockNextEvent(); 254 | var task1 = _controller.ProcessEventAsync(new CustomResourceEvent(eventType1, resource1), DUMMY_TOKEN); 255 | 256 | // Update #2 257 | var token2 = _controller.BlockNextEvent(); 258 | var task2 = _controller.ProcessEventAsync(new CustomResourceEvent(eventType2, resource2), DUMMY_TOKEN); 259 | 260 | // Updates are processed concurrently 261 | VerifyCalledEvents(_controller, (resource1, delete1), (resource2, delete2)); 262 | 263 | // Updates end 264 | _controller.UnblockEvent(token1); 265 | _controller.UnblockEvent(token2); 266 | Task.WaitAll(task2, task1); 267 | } 268 | 269 | [Theory] 270 | [InlineData(WatchEventType.Added, true)] 271 | [InlineData(WatchEventType.Modified, true)] 272 | [InlineData(WatchEventType.Added, false)] 273 | [InlineData(WatchEventType.Modified, false)] 274 | public async Task ProcessEventAsync_RetryOnFailure(WatchEventType eventType, bool delete) 275 | { 276 | // Arrange 277 | var resource = CreateCustomResource(deletionTimeStamp: delete ? DateTime.Now : (DateTime?) null); 278 | var resourceEvent = new CustomResourceEvent(eventType, resource); 279 | _controller.ThrowExceptionOnNextEvents(_controller.RetryPolicy.MaxAttempts); // ProcessEventAsync will fail n times, where n = MaxAttempts 280 | 281 | // Act 282 | await _controller.ProcessEventAsync(resourceEvent, DUMMY_TOKEN); 283 | 284 | // Assert 285 | VerifyCompletedEvents(_controller, (resource, deleteEvent: delete)); 286 | } 287 | 288 | [Theory] 289 | [InlineData(WatchEventType.Added, true)] 290 | [InlineData(WatchEventType.Modified, true)] 291 | [InlineData(WatchEventType.Added, false)] 292 | [InlineData(WatchEventType.Modified, false)] 293 | public async Task ProcessEventAsync_NoRetryAfterMaxAttempts(WatchEventType eventType, bool delete) 294 | { 295 | // Arrange 296 | var resource = CreateCustomResource(deletionTimeStamp: delete ? DateTime.Now : (DateTime?) null); 297 | var resourceEvent = new CustomResourceEvent(eventType, resource); 298 | _controller.ThrowExceptionOnNextEvents(_controller.RetryPolicy.MaxAttempts + 1); // ProcessEventAsync will fail n + 1 times, where n = MaxAttempts 299 | 300 | // Act 301 | await _controller.ProcessEventAsync(resourceEvent, DUMMY_TOKEN); 302 | 303 | // Assert 304 | VerifyCompletedEvents(_controller); 305 | } 306 | 307 | [Theory] 308 | [InlineData(WatchEventType.Added, true)] 309 | [InlineData(WatchEventType.Modified, true)] 310 | [InlineData(WatchEventType.Added, false)] 311 | [InlineData(WatchEventType.Modified, false)] 312 | public void ProcessEventAsync_NoRetryIfNewEventForSameResourceIsQueued(WatchEventType eventType, bool delete) 313 | { 314 | // Arrange 315 | var resource_v1 = CreateCustomResource(generation: 1, deletionTimeStamp: delete ? DateTime.Now : (DateTime?) null); 316 | var resource_v2 = CreateCustomResource(generation: 2, deletionTimeStamp: delete ? DateTime.Now : (DateTime?) null); 317 | _controller.ThrowExceptionOnNextEvents(1); 318 | var block = _controller.BlockNextEvent(); 319 | 320 | // Event #1, will block 321 | var task1 = _controller.ProcessEventAsync(new CustomResourceEvent(eventType, resource_v1), DUMMY_TOKEN); 322 | 323 | // Event #2, queued 324 | var task2 = _controller.ProcessEventAsync(new CustomResourceEvent(eventType, resource_v2), DUMMY_TOKEN); 325 | 326 | // Unblock #1, will throw an exception. But since #2 has arrived in the meantime, #1 will be discarded 327 | _controller.UnblockEvent(block); 328 | 329 | Task.WaitAll(task2, task1); 330 | 331 | // Assert 332 | VerifyCompletedEvents(_controller, (resource_v2, deleteEvent: delete)); 333 | } 334 | 335 | [Theory] 336 | [InlineData(WatchEventType.Added, true)] 337 | [InlineData(WatchEventType.Modified, true)] 338 | [InlineData(WatchEventType.Added, false)] 339 | [InlineData(WatchEventType.Modified, false)] 340 | public void ProcessEventAsync_RetryIfNewEventForAnotherResourceIsQueued(WatchEventType eventType, bool delete) 341 | { 342 | // Arrange 343 | var resource1 = CreateCustomResource(uid: "1", deletionTimeStamp: delete ? DateTime.Now : (DateTime?) null); 344 | var resource2 = CreateCustomResource(uid: "2", deletionTimeStamp: delete ? DateTime.Now : (DateTime?) null); 345 | _controller.ThrowExceptionOnNextEvents(1); 346 | var block1 = _controller.BlockNextEvent(); 347 | 348 | // Event #1, will block 349 | var task1 = _controller.ProcessEventAsync(new CustomResourceEvent(eventType, resource1), DUMMY_TOKEN); 350 | 351 | // Event #2, will be processed since it's a different resource 352 | var task2 = _controller.ProcessEventAsync(new CustomResourceEvent(eventType, resource2), DUMMY_TOKEN); 353 | 354 | // Unblock #1 355 | _controller.UnblockEvent(block1); 356 | 357 | Task.WaitAll(task2, task1); 358 | 359 | // Assert 360 | VerifyCompletedEvents(_controller, (resource1, deleteEvent: delete), (resource2, deleteEvent: delete)); 361 | } 362 | 363 | [Theory] 364 | [InlineData(WatchEventType.Added, true)] 365 | [InlineData(WatchEventType.Modified, true)] 366 | [InlineData(WatchEventType.Added, false)] 367 | [InlineData(WatchEventType.Modified, false)] 368 | public void ProcessEventAsync_NoRetryIfCancelIsRequested(WatchEventType eventType, bool delete) 369 | { 370 | // Arrange 371 | var resource = CreateCustomResource(deletionTimeStamp: delete ? DateTime.Now : (DateTime?) null); 372 | var cts = new CancellationTokenSource(); 373 | _controller.ThrowExceptionOnNextEvents(1); 374 | var block = _controller.BlockNextEvent(); 375 | 376 | // Will block 377 | var task1 = _controller.ProcessEventAsync(new CustomResourceEvent(eventType, resource), cts.Token); 378 | 379 | // 380 | cts.Cancel(); 381 | 382 | // Unblock #1, will throw an exception. But since #2 has arrived in the meantime, #1 will be discarded 383 | _controller.UnblockEvent(block); 384 | 385 | Task.WaitAll(task1); 386 | 387 | // Assert 388 | VerifyCompletedEvents(_controller); 389 | } 390 | 391 | [Theory] 392 | [InlineData(WatchEventType.Added)] 393 | [InlineData(WatchEventType.Modified)] 394 | public async Task ProcessEventAsync_WithDynamicCustomResource(WatchEventType eventType) 395 | { 396 | // Arrange 397 | var resource = new TestableDynamicCustomResource(); 398 | resource.Spec.property = "desired"; 399 | resource.Status.property = "actual"; 400 | var resourceEvent = new CustomResourceEvent(eventType, resource); 401 | var controller = new TestableDynamicController(); 402 | 403 | // Act 404 | await controller.ProcessEventAsync(resourceEvent, DUMMY_TOKEN); 405 | 406 | // Assert 407 | Assert.Equal(resource.Spec.property, resource.Status.property); 408 | } 409 | 410 | private void VerifyNoOtherApiIsCalled() => _clientMock.VerifyNoOtherCalls(); 411 | 412 | private void VerifyReplaceIsCalled(object input, out TestableCustomResource updatedResource) 413 | { 414 | // Get the resource that has been passed to the API 415 | var resource = _clientMock.Invocations[0].Arguments[0] as TestableCustomResource; 416 | 417 | // Verify the API has been called once 418 | _clientMock.Verify(x => x.ReplaceNamespacedCustomObjectWithHttpMessagesAsync 419 | ( 420 | input, 421 | "group", 422 | "v1", 423 | "ns1", 424 | "resources", 425 | "resource1", 426 | null, 427 | default(System.Threading.CancellationToken) 428 | ), Times.Once); 429 | 430 | updatedResource = resource; 431 | 432 | _clientMock.VerifyNoOtherCalls(); 433 | } 434 | 435 | private void VerifyPatchIsCalled(IJsonPatchDocument expected) 436 | { 437 | // Verify the API has been called once 438 | _clientMock.Verify(x => x.PatchNamespacedCustomObjectStatusWithHttpMessagesAsync 439 | ( 440 | It.IsAny(), 441 | "group", 442 | "v1", 443 | "ns1", 444 | "resources", 445 | "resource1", 446 | null, 447 | default(System.Threading.CancellationToken) 448 | ), Times.Once); 449 | 450 | // Semantic equal assertion (JsonPatchDocument.Equals doesn't compare the content) 451 | var actual = (_clientMock.Invocations[0].Arguments[0] as V1Patch).Content as IJsonPatchDocument; 452 | Assert.Equal(JsonConvert.SerializeObject(expected), JsonConvert.SerializeObject(actual)); 453 | 454 | _clientMock.VerifyNoOtherCalls(); 455 | } 456 | } 457 | } --------------------------------------------------------------------------------