├── .gitignore ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── ReadMe.md ├── SimmyDemo_WebApi.sln └── SimmyDemo_WebApi ├── Chaos ├── AppChaosSettings.cs ├── OperationChaosSetting.cs ├── SimmyContextExtensions.cs └── SimmyExtensions.cs ├── Controllers └── MonitoringController.cs ├── EndpointResult.cs ├── MonitoringResults.cs ├── MonitoringSettings.cs ├── Program.cs ├── Properties └── launchSettings.json ├── ResilientHttpClient.cs ├── SimmyDemo_WebApi.csproj ├── Startup.cs ├── appsettings.Development.json └── appsettings.json /.gitignore: -------------------------------------------------------------------------------- 1 | # Build Folders (you can keep bin if you'd like, to store dlls and pdbs) 2 | [Bb]in/ 3 | [Oo]bj/ 4 | 5 | # mstest test results 6 | TestResults 7 | 8 | ## Ignore Visual Studio temporary files, build results, and 9 | ## files generated by popular Visual Studio add-ons. 10 | 11 | # User-specific files 12 | *.suo 13 | *.user 14 | *.sln.docstates 15 | .vs/ 16 | .vscode/ 17 | 18 | # Build results 19 | [Dd]ebug/ 20 | [Rr]elease/ 21 | x64/ 22 | *_i.c 23 | *_p.c 24 | *.ilk 25 | *.meta 26 | *.obj 27 | *.pch 28 | *.pdb 29 | *.pgc 30 | *.pgd 31 | *.rsp 32 | *.sbr 33 | *.tlb 34 | *.tli 35 | *.tlh 36 | *.tmp 37 | *.log 38 | *.vspscc 39 | *.vssscc 40 | .builds 41 | 42 | # Visual C++ cache files 43 | ipch/ 44 | *.aps 45 | *.ncb 46 | *.opensdf 47 | *.sdf 48 | 49 | # Visual Studio profiler 50 | *.psess 51 | *.vsp 52 | *.vspx 53 | 54 | # Guidance Automation Toolkit 55 | *.gpState 56 | 57 | # ReSharper is a .NET coding add-in 58 | _ReSharper* 59 | 60 | # NCrunch 61 | *.ncrunch* 62 | .*crunch*.local.xml 63 | 64 | # GhostDoc 65 | *.GhostDoc.xml 66 | 67 | # Installshield output folder 68 | [Ee]xpress 69 | 70 | # DocProject is a documentation generator add-in 71 | DocProject/buildhelp/ 72 | DocProject/Help/*.HxT 73 | DocProject/Help/*.HxC 74 | DocProject/Help/*.hhc 75 | DocProject/Help/*.hhk 76 | DocProject/Help/*.hhp 77 | DocProject/Help/Html2 78 | DocProject/Help/html 79 | 80 | # Click-Once directory 81 | publish 82 | 83 | # Publish Web Output 84 | *.Publish.xml 85 | 86 | # NuGet Packages Directory 87 | packages 88 | 89 | # Windows Azure Build Output 90 | csx 91 | *.build.csdef 92 | 93 | # Windows Store app package directory 94 | AppPackages/ 95 | 96 | # Others 97 | [Bb]in 98 | [Oo]bj 99 | sql 100 | TestResults 101 | [Tt]est[Rr]esult* 102 | *.Cache 103 | ClientBin 104 | [Ss]tyle[Cc]op.* 105 | ~$* 106 | *.dbmdl 107 | Generated_Code #added for RIA/Silverlight projects 108 | 109 | # Backup & report files from converting an old project file to a newer 110 | # Visual Studio version. Backup files are not needed, because we have git ;-) 111 | _UpgradeReport_Files/ 112 | Backup*/ 113 | UpgradeLog*.XML 114 | 115 | artifacts 116 | build 117 | tools 118 | 119 | *.lock.json 120 | *.nuget.targets 121 | *.nuget.props -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 0.2.0 2 | - Updates Simmy to the latest version 3 | - Refactors monkey policies to use the new syntax API 4 | - Updates project to use .net core 3.1 5 | 6 | ## 0.1.0 7 | - Initial launch 8 | 9 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | We ask our contributors to abide by the [Code of Conduct of the .NET Foundation](https://www.dotnetfoundation.org/code-of-conduct). 2 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Please see the Instructions for Contributing in the [Polly ReadMe](https://github.com/App-vNext/Polly#instructions-for-contributing) and [Polly wiki](https://github.com/App-vNext/Polly/wiki/Git-Workflow). 2 | 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2019, Polly.Contrib 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /ReadMe.md: -------------------------------------------------------------------------------- 1 | # Simmy sample app 2 | 3 | [Simmy](https://github.com/Polly-Contrib/Simmy) is a chaos-engineering and fault-injection tool, integrating with the Polly resilience project for .NET. Simmy allows you to introduce a chaos-injection policy or policies at any location where you execute code through Polly. 4 | 5 | This repo presents an intentionally simple example .NET Core WebAPI app demonstrating Simmy (originally put together as a simple demo for use in conference presentations). 6 | 7 | >_Be sure also to check out the [introductory blog post](http://elvanydev.com/chaos-injection-with-simmy/) and [demo](https://github.com/vany0114/chaos-injection-using-simmy) from lead contributor [Geovanny Alzate Sandoval](https://github.com/vany0114), if you want a more developed example demonstrating Simmy among a set of distributed microservices run from Docker containers._ 8 | 9 | The app in this repo demonstrates the following patterns with Simmy: 10 | 11 | + Configuring `StartUp` so that Simmy chaos policies are only introduced in builds for certain environments (for instance, Dev but not Prod) 12 | + Configuring Simmy chaos policies to be injected into the app without changing any existing configuration code 13 | + Injecting faults or chaos by modifying external configuration. 14 | 15 | The patterns shown in this sample app are not mandatory. They are intended to demonstrate approaches you could take when introducing Simmy to an app, but Simmy is very flexible, and comments in this article describe how you could also take Simmy further. 16 | 17 | ## The sample app: a simple monitoring service 18 | 19 | The example app is an intentionally simplified endpoint monitoring service, reporting on the health of endpoints configured in the `MonitoringEndpoints` section of `appsettings.json`. 20 | 21 | The app offers two public endpoints: 22 | 23 | + `monitoring/status` returns the health of each monitored url, just as the HttpStatusCode 24 | + `monitoring/responsetime` returns the ResponseTime of each monitored url, in ms. 25 | 26 | (The use of two separate endpoints here just helps us demonstrate injecting faults into one downstream operation but not another.) 27 | 28 | The simple metrics returned - status code and response time - allow us to easily see the results of introducing faults in to our calls. ;~) 29 | 30 | ## Run the app without fault injection 31 | 32 | Run the app without fault injection. If in Visual Studio, starting the app should open a web page calling `/monitoring/status`. If it doesn't, navigate to that endpoint manually. You should receive results something like this: 33 | 34 | ``` 35 | [ 36 | { 37 | "url": "www.bbc.co.uk", 38 | "value": 200, 39 | }, 40 | { 41 | "url": "www.google.co.uk", 42 | "value": 200 43 | } 44 | ] 45 | ``` 46 | 47 | ## Injecting faults or chaos 48 | 49 | Open the file `appsettings.json` in the root folder of the app and look for the region `"ChaosSettings": { }`. This can be configured with fault-injection settings for any number of call sites within your app: 50 | 51 | "ChaosSettings": { 52 | "OperationChaosSettings": [ 53 | { 54 | "OperationKey": "Status", 55 | "Enabled": true, 56 | "InjectionRate": 0.75, 57 | "LatencyMs": 0, 58 | "StatusCode": 503, 59 | }, 60 | { 61 | "OperationKey": "ResponseTime", 62 | "Enabled": false, 63 | "InjectionRate": 0.1, 64 | "LatencyMs": 2000, 65 | "Exception": "System.SocketException" 66 | } 67 | ] 68 | } 69 | 70 | _The use of_ `appsettings.json` _here to drive chaos is just the simplest-possible technique to provide a demo of varying chaos settings which is self-contained and can be run locally. In a production environment, you are likely to want to drive chaos settings from an easier-to-manipulate source._ 71 | 72 | The elements are: 73 | 74 | #### OperationKey 75 | 76 | Which operation within your app these chaos settings apply to. Each call site in your codebase which uses Polly and Simmy can be tagged with an `OperationKey`: 77 | 78 | Context context = new Context("FooOperationKey"); 79 | 80 | 81 | This is simply a string tag you choose, to identify different call paths in your app. Steps to attach this further to your http call are shown in the sample app. 82 | 83 | #### Enabled 84 | 85 | A master switch for this call site. When `true`, faults may be injected at this call site per the other parameters; when `false`, no faults will be injected. 86 | 87 | #### InjectionRate 88 | 89 | A `double` between 0 and 1, indicating what proportion of calls should be subject to failure-injection. For example, if `0.2`, twenty percent of calls will be randomly affected; if `0.01`, one percent of calls; if `1`, all calls. 90 | 91 | #### Latency 92 | 93 | If set, this much extra latency in ms will be added to affected calls, before the http request is made. 94 | 95 | #### StatusCode 96 | 97 | If set, a result with the given http status code will be returned for affected calls. (The original outbound http call will not be placed.) 98 | 99 | #### Exception 100 | 101 | If set, affected calls will throw the given exception. (The original outbound http call will not be placed.) 102 | 103 | ### Live update during running 104 | 105 | The sample app is constructed using `IOptionsSnapshot<>` so that adjusting the settings immediately affects subsequent calls. 106 | 107 | ### Complete example: Inject a different status code 108 | 109 | "ChaosSettings": { 110 | "OperationChaosSettings": [ 111 | { 112 | "OperationKey": "Status", 113 | "Enabled": true, 114 | "InjectionRate": 1, 115 | "StatusCode": 503, 116 | } 117 | ] 118 | } 119 | 120 | #### Expected result (/monitoring/status) 121 | 122 | {"results":[{"url":"http://www.google.co.uk/","value":503},{"url":"http://www.bbc.co.uk/","value":503}]} 123 | 124 | > _Note:_ During startup the sample app configures a limited resilience policy which retries typical failure status codes a couple of times. Therefore, do not be surprised if you configure a 50% injection rate (`"InjectionRate": 0.5`) for a 503 code but see 503s actually surfacing less frequently in the demo - the resilience policy will be handling _some_ of them. 125 | 126 | ### Complete example: Inject latency 127 | 128 | "ChaosSettings": { 129 | "OperationChaosSettings": [ 130 | { 131 | "OperationKey": "ResponseTime", 132 | "Enabled": true, 133 | "InjectionRate": 1, 134 | "LatencyMs": 2000, 135 | } 136 | ] 137 | } 138 | 139 | #### Expected result (/monitoring/responsetime) 140 | 141 | {"results":[{"url":"http://www.google.co.uk/","value":2262},{"url":"http://www.bbc.co.uk/","value":2526}]} 142 | 143 | ### Complete example: Inject OperationCanceledException 144 | 145 | "ChaosSettings": { 146 | "OperationChaosSettings": [ 147 | { 148 | "OperationKey": "Status", 149 | "Enabled": true, 150 | "InjectionRate": 1, 151 | "Exception": "System.OperationCanceledException" 152 | } 153 | ] 154 | } 155 | 156 | #### Expected result (/monitoring/status) 157 | 158 | An unhandled exception occurred while processing the request. 159 | OperationCanceledException: The operation was canceled. 160 | 161 | ## How the sample app injects the chaos 162 | 163 | Calls guarded by Polly policies often wrap a series of policies around a call using `PolicyWrap`. The policies in the PolicyWrap act as nesting middleware around the outbound call. 164 | 165 | The recommended technique for introducing `Simmy` is to use one or more Simmy chaos policies as the _innermost_ policies in a `PolicyWrap`. 166 | 167 | By placing the chaos policies innermost, they subvert the usual outbound call at the last minute, substituting their fault or adding extra latency. 168 | 169 | The existing Polly policies - further out in the PolicyWrap - still apply, so you can test how the Polly resilience you have configured handles the chaos/faults injected by Simmy. 170 | 171 | ## Experimenting with adjusting resilience policies to handle injected faults (a simple example) 172 | 173 | The sample app is intentionally undefended from exceptions, so that you can see the exceptions surface in the examples above. 174 | 175 | Now you can experiment with changing the resilience in your app to handle the faults that occur. 176 | 177 | First, in `appsettings.json`, change the injection rate for `OperationCanceledException` to inject faults 50% of the time: `"InjectionRate": 0.5`. 178 | 179 | Run the endpoint and you should see many calls fail with `OperationCanceledException`. 180 | 181 | Now, in the sample app, in the method `GetResiliencePolicy()`, change the retry policy so that it also handles `OperationCanceledException`, retrying a number of times: 182 | 183 | var retry = HttpPolicyExtensions.HandleTransientHttpError() 184 | .Or() 185 | .RetryAsync(3); 186 | 187 | Running the endpoint with the extra configured resilience should significantly reduce the number of `OperationCanceledException` which actually surface to the caller as errors. 188 | 189 | This is an intentionally simplistic example to demonstrate iterating a feedback loop from experimenting with faults to adjusting policies. Of course, you can run far more sophisticated chaos experiments on a real app: introducing 100ms latency to all database calls briefly, and see if your retry/circuit-breaker policies are configured to give a good customer experience in those circumstances; block all calls to a recommendations subsystem - whatever. 190 | 191 | ## Adding Simmy chaos without changing existing configuration code 192 | 193 | As mentioned above, the usual technique to add chaos-injection is to configure Simmy policies innermost in your app's `PolicyWrap`s. 194 | 195 | One of the simplest ways to do this all across your app is to make all policies used in your app be stored in and drawn from `PolicyRegistry`. This is the technique demonstrated in this sample app. 196 | 197 | In `StartUp`, all the Polly policies which will be used are configured, and registered in `PolicyRegistry`: 198 | 199 | var policyRegistry = services.AddPolicyRegistry(); 200 | policyRegistry["ResiliencePolicy"] = GetResiliencePolicy(); 201 | 202 | Typed-clients are configured on `HttpClientFactory`, which will use policies from `PolicyRegistry`: 203 | 204 | services.AddHttpClient() 205 | .AddPolicyHandlerFromRegistry("ResiliencePolicy"); 206 | 207 | 208 | > (_When using Polly and Simmy without HttpClientFactory_, simply pass the `PolicyRegistry` by DI into the components making outbound calls, and pull the appropriate policy out of `PolicyRegistry` at the call site.) 209 | 210 | If you have taken the above `PolicyRegistry`-driven approach, the sample app demonstrates a very simple technique that can be used to add Simmy throughout your app, . The `AddChaosInjectors()` extension method on `IPolicyRegistry<>` simply takes every policy in your `PolicyRegistry` and wraps Simmy policies (as the innermost policy) inside. 211 | 212 | // Only add Simmy chaos injection in development-environment runs 213 | // (ie prevent chaos-injection ever reaching staging or prod - if that is what you want). 214 | if (env.IsDevelopment()) 215 | { 216 | // Wrap every policy in the policy registry in Simmy chaos injectors. 217 | var registry = app.ApplicationServices.GetRequiredService>(); 218 | registry?.AddChaosInjectors(); 219 | } 220 | 221 | 222 | This allows you to inject Simmy into your app without changing any of your existing app configuration of Polly policies. 223 | 224 | This extension method configures the policies in your PolicyRegistry with Simmy policies which react to chaos configured by `chaossettings.json`. 225 | 226 | The code lines above also demonstrate a construct to ensure that fault-injection is only included in builds for certain environments: for example if you want to inject chaos into stage environments but not prod. 227 | 228 | ## Using other sources to control chaos settings 229 | 230 | The use of `chaossettings.json` here is just the simplest-possible technique to provide a self-contained demo of varying chaos settings, which can be run locally. In a production environment, you are likely not to be wanting to vary appsettings to control chaos. 231 | 232 | + Any other config source can equally be used drive chaos settings; 233 | + An http endpoint (suitably secured!) could be used to set chaos settings. 234 | 235 | ## Filtering how and what chaos is applied using constructs particular to your app 236 | 237 | The fault-injection policies configured by `InjectBehaviour(...)`, `InjectLatency(...)` and `InjectFault(...)` can all be configured with `Func<>`s which take `Polly.Context` as an input parameter. And `Polly.Context` can carry any arbitrary data, using `Dictionary` semantics. 238 | 239 | You can therefore build policies to control chaos based on _any_ custom data particular to your app. 240 | 241 | For example, it may be that the urls of downstream systems in your app follow certain patterns, and you filter on the url to introduce chaos to only certain subsystems, or only certain primaries/failovers. 242 | 243 | Or you might choose to whitelist or blacklist certain callers, so that chaos is only introduced for your test callers but not for your live customers. 244 | 245 | Every parameter of the chaos policy exists in a form taking a `Func` for configuration, so all dimensions of the chaos policy - whether it is enabled, what proportion of calls should be affected, and what chaos should be injected - can be inflected by data set on the `Context` passed to execution. 246 | 247 | ## Going beyond http calls 248 | 249 | The sample app here demonstrates Simmy policies configured into `HttpClient` instances provided by `HttpClientFactory`. The chaos therefore governs outbound http calls. 250 | 251 | Again, this is just an example. Polly and Simmy policies are not tied to http calls. All Polly and Simmy policies exist in generic `` forms, and can be used around any type of call, including: 252 | 253 | + calls to SDKs for your storage, be that via Entity Framework, MongoDB, whatever 254 | + calls to any part a cloud SDK (Azure, AWS, GCP). 255 | -------------------------------------------------------------------------------- /SimmyDemo_WebApi.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.28307.438 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SimmyDemo_WebApi", "SimmyDemo_WebApi\SimmyDemo_WebApi.csproj", "{6177775A-FED5-4A37-AE82-B82703292511}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Release|Any CPU = Release|Any CPU 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {6177775A-FED5-4A37-AE82-B82703292511}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 15 | {6177775A-FED5-4A37-AE82-B82703292511}.Debug|Any CPU.Build.0 = Debug|Any CPU 16 | {6177775A-FED5-4A37-AE82-B82703292511}.Release|Any CPU.ActiveCfg = Release|Any CPU 17 | {6177775A-FED5-4A37-AE82-B82703292511}.Release|Any CPU.Build.0 = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(SolutionProperties) = preSolution 20 | HideSolutionNode = FALSE 21 | EndGlobalSection 22 | GlobalSection(ExtensibilityGlobals) = postSolution 23 | SolutionGuid = {2B5D2E55-3196-44B8-B1D1-4C4F0FA2F582} 24 | EndGlobalSection 25 | EndGlobal 26 | -------------------------------------------------------------------------------- /SimmyDemo_WebApi/Chaos/AppChaosSettings.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | 4 | namespace SimmyDemo_WebApi.Chaos 5 | { 6 | public class AppChaosSettings 7 | { 8 | public List OperationChaosSettings { get; set; } 9 | 10 | public OperationChaosSetting GetSettingsFor(string operationKey) => OperationChaosSettings?.SingleOrDefault(i => i.OperationKey == operationKey); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /SimmyDemo_WebApi/Chaos/OperationChaosSetting.cs: -------------------------------------------------------------------------------- 1 | namespace SimmyDemo_WebApi.Chaos 2 | { 3 | public class OperationChaosSetting 4 | { 5 | public string OperationKey { get; set; } 6 | 7 | public bool Enabled { get; set; } 8 | 9 | public double InjectionRate { get; set; } 10 | 11 | public int StatusCode { get; set; } 12 | 13 | public int LatencyMs { get; set; } 14 | 15 | public string Exception { get; set; } 16 | } 17 | } -------------------------------------------------------------------------------- /SimmyDemo_WebApi/Chaos/SimmyContextExtensions.cs: -------------------------------------------------------------------------------- 1 | using Polly; 2 | 3 | namespace SimmyDemo_WebApi.Chaos 4 | { 5 | public static class SimmyContextExtensions 6 | { 7 | public const string ChaosSettings = "ChaosSettings"; 8 | 9 | public static Context WithChaosSettings(this Context context, AppChaosSettings options) 10 | { 11 | context[ChaosSettings] = options; 12 | return context; 13 | } 14 | 15 | public static AppChaosSettings GetChaosSettings(this Context context) => context.GetSetting(ChaosSettings); 16 | 17 | private static T GetSetting(this Context context, string key) 18 | { 19 | if (context.TryGetValue(key, out object setting)) 20 | { 21 | if (setting is T) 22 | { 23 | return (T)setting; 24 | } 25 | } 26 | return default(T); 27 | } 28 | 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /SimmyDemo_WebApi/Chaos/SimmyExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Net; 4 | using System.Net.Http; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | using Polly; 8 | using Polly.Contrib.Simmy; 9 | using Polly.Contrib.Simmy.Latency; 10 | using Polly.Contrib.Simmy.Outcomes; 11 | using Polly.Registry; 12 | using SimmyDemo_WebApi.Chaos; 13 | 14 | namespace SimmyDemo_WebApi 15 | { 16 | public static class SimmyExtensions 17 | { 18 | private static OperationChaosSetting GetOperationChaosSettings(this Context context) => context.GetChaosSettings()?.GetSettingsFor(context.OperationKey); 19 | 20 | private static readonly Task NotEnabled = Task.FromResult(false); 21 | private static readonly Task NoInjectionRate = Task.FromResult(0); 22 | private static readonly Task NoExceptionResult = Task.FromResult(null); 23 | private static readonly Task NoHttpResponse = Task.FromResult(null); 24 | private static readonly Task NoLatency = Task.FromResult(TimeSpan.Zero); 25 | 26 | /// 27 | /// Add chaos-injection policies to every policy returning 28 | /// in the supplied 29 | /// 30 | /// The whose policies should be decorated with chaos policies. 31 | /// The policy registry. 32 | public static IPolicyRegistry AddChaosInjectors(this IPolicyRegistry registry) 33 | { 34 | foreach (KeyValuePair policyEntry in registry) 35 | { 36 | if (policyEntry.Value is IAsyncPolicy policy) 37 | { 38 | registry[policyEntry.Key] = policy 39 | .WrapAsync(MonkeyPolicy.InjectExceptionAsync(with => 40 | with.Fault(GetException) 41 | .InjectionRate(GetInjectionRate) 42 | .EnabledWhen(GetEnabled))) 43 | .WrapAsync(MonkeyPolicy.InjectResultAsync(with => 44 | with.Result(GetHttpResponseMessage) 45 | .InjectionRate(GetInjectionRate) 46 | .EnabledWhen(GetHttpResponseEnabled))) 47 | .WrapAsync(MonkeyPolicy.InjectLatencyAsync(with => 48 | with.Latency(GetLatency) 49 | .InjectionRate(GetInjectionRate) 50 | .EnabledWhen(GetEnabled))); 51 | } 52 | } 53 | 54 | return registry; 55 | } 56 | 57 | private static Task GetEnabled(Context context, CancellationToken token) 58 | { 59 | OperationChaosSetting chaosSettings = context.GetOperationChaosSettings(); 60 | if (chaosSettings == null) return NotEnabled; 61 | 62 | return Task.FromResult(chaosSettings.Enabled); 63 | } 64 | 65 | private static Task GetInjectionRate(Context context, CancellationToken token) 66 | { 67 | OperationChaosSetting chaosSettings = context.GetOperationChaosSettings(); 68 | if (chaosSettings == null) return NoInjectionRate; 69 | 70 | return Task.FromResult(chaosSettings.InjectionRate); 71 | } 72 | 73 | private static Task GetException(Context context, CancellationToken token) 74 | { 75 | OperationChaosSetting chaosSettings = context.GetOperationChaosSettings(); 76 | if (chaosSettings == null) return NoExceptionResult; 77 | 78 | string exceptionName = chaosSettings.Exception; 79 | if (String.IsNullOrWhiteSpace(exceptionName)) return NoExceptionResult; 80 | 81 | try 82 | { 83 | Type exceptionType = Type.GetType(exceptionName); 84 | if (exceptionType == null) return NoExceptionResult; 85 | 86 | if (!typeof(Exception).IsAssignableFrom(exceptionType)) return NoExceptionResult; 87 | 88 | var instance = Activator.CreateInstance(exceptionType); 89 | return Task.FromResult(instance as Exception); 90 | } 91 | catch 92 | { 93 | return NoExceptionResult; 94 | } 95 | } 96 | 97 | private static Task GetHttpResponseEnabled(Context context, CancellationToken token) 98 | { 99 | if (GetHttpResponseMessage(context, CancellationToken.None) == NoHttpResponse) return NotEnabled; 100 | 101 | return GetEnabled(context, token); 102 | } 103 | 104 | private static Task GetHttpResponseMessage(Context context, CancellationToken token) 105 | { 106 | OperationChaosSetting chaosSettings = context.GetOperationChaosSettings(); 107 | if (chaosSettings == null) return NoHttpResponse; 108 | 109 | int statusCode = chaosSettings.StatusCode; 110 | if (statusCode < 200) return NoHttpResponse; 111 | 112 | try 113 | { 114 | return Task.FromResult(new HttpResponseMessage((HttpStatusCode) statusCode)); 115 | } 116 | catch 117 | { 118 | return NoHttpResponse; 119 | } 120 | } 121 | 122 | private static Task GetLatency(Context context, CancellationToken token) 123 | { 124 | OperationChaosSetting chaosSettings = context.GetOperationChaosSettings(); 125 | if (chaosSettings == null) return NoLatency; 126 | 127 | int milliseconds = chaosSettings.LatencyMs; 128 | if (milliseconds <= 0) return NoLatency; 129 | 130 | return Task.FromResult(TimeSpan.FromMilliseconds(milliseconds)); 131 | } 132 | 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /SimmyDemo_WebApi/Controllers/MonitoringController.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using Microsoft.AspNetCore.Mvc; 3 | using Microsoft.Extensions.Options; 4 | using Polly; 5 | using SimmyDemo_WebApi.Chaos; 6 | 7 | namespace SimmyDemo_WebApi.Controllers 8 | { 9 | [Route("api/[controller]/[action]")] 10 | [ApiController] 11 | public class MonitoringController : ControllerBase 12 | { 13 | private readonly MonitoringSettings monitoringSettings; 14 | private readonly AppChaosSettings chaosSettings; 15 | private readonly ResilientHttpClient client; 16 | 17 | public MonitoringController(ResilientHttpClient client, IOptions monitoringOptions, IOptionsSnapshot chaosOptionsSnapshot) 18 | { 19 | this.client = client; 20 | monitoringSettings = monitoringOptions.Value; 21 | chaosSettings = chaosOptionsSnapshot.Value; 22 | } 23 | 24 | [HttpGet] 25 | public async Task> Status() 26 | { 27 | Context context = new Context(nameof(Status)).WithChaosSettings(chaosSettings); 28 | 29 | return await client.GetStatus(monitoringSettings, context); 30 | } 31 | 32 | [HttpGet] 33 | public async Task> ResponseTime() 34 | { 35 | Context context = new Context(nameof(ResponseTime)).WithChaosSettings(chaosSettings); 36 | 37 | return await client.GetResponseReadTimeMs(monitoringSettings, context); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /SimmyDemo_WebApi/EndpointResult.cs: -------------------------------------------------------------------------------- 1 | namespace SimmyDemo_WebApi 2 | { 3 | public class EndpointResult 4 | { 5 | public string Url { get; set; } 6 | public long Value { get; set; } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /SimmyDemo_WebApi/MonitoringResults.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace SimmyDemo_WebApi 4 | { 5 | public class MonitoringResults 6 | { 7 | public IList Results { get; set; } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /SimmyDemo_WebApi/MonitoringSettings.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace SimmyDemo_WebApi 4 | { 5 | public class MonitoringSettings 6 | { 7 | public List Endpoints { get; set; } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /SimmyDemo_WebApi/Program.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.AspNetCore; 2 | using Microsoft.AspNetCore.Hosting; 3 | 4 | namespace SimmyDemo_WebApi 5 | { 6 | public class Program 7 | { 8 | public static void Main(string[] args) 9 | { 10 | CreateWebHostBuilder(args).Build().Run(); 11 | } 12 | 13 | public static IWebHostBuilder CreateWebHostBuilder(string[] args) => 14 | WebHost.CreateDefaultBuilder(args) 15 | .UseStartup(); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /SimmyDemo_WebApi/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "iisSettings": { 3 | "windowsAuthentication": false, 4 | "anonymousAuthentication": true, 5 | "iisExpress": { 6 | "applicationUrl": "http://localhost:57569", 7 | "sslPort": 44383 8 | } 9 | }, 10 | "$schema": "http://json.schemastore.org/launchsettings.json", 11 | "profiles": { 12 | "IIS Express": { 13 | "commandName": "IISExpress", 14 | "launchBrowser": true, 15 | "launchUrl": "api/monitoring/status", 16 | "environmentVariables": { 17 | "ASPNETCORE_ENVIRONMENT": "Development" 18 | } 19 | }, 20 | "SimmyDemo_WebApi": { 21 | "commandName": "Project", 22 | "launchBrowser": true, 23 | "launchUrl": "api/values", 24 | "environmentVariables": { 25 | "ASPNETCORE_ENVIRONMENT": "Development" 26 | }, 27 | "applicationUrl": "https://localhost:5001;http://localhost:5000" 28 | } 29 | } 30 | } -------------------------------------------------------------------------------- /SimmyDemo_WebApi/ResilientHttpClient.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.Linq; 5 | using System.Net.Http; 6 | using System.Threading.Tasks; 7 | using Polly; 8 | 9 | namespace SimmyDemo_WebApi 10 | { 11 | public class ResilientHttpClient 12 | { 13 | private readonly HttpClient client; 14 | 15 | public ResilientHttpClient(HttpClient client) 16 | { 17 | this.client = client; 18 | } 19 | 20 | public async Task GetStatus(MonitoringSettings settings, Context context) 21 | { 22 | MonitoringResults results = new MonitoringResults{Results = new List()}; 23 | 24 | // In a real app, would use a Task.WhenAll() fanout pattern. 25 | foreach (var endpoint in settings.Endpoints) 26 | { 27 | var response = await GetAsyncUsingContext(endpoint, context); 28 | results.Results.Add(new EndpointResult(){Url = endpoint, Value = (int)response.StatusCode}); 29 | } 30 | 31 | return results; 32 | } 33 | 34 | public async Task GetResponseReadTimeMs(MonitoringSettings settings, Context context) 35 | { 36 | MonitoringResults results = new MonitoringResults { Results = new List() }; 37 | 38 | // In a real app, would use a Task.WhenAll() fanout pattern. 39 | foreach (var endpoint in settings.Endpoints) 40 | { 41 | var watch = Stopwatch.StartNew(); 42 | 43 | var response = await GetAsyncUsingContext(endpoint, context); 44 | 45 | // Returning the response read time should throw if - after all resilience attempts - we can't read and time a valid response. 46 | response.EnsureSuccessStatusCode(); 47 | 48 | var contentActuallyOfNoInterest = await (response.Content?.ReadAsStringAsync()??Task.FromResult(String.Empty)); 49 | 50 | results.Results.Add(new EndpointResult() { Url = endpoint, Value = watch.ElapsedMilliseconds }); 51 | } 52 | 53 | return results; 54 | } 55 | 56 | private async Task GetAsyncUsingContext(string url, Context context) 57 | { 58 | // This will include configured Polly resilience policies; and Simmy chaos policies in dev environments. 59 | // - Polly resilience policies were configured in StartUp 60 | // - A call to .AddChaosInjectors() added chaos policies to all policies in the registry, during startup, for dev environments. 61 | 62 | // We attach the Polly context to the HttpRequestMessage using an extension method provided by HttpClientFactory. 63 | HttpRequestMessage message = new HttpRequestMessage(HttpMethod.Get, url); 64 | message.SetPolicyExecutionContext(context); 65 | 66 | // Make the request using the client configured by HttpClientFactory, which embeds the Polly and Simmy policies. 67 | return await client.SendAsync(message); 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /SimmyDemo_WebApi/SimmyDemo_WebApi.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp3.1 5 | InProcess 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /SimmyDemo_WebApi/Startup.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Net.Http; 3 | using Microsoft.AspNetCore.Builder; 4 | using Microsoft.AspNetCore.Hosting; 5 | using Microsoft.AspNetCore.Mvc; 6 | using Microsoft.Extensions.Configuration; 7 | using Microsoft.Extensions.DependencyInjection; 8 | using Microsoft.Extensions.Hosting; 9 | using Polly; 10 | using Polly.Extensions.Http; 11 | using Polly.Registry; 12 | using SimmyDemo_WebApi.Chaos; 13 | 14 | namespace SimmyDemo_WebApi 15 | { 16 | public class Startup 17 | { 18 | public const string ResiliencePolicy = "ResiliencePolicy"; 19 | 20 | public Startup(IConfiguration configuration) 21 | { 22 | Configuration = configuration; 23 | } 24 | 25 | public IConfiguration Configuration { get; } 26 | 27 | public void ConfigureServices(IServiceCollection services) 28 | { 29 | services.AddMvc() 30 | .AddMvcOptions(options => options.EnableEndpointRouting = false) 31 | .SetCompatibilityVersion(CompatibilityVersion.Version_3_0); 32 | 33 | // Read the endpoints we expect the service to monitor. 34 | services.Configure(Configuration.GetSection("MonitoringEndpoints")); 35 | 36 | // Create (and register with DI) a policy registry containing some policies we want to use. 37 | services.AddPolicyRegistry(new PolicyRegistry 38 | { 39 | { ResiliencePolicy, GetResiliencePolicy() } 40 | }); 41 | 42 | // Register a typed client via HttpClientFactory, set to use the policy we placed in the policy registry. 43 | services.AddHttpClient(client => 44 | { 45 | client.Timeout = TimeSpan.FromSeconds(5); 46 | }) 47 | .AddPolicyHandlerFromRegistry(ResiliencePolicy); 48 | 49 | // Add ability for the app to populate ChaosSettings from json file (or any other .NET Core configuration source) 50 | services.Configure(Configuration.GetSection("ChaosSettings")); 51 | } 52 | 53 | public void Configure(IApplicationBuilder app, IWebHostEnvironment env) 54 | { 55 | if (env.IsDevelopment()) 56 | { 57 | app.UseDeveloperExceptionPage(); 58 | } 59 | else 60 | { 61 | // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts. 62 | app.UseHsts(); 63 | } 64 | 65 | // Only add Simmy chaos injection in development-environment runs (ie prevent chaos-injection ever reaching staging or prod - if that is what you want). 66 | if (env.IsDevelopment()) 67 | { 68 | // Wrap every policy in the policy registry in Simmy chaos injectors. 69 | var registry = app.ApplicationServices.GetRequiredService>(); 70 | registry?.AddChaosInjectors(); 71 | } 72 | 73 | app.UseHttpsRedirection(); 74 | app.UseMvc(); 75 | } 76 | 77 | private IAsyncPolicy GetResiliencePolicy() 78 | { 79 | // Define a policy which will form our resilience strategy. These could be anything. The settings for them could obviously be drawn from config too. 80 | var retry = HttpPolicyExtensions.HandleTransientHttpError() 81 | .RetryAsync(2); 82 | 83 | return retry; 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /SimmyDemo_WebApi/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Debug", 5 | "System": "Information", 6 | "Microsoft": "Information" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /SimmyDemo_WebApi/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Warning" 5 | } 6 | }, 7 | "AllowedHosts": "*", 8 | "MonitoringEndpoints": { 9 | "Endpoints": [ 10 | "http://www.google.co.uk/", 11 | "https://github.com/Polly-Contrib/Simmy/" 12 | ] 13 | }, 14 | "ChaosSettings": { 15 | "OperationChaosSettings": [ 16 | { 17 | "OperationKey": "Status", 18 | "Enabled": true, 19 | "InjectionRate": 0.75, 20 | "LatencyMs": 0, 21 | "StatusCode": 503, 22 | "Exception": "System.SetToAnExceptionTypeWhichExistsAndItWillInject" 23 | }, 24 | { 25 | "OperationKey": "ResponseTime", 26 | "Enabled": true, 27 | "InjectionRate": 0.5, 28 | "LatencyMs": 2000, 29 | "Exception": "System.OperationCanceledException" 30 | } 31 | ] 32 | } 33 | } 34 | --------------------------------------------------------------------------------