├── .gitignore ├── LICENSE ├── PipelineImplementations.sln └── PipelineImplementations ├── App.config ├── Part1 ├── BlockingCollection │ ├── CastingPipeline.cs │ ├── CastingPipelineWithAwait.cs │ ├── CastingPipelineWithMaxCapacity.cs │ ├── CastingPipelineWithParallelism.cs │ ├── GenericBCPipeline.cs │ ├── GenericBCPipelineAwait.cs │ ├── IPipeline.cs │ ├── InnerCastingPipeline.cs │ └── UsagePart1.cs └── MyPipeline.cs ├── Part2 ├── TPLDataflow │ ├── TPLDataflowPipelineSimple.cs │ ├── TPLDataflowPipelineSimpleWithOptionalRequirements.cs │ ├── TPLDataflowPipelineWithAwaitAttempt1.cs │ ├── TPLPipelineWithAwaitAttempt2.cs │ └── TPLPipelineWithAwaitFinal.cs └── UsagePart2.cs ├── Part3 ├── Disruptor │ ├── DisruptorExample.cs │ ├── DisruptorSimple.cs │ └── DisruptorSimpleAwaitable.cs ├── TPLDataflowWithAsync │ ├── TPLDataflowSteppedAsync.cs │ ├── TPLDataflowSteppedAsyncFinal2.cs │ └── TPLDataflowSteppedSimple.cs └── UsagePart3.cs ├── PartN ├── IPipeline.cs ├── IPipelineBuilder.cs ├── IPipelineBuilderStep.cs ├── Impl │ ├── DisruptorEvent.cs │ ├── DisruptorPipeline.cs │ └── DisruptorPipelineBuilder.cs └── UsagePartN.cs ├── PipelineImplementations.csproj ├── Program.cs ├── Properties └── AssemblyInfo.cs ├── Utils.cs └── packages.config /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.suo 8 | *.user 9 | *.userosscache 10 | *.sln.docstates 11 | 12 | # User-specific files (MonoDevelop/Xamarin Studio) 13 | *.userprefs 14 | 15 | # Build results 16 | [Dd]ebug/ 17 | [Dd]ebugPublic/ 18 | [Rr]elease/ 19 | [Rr]eleases/ 20 | x64/ 21 | x86/ 22 | bld/ 23 | [Bb]in/ 24 | [Oo]bj/ 25 | [Ll]og/ 26 | 27 | # Visual Studio 2015/2017 cache/options directory 28 | .vs/ 29 | # Uncomment if you have tasks that create the project's static files in wwwroot 30 | #wwwroot/ 31 | 32 | # Visual Studio 2017 auto generated files 33 | Generated\ Files/ 34 | 35 | # MSTest test Results 36 | [Tt]est[Rr]esult*/ 37 | [Bb]uild[Ll]og.* 38 | 39 | # NUNIT 40 | *.VisualState.xml 41 | TestResult.xml 42 | 43 | # Build Results of an ATL Project 44 | [Dd]ebugPS/ 45 | [Rr]eleasePS/ 46 | dlldata.c 47 | 48 | # Benchmark Results 49 | BenchmarkDotNet.Artifacts/ 50 | 51 | # .NET Core 52 | project.lock.json 53 | project.fragment.lock.json 54 | artifacts/ 55 | **/Properties/launchSettings.json 56 | 57 | # StyleCop 58 | StyleCopReport.xml 59 | 60 | # Files built by Visual Studio 61 | *_i.c 62 | *_p.c 63 | *_i.h 64 | *.ilk 65 | *.meta 66 | *.obj 67 | *.iobj 68 | *.pch 69 | *.pdb 70 | *.ipdb 71 | *.pgc 72 | *.pgd 73 | *.rsp 74 | *.sbr 75 | *.tlb 76 | *.tli 77 | *.tlh 78 | *.tmp 79 | *.tmp_proj 80 | *.log 81 | *.vspscc 82 | *.vssscc 83 | .builds 84 | *.pidb 85 | *.svclog 86 | *.scc 87 | 88 | # Chutzpah Test files 89 | _Chutzpah* 90 | 91 | # Visual C++ cache files 92 | ipch/ 93 | *.aps 94 | *.ncb 95 | *.opendb 96 | *.opensdf 97 | *.sdf 98 | *.cachefile 99 | *.VC.db 100 | *.VC.VC.opendb 101 | 102 | # Visual Studio profiler 103 | *.psess 104 | *.vsp 105 | *.vspx 106 | *.sap 107 | 108 | # Visual Studio Trace Files 109 | *.e2e 110 | 111 | # TFS 2012 Local Workspace 112 | $tf/ 113 | 114 | # Guidance Automation Toolkit 115 | *.gpState 116 | 117 | # ReSharper is a .NET coding add-in 118 | _ReSharper*/ 119 | *.[Rr]e[Ss]harper 120 | *.DotSettings.user 121 | 122 | # JustCode is a .NET coding add-in 123 | .JustCode 124 | 125 | # TeamCity is a build add-in 126 | _TeamCity* 127 | 128 | # DotCover is a Code Coverage Tool 129 | *.dotCover 130 | 131 | # AxoCover is a Code Coverage Tool 132 | .axoCover/* 133 | !.axoCover/settings.json 134 | 135 | # Visual Studio code coverage results 136 | *.coverage 137 | *.coveragexml 138 | 139 | # NCrunch 140 | _NCrunch_* 141 | .*crunch*.local.xml 142 | nCrunchTemp_* 143 | 144 | # MightyMoose 145 | *.mm.* 146 | AutoTest.Net/ 147 | 148 | # Web workbench (sass) 149 | .sass-cache/ 150 | 151 | # Installshield output folder 152 | [Ee]xpress/ 153 | 154 | # DocProject is a documentation generator add-in 155 | DocProject/buildhelp/ 156 | DocProject/Help/*.HxT 157 | DocProject/Help/*.HxC 158 | DocProject/Help/*.hhc 159 | DocProject/Help/*.hhk 160 | DocProject/Help/*.hhp 161 | DocProject/Help/Html2 162 | DocProject/Help/html 163 | 164 | # Click-Once directory 165 | publish/ 166 | 167 | # Publish Web Output 168 | *.[Pp]ublish.xml 169 | *.azurePubxml 170 | # Note: Comment the next line if you want to checkin your web deploy settings, 171 | # but database connection strings (with potential passwords) will be unencrypted 172 | *.pubxml 173 | *.publishproj 174 | 175 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 176 | # checkin your Azure Web App publish settings, but sensitive information contained 177 | # in these scripts will be unencrypted 178 | PublishScripts/ 179 | 180 | # NuGet Packages 181 | *.nupkg 182 | # The packages folder can be ignored because of Package Restore 183 | **/[Pp]ackages/* 184 | # except build/, which is used as an MSBuild target. 185 | !**/[Pp]ackages/build/ 186 | # Uncomment if necessary however generally it will be regenerated when needed 187 | #!**/[Pp]ackages/repositories.config 188 | # NuGet v3's project.json files produces more ignorable files 189 | *.nuget.props 190 | *.nuget.targets 191 | 192 | # Microsoft Azure Build Output 193 | csx/ 194 | *.build.csdef 195 | 196 | # Microsoft Azure Emulator 197 | ecf/ 198 | rcf/ 199 | 200 | # Windows Store app package directories and files 201 | AppPackages/ 202 | BundleArtifacts/ 203 | Package.StoreAssociation.xml 204 | _pkginfo.txt 205 | *.appx 206 | 207 | # Visual Studio cache files 208 | # files ending in .cache can be ignored 209 | *.[Cc]ache 210 | # but keep track of directories ending in .cache 211 | !*.[Cc]ache/ 212 | 213 | # Others 214 | ClientBin/ 215 | ~$* 216 | *~ 217 | *.dbmdl 218 | *.dbproj.schemaview 219 | *.jfm 220 | *.pfx 221 | *.publishsettings 222 | orleans.codegen.cs 223 | 224 | # Including strong name files can present a security risk 225 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 226 | #*.snk 227 | 228 | # Since there are multiple workflows, uncomment next line to ignore bower_components 229 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 230 | #bower_components/ 231 | 232 | # RIA/Silverlight projects 233 | Generated_Code/ 234 | 235 | # Backup & report files from converting an old project file 236 | # to a newer Visual Studio version. Backup files are not needed, 237 | # because we have git ;-) 238 | _UpgradeReport_Files/ 239 | Backup*/ 240 | UpgradeLog*.XML 241 | UpgradeLog*.htm 242 | ServiceFabricBackup/ 243 | *.rptproj.bak 244 | 245 | # SQL Server files 246 | *.mdf 247 | *.ldf 248 | *.ndf 249 | 250 | # Business Intelligence projects 251 | *.rdl.data 252 | *.bim.layout 253 | *.bim_*.settings 254 | *.rptproj.rsuser 255 | 256 | # Microsoft Fakes 257 | FakesAssemblies/ 258 | 259 | # GhostDoc plugin setting file 260 | *.GhostDoc.xml 261 | 262 | # Node.js Tools for Visual Studio 263 | .ntvs_analysis.dat 264 | node_modules/ 265 | 266 | # Visual Studio 6 build log 267 | *.plg 268 | 269 | # Visual Studio 6 workspace options file 270 | *.opt 271 | 272 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 273 | *.vbw 274 | 275 | # Visual Studio LightSwitch build output 276 | **/*.HTMLClient/GeneratedArtifacts 277 | **/*.DesktopClient/GeneratedArtifacts 278 | **/*.DesktopClient/ModelManifest.xml 279 | **/*.Server/GeneratedArtifacts 280 | **/*.Server/ModelManifest.xml 281 | _Pvt_Extensions 282 | 283 | # Paket dependency manager 284 | .paket/paket.exe 285 | paket-files/ 286 | 287 | # FAKE - F# Make 288 | .fake/ 289 | 290 | # JetBrains Rider 291 | .idea/ 292 | *.sln.iml 293 | 294 | # CodeRush 295 | .cr/ 296 | 297 | # Python Tools for Visual Studio (PTVS) 298 | __pycache__/ 299 | *.pyc 300 | 301 | # Cake - Uncomment if you are using it 302 | # tools/** 303 | # !tools/packages.config 304 | 305 | # Tabs Studio 306 | *.tss 307 | 308 | # Telerik's JustMock configuration file 309 | *.jmconfig 310 | 311 | # BizTalk build output 312 | *.btp.cs 313 | *.btm.cs 314 | *.odx.cs 315 | *.xsd.cs 316 | 317 | # OpenCover UI analysis results 318 | OpenCover/ 319 | 320 | # Azure Stream Analytics local run output 321 | ASALocalRun/ 322 | 323 | # MSBuild Binary and Structured Log 324 | *.binlog 325 | 326 | # NVidia Nsight GPU debugger configuration file 327 | *.nvuser 328 | 329 | # MFractors (Xamarin productivity tool) working folder 330 | .mfractor/ 331 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Michael Shpilt 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /PipelineImplementations.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.28307.705 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PipelineImplementations", "PipelineImplementations\PipelineImplementations.csproj", "{0EA3DBCB-2E0F-4670-861F-871D4626AB84}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Release|Any CPU = Release|Any CPU 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {0EA3DBCB-2E0F-4670-861F-871D4626AB84}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 15 | {0EA3DBCB-2E0F-4670-861F-871D4626AB84}.Debug|Any CPU.Build.0 = Debug|Any CPU 16 | {0EA3DBCB-2E0F-4670-861F-871D4626AB84}.Release|Any CPU.ActiveCfg = Release|Any CPU 17 | {0EA3DBCB-2E0F-4670-861F-871D4626AB84}.Release|Any CPU.Build.0 = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(SolutionProperties) = preSolution 20 | HideSolutionNode = FALSE 21 | EndGlobalSection 22 | GlobalSection(ExtensibilityGlobals) = postSolution 23 | SolutionGuid = {E18F1B21-6232-49B0-8F2F-37481BF860E3} 24 | EndGlobalSection 25 | EndGlobal 26 | -------------------------------------------------------------------------------- /PipelineImplementations/App.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /PipelineImplementations/Part1/BlockingCollection/CastingPipeline.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | 9 | namespace PipelineImplementations.Part1.BlockingCollection 10 | { 11 | public class CastingPipelineBuilder : IPipeline 12 | { 13 | List> _pipelineSteps = new List>(); 14 | BlockingCollection[] _buffers; 15 | 16 | public event Action Finished; 17 | 18 | public void AddStep(Func stepFunc) 19 | { 20 | _pipelineSteps.Add(stepFunc); 21 | } 22 | 23 | public void Execute(object input) 24 | { 25 | var first = _buffers[0]; 26 | first.Add(input); 27 | } 28 | 29 | public IPipeline GetPipeline() 30 | { 31 | _buffers = _pipelineSteps.Select(step => new BlockingCollection()).ToArray(); 32 | 33 | int bufferIndex = 0; 34 | foreach (var pipelineStep in _pipelineSteps) 35 | { 36 | var bufferIndexLocal = bufferIndex; 37 | Task.Run(() => 38 | { 39 | foreach (var input in _buffers[bufferIndexLocal].GetConsumingEnumerable()) 40 | { 41 | var output = pipelineStep.Invoke(input); 42 | 43 | bool isLastStep = bufferIndexLocal == _pipelineSteps.Count - 1; 44 | if (isLastStep) 45 | { 46 | // This is dangerous as the invocation is added to the last step 47 | // Alternatively, you can utilize 'BeginInvoke' like here: https://stackoverflow.com/a/16336361/1229063 48 | Finished?.Invoke(output); 49 | } 50 | else 51 | { 52 | var next = _buffers[bufferIndexLocal + 1]; 53 | next.Add(output); 54 | } 55 | } 56 | }); 57 | bufferIndex++; 58 | } 59 | return this; 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /PipelineImplementations/Part1/BlockingCollection/CastingPipelineWithAwait.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | 9 | namespace PipelineImplementations.Part1.BlockingCollection 10 | { 11 | 12 | public class CastingPipelineWithAwait : IAwaitablePipeline 13 | { 14 | class Step 15 | { 16 | public Func Func { get; set; } 17 | public int DegreeOfParallelism { get; set; } 18 | public int MaxCapacity { get; set; } 19 | } 20 | 21 | class Item 22 | { 23 | public object Input { get; set; } 24 | public TaskCompletionSource TaskCompletionSource { get; set; } 25 | } 26 | 27 | List _pipelineSteps = new List(); 28 | BlockingCollection[] _buffers; 29 | 30 | public event Action Finished; 31 | 32 | public void AddStep(Func stepFunc, int degreeOfParallelism, int maxCapacity) 33 | { 34 | _pipelineSteps.Add(new Step() {Func = stepFunc, DegreeOfParallelism = degreeOfParallelism, 35 | MaxCapacity = maxCapacity, }); 36 | } 37 | 38 | public Task Execute(object input) 39 | { 40 | var first = _buffers[0]; 41 | var item = new Item() 42 | { 43 | Input = input, 44 | TaskCompletionSource = new TaskCompletionSource() 45 | }; 46 | first.Add(item); 47 | return item.TaskCompletionSource.Task; 48 | } 49 | 50 | public IAwaitablePipeline GetPipeline() 51 | { 52 | _buffers = _pipelineSteps.Select(step => new BlockingCollection()).ToArray(); 53 | 54 | int bufferIndex = 0; 55 | foreach (var pipelineStep in _pipelineSteps) 56 | { 57 | var bufferIndexLocal = bufferIndex; 58 | 59 | for (int i = 0; i < pipelineStep.DegreeOfParallelism; i++) 60 | { 61 | Task.Run(() => { StartStep(bufferIndexLocal, pipelineStep); }); 62 | } 63 | 64 | bufferIndex++; 65 | } 66 | return this; 67 | } 68 | 69 | private void StartStep(int bufferIndexLocal, Step pipelineStep) 70 | { 71 | foreach (var input in _buffers[bufferIndexLocal].GetConsumingEnumerable()) 72 | { 73 | object output; 74 | try 75 | { 76 | output = pipelineStep.Func.Invoke(input.Input); 77 | } 78 | catch (Exception e) 79 | { 80 | input.TaskCompletionSource.SetException(e); 81 | continue; 82 | } 83 | 84 | bool isLastStep = bufferIndexLocal == _pipelineSteps.Count - 1; 85 | if (isLastStep) 86 | { 87 | input.TaskCompletionSource.SetResult((TOutput)(object)output); 88 | } 89 | else 90 | { 91 | var next = _buffers[bufferIndexLocal + 1]; 92 | next.Add(new Item() { Input = output, TaskCompletionSource = input.TaskCompletionSource}); 93 | } 94 | } 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /PipelineImplementations/Part1/BlockingCollection/CastingPipelineWithMaxCapacity.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | 9 | namespace PipelineImplementations.Part1.BlockingCollection 10 | { 11 | 12 | public class CastingPipelineWithMaxCapacity : IPipeline 13 | { 14 | class StepInfo 15 | { 16 | public Func Func { get; set; } 17 | public int DegreeOfParallelism { get; set; } 18 | public int MaxCapacity { get; set; } 19 | } 20 | 21 | List _pipelineSteps = new List(); 22 | BlockingCollection[] _buffers; 23 | 24 | public event Action Finished; 25 | 26 | public void AddStep(Func stepFunc, int degreeOfParallelism, int maxCapacity) 27 | { 28 | _pipelineSteps.Add(new StepInfo() {Func = stepFunc, DegreeOfParallelism = degreeOfParallelism, MaxCapacity = maxCapacity}); 29 | } 30 | 31 | public void Execute(object input) 32 | { 33 | var first = _buffers[0]; 34 | first.Add(input); 35 | } 36 | 37 | public IPipeline GetPipeline() 38 | { 39 | _buffers = _pipelineSteps.Select(step => new BlockingCollection(step.MaxCapacity)).ToArray(); 40 | 41 | int bufferIndex = 0; 42 | foreach (var pipelineStep in _pipelineSteps) 43 | { 44 | var bufferIndexLocal = bufferIndex; 45 | 46 | for (int i = 0; i < pipelineStep.DegreeOfParallelism; i++) 47 | { 48 | Task.Run(() => { StartStep(bufferIndexLocal, pipelineStep); }); 49 | } 50 | 51 | bufferIndex++; 52 | } 53 | return this; 54 | } 55 | 56 | private void StartStep(int bufferIndexLocal, StepInfo pipelineStep) 57 | { 58 | foreach (var input in _buffers[bufferIndexLocal].GetConsumingEnumerable()) 59 | { 60 | var output = pipelineStep.Func.Invoke(input); 61 | bool isLastStep = bufferIndexLocal == _pipelineSteps.Count - 1; 62 | if (isLastStep) 63 | { 64 | // This is dangerous as the invocation is added to the last step 65 | // Alternatively, you can utilize 'BeginInvoke' like here: https://stackoverflow.com/a/16336361/1229063 66 | Finished?.Invoke(output); 67 | } 68 | else 69 | { 70 | var next = _buffers[bufferIndexLocal + 1]; 71 | next.Add(output); 72 | } 73 | } 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /PipelineImplementations/Part1/BlockingCollection/CastingPipelineWithParallelism.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | 9 | namespace PipelineImplementations.Part1.BlockingCollection 10 | { 11 | 12 | public class CastingPipelineWithParallelism : IPipeline 13 | { 14 | class StepInfo 15 | { 16 | public Func Func { get; set; } 17 | public int DegreeOfParallelism { get; set; } 18 | } 19 | 20 | List _pipelineSteps = new List(); 21 | BlockingCollection[] _buffers; 22 | 23 | public event Action Finished; 24 | 25 | public void AddStep(Func stepFunc, int degreeOfParallelism) 26 | { 27 | _pipelineSteps.Add(new StepInfo() { Func = stepFunc, DegreeOfParallelism = degreeOfParallelism }); 28 | } 29 | 30 | public void Execute(object input) 31 | { 32 | var first = _buffers[0]; 33 | first.Add(input); 34 | } 35 | 36 | public IPipeline GetPipeline() 37 | { 38 | _buffers = _pipelineSteps.Select(step => new BlockingCollection()).ToArray(); 39 | 40 | int bufferIndex = 0; 41 | foreach (var pipelineStep in _pipelineSteps) 42 | { 43 | var bufferIndexLocal = bufferIndex; 44 | 45 | for (int i = 0; i < pipelineStep.DegreeOfParallelism; i++) 46 | { 47 | Task.Run(() => { StartStep(bufferIndexLocal, pipelineStep); }); 48 | } 49 | 50 | bufferIndex++; 51 | } 52 | return this; 53 | } 54 | 55 | private void StartStep(int bufferIndexLocal, StepInfo pipelineStep) 56 | { 57 | foreach (var input in _buffers[bufferIndexLocal].GetConsumingEnumerable()) 58 | { 59 | var output = pipelineStep.Func.Invoke(input); 60 | bool isLastStep = bufferIndexLocal == _pipelineSteps.Count - 1; 61 | if (isLastStep) 62 | { 63 | // This is dangerous as the invocation is added to the last step 64 | // Alternatively, you can utilize 'BeginInvoke' like here: https://stackoverflow.com/a/16336361/1229063 65 | Finished?.Invoke(output); 66 | } 67 | else 68 | { 69 | var next = _buffers[bufferIndexLocal + 1]; 70 | next.Add(output); 71 | } 72 | } 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /PipelineImplementations/Part1/BlockingCollection/GenericBCPipeline.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | 9 | namespace PipelineImplementations.Part1.BlockingCollection 10 | { 11 | public interface IPipelineStep 12 | { 13 | BlockingCollection Buffer { get; set; } 14 | } 15 | 16 | public class GenericBCPipelineStep : IPipelineStep 17 | { 18 | public BlockingCollection Buffer { get; set; } = new BlockingCollection(); 19 | public Func StepAction { get; set; } 20 | } 21 | 22 | public static class GenericBCPipelineExtensions 23 | { 24 | 25 | public static TOutput Step(this TInput inputType, 26 | GenericBCPipeline pipelineBuilder, 27 | Func step) 28 | { 29 | var pipelineStep = pipelineBuilder.GenerateStep(); 30 | pipelineStep.StepAction = step; 31 | return default(TOutput); 32 | } 33 | } 34 | 35 | public class GenericBCPipeline 36 | { 37 | List _pipelineSteps = new List(); 38 | 39 | public event Action Finished; 40 | 41 | public GenericBCPipeline(Func, TPipeOut> steps) 42 | { 43 | steps.Invoke(default(TPipeIn), this);//Invoke just once to build blocking collections 44 | } 45 | 46 | public void Execute(TPipeIn input) 47 | { 48 | var first = _pipelineSteps[0] as IPipelineStep; 49 | first.Buffer.Add(input); 50 | } 51 | 52 | public GenericBCPipelineStep GenerateStep() 53 | { 54 | var pipelineStep = new GenericBCPipelineStep(); 55 | var stepIndex = _pipelineSteps.Count; 56 | 57 | Task.Run(() => 58 | { 59 | IPipelineStep nextPipelineStep = null; 60 | 61 | foreach (var input in pipelineStep.Buffer.GetConsumingEnumerable()) 62 | { 63 | bool isLastStep = stepIndex == _pipelineSteps.Count - 1; 64 | var output = pipelineStep.StepAction(input); 65 | if (isLastStep) 66 | { 67 | // This is dangerous as the invocation is added to the last step 68 | // Alternatively, you can utilize BeginInvoke like here: https://stackoverflow.com/a/16336361/1229063 69 | Finished?.Invoke((TPipeOut)(object)output); 70 | } 71 | else 72 | { 73 | nextPipelineStep = nextPipelineStep ?? (isLastStep ? null : _pipelineSteps[stepIndex + 1] as IPipelineStep); 74 | nextPipelineStep.Buffer.Add(output); 75 | } 76 | } 77 | }); 78 | 79 | _pipelineSteps.Add(pipelineStep); 80 | return pipelineStep; 81 | 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /PipelineImplementations/Part1/BlockingCollection/GenericBCPipelineAwait.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | 9 | namespace PipelineImplementations.Part1.BlockingCollection 10 | { 11 | public static class GenericBCPipelineAwaitExtensions 12 | { 13 | public static TOutput Step2(this TInput inputType, 14 | GenericBCPipelineAwait pipelineBuilder, 15 | Func step) 16 | { 17 | var pipelineStep = pipelineBuilder.GenerateStep(); 18 | pipelineStep.StepAction = step; 19 | return default(TOutput); 20 | } 21 | } 22 | 23 | public class GenericBCPipelineAwait 24 | { 25 | public interface IPipelineAwaitStep 26 | { 27 | BlockingCollection> Buffer { get; set; } 28 | } 29 | 30 | public class GenericBCPipelineAwaitStep : IPipelineAwaitStep 31 | { 32 | public BlockingCollection> Buffer { get; set; } = new BlockingCollection>(); 33 | public Func StepAction { get; set; } 34 | } 35 | 36 | public class Item 37 | { 38 | public T Input { get; set; } 39 | public TaskCompletionSource TaskCompletionSource { get; set; } 40 | } 41 | 42 | List _pipelineSteps = new List(); 43 | 44 | 45 | public GenericBCPipelineAwait(Func, TPipeOut> steps) 46 | { 47 | steps.Invoke(default(TPipeIn), this);//Invoke just once to build blocking collections 48 | } 49 | 50 | public Task Execute(TPipeIn input) 51 | { 52 | var first = _pipelineSteps[0] as IPipelineAwaitStep; 53 | TaskCompletionSource tsk = new TaskCompletionSource(); 54 | first.Buffer.Add(/*input*/new Item() 55 | { 56 | Input = input, 57 | TaskCompletionSource = tsk 58 | }); 59 | return tsk.Task; 60 | } 61 | 62 | public GenericBCPipelineAwaitStep GenerateStep() 63 | { 64 | var pipelineStep = new GenericBCPipelineAwaitStep(); 65 | var stepIndex = _pipelineSteps.Count; 66 | 67 | Task.Run(() => 68 | { 69 | IPipelineAwaitStep nextPipelineStep = null; 70 | 71 | foreach (var input in pipelineStep.Buffer.GetConsumingEnumerable()) 72 | { 73 | bool isLastStep = stepIndex == _pipelineSteps.Count - 1; 74 | TStepOut output; 75 | try 76 | { 77 | output = pipelineStep.StepAction(input.Input); 78 | } 79 | catch (Exception e) 80 | { 81 | input.TaskCompletionSource.SetException(e); 82 | continue; 83 | } 84 | if (isLastStep) 85 | { 86 | input.TaskCompletionSource.SetResult((TPipeOut)(object)output); 87 | } 88 | else 89 | { 90 | nextPipelineStep = nextPipelineStep ?? (isLastStep ? null : _pipelineSteps[stepIndex + 1] as IPipelineAwaitStep); 91 | nextPipelineStep.Buffer.Add(new Item() { Input = output, TaskCompletionSource = input.TaskCompletionSource }); 92 | } 93 | } 94 | }); 95 | 96 | _pipelineSteps.Add(pipelineStep); 97 | return pipelineStep; 98 | 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /PipelineImplementations/Part1/BlockingCollection/IPipeline.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | 4 | namespace PipelineImplementations.Part1.BlockingCollection 5 | { 6 | public interface IPipeline 7 | { 8 | void Execute(object input); 9 | event Action Finished; 10 | } 11 | 12 | public interface IAwaitablePipeline 13 | { 14 | Task Execute(object input); 15 | } 16 | } -------------------------------------------------------------------------------- /PipelineImplementations/Part1/BlockingCollection/InnerCastingPipeline.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | 9 | namespace PipelineImplementations.Part1.BlockingCollection 10 | { 11 | 12 | public class InnerPipelineBuilder : IPipeline 13 | { 14 | List> _pipelineSteps = new List>(); 15 | BlockingCollection[] _buffers; 16 | 17 | public event Action Finished; 18 | 19 | public void AddStep(Func stepFunc) 20 | { 21 | _pipelineSteps.Add(objInput => stepFunc.Invoke((TStepIn)(object)objInput)); 22 | } 23 | 24 | public void Execute(object input) 25 | { 26 | var first = _buffers[0]; 27 | first.Add(input); 28 | } 29 | 30 | public IPipeline GetPipeline() 31 | { 32 | _buffers = _pipelineSteps.Select(step => new BlockingCollection()).ToArray(); 33 | 34 | int bufferIndex = 0; 35 | foreach (var pipelineStep in _pipelineSteps) 36 | { 37 | var bufferIndexLocal = bufferIndex; 38 | Task.Run(() => 39 | { 40 | foreach (var input in _buffers[bufferIndexLocal].GetConsumingEnumerable()) 41 | { 42 | var output = pipelineStep.Invoke(input); 43 | 44 | bool isLastStep = bufferIndexLocal == _pipelineSteps.Count - 1; 45 | if (isLastStep) 46 | { 47 | // This is dangerous as the invocation is added to the last step 48 | // Alternatively, you can utilize 'BeginInvoke' like here: https://stackoverflow.com/a/16336361/1229063 49 | Finished?.Invoke(output); 50 | } 51 | else 52 | { 53 | var next = _buffers[bufferIndexLocal + 1]; 54 | next.Add(output); 55 | } 56 | } 57 | }); 58 | bufferIndex++; 59 | } 60 | return this; 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /PipelineImplementations/Part1/BlockingCollection/UsagePart1.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace PipelineImplementations.Part1.BlockingCollection 8 | { 9 | public class UsagePart1 10 | { 11 | public static void Use() 12 | { 13 | //GenericBCPipeline(); 14 | //var pipeline = CastingPipeline(); 15 | //var pipeline = InnerCastingPipeline(); 16 | //var pipeline = GenericBCPipeline(); 17 | //var pipeline = CastingPipelineWithParallelism(); 18 | var pipeline = CastingPipelineWithAwait(); 19 | //var pipeline = CreateGenericBCPipelineAwait(); 20 | 21 | var tsk = System.Threading.Tasks.Task.Run(async () => 22 | { 23 | Console.WriteLine(await pipeline.Execute("The pipeline pattern is the best pattern")); 24 | Console.WriteLine(await pipeline.Execute("The pipeline pattern is the best pattern")); 25 | Console.WriteLine(await pipeline.Execute("The pipeline pattern is the best pattern")); 26 | Console.WriteLine(await pipeline.Execute("The pipeline patter is the best patter")); 27 | Console.WriteLine(await pipeline.Execute("The pipeline pattern is the best pattern")); 28 | }); 29 | tsk.Wait(); 30 | 31 | //pipeline.Execute("The pipeline pattern is the best pattern"); 32 | //pipeline.Execute("The pipeline pattern is the best pattern"); 33 | //pipeline.Execute("The pipeline patter is the best patter"); 34 | //pipeline.Execute("The pipeline pattern is the best pattern"); 35 | 36 | //pipeline.Finished += res => Console.WriteLine(res); 37 | } 38 | 39 | private static GenericBCPipelineAwait CreateGenericBCPipelineAwait() 40 | { 41 | var pipeline = new GenericBCPipelineAwait((inputFirst, builder) => 42 | inputFirst.Step2(builder, input => FindMostCommon(input)) 43 | .Step2(builder, input => input.Length) 44 | .Step2(builder, input => input % 2 == 1)); 45 | return pipeline; 46 | } 47 | 48 | private static IAwaitablePipeline CastingPipelineWithAwait() 49 | { 50 | var builder = new CastingPipelineWithAwait(); 51 | builder.AddStep(input => FindMostCommon(input as string), 2, 10); 52 | builder.AddStep(input => (input as string).Length, 2, 10); 53 | builder.AddStep(input => ((int)input) % 2 == 1, 2, 10); 54 | var pipeline = builder.GetPipeline(); 55 | return pipeline; 56 | } 57 | 58 | private static IPipeline CastingPipelineWithParallelism() 59 | { 60 | var builder = new CastingPipelineWithParallelism(); 61 | builder.AddStep(input => FindMostCommon(input as string), 2); 62 | builder.AddStep(input => (input as string).Length, 2); 63 | builder.AddStep(input => ((int)input) % 2 == 1, 2); 64 | var pipeline = builder.GetPipeline(); 65 | return pipeline; 66 | } 67 | 68 | private static IPipeline CastingPipeline() 69 | { 70 | var builder = new CastingPipelineBuilder(); 71 | builder.AddStep(input => FindMostCommon(input as string)); 72 | builder.AddStep(input => (input as string).Length); 73 | builder.AddStep(input => ((int)input) % 2 == 1); 74 | var pipeline = builder.GetPipeline(); 75 | return pipeline; 76 | } 77 | 78 | private static IPipeline InnerCastingPipeline() 79 | { 80 | var builder = new InnerPipelineBuilder(); 81 | builder.AddStep(input => FindMostCommon(input)); 82 | builder.AddStep(input => CountChars(input)); 83 | builder.AddStep(input => IsOdd(input)); 84 | var pipeline = builder.GetPipeline(); 85 | return pipeline; 86 | } 87 | 88 | private static GenericBCPipeline GenericBCPipeline() 89 | { 90 | var pipeline = new GenericBCPipeline((inputFirst, builder) => 91 | inputFirst.Step(builder, input => FindMostCommon(input)) 92 | .Step(builder, input => input.Length) 93 | .Step(builder, input => input % 2 == 1)); 94 | return pipeline; 95 | 96 | //pipeline.Execute("The pipeline pattern is the best pattern"); 97 | //pipeline.Execute("The pipeline pattern is the best pattern"); 98 | //pipeline.Execute("The pipeline pattern is the best pattern"); 99 | //pipeline.Execute("The pipeline patter is the best patter"); 100 | //pipeline.Execute("The pipeline pattern is the best pattern"); 101 | 102 | //pipeline.Finished += res => Console.WriteLine(res); 103 | } 104 | 105 | private static string FindMostCommon(string input) 106 | { 107 | return input.Split(' ') 108 | .GroupBy(word => word) 109 | .OrderBy(group => group.Count()) 110 | .Last() 111 | .Key; 112 | } 113 | 114 | private static int CountChars(string mostCommon) 115 | { 116 | return mostCommon.Length; 117 | } 118 | 119 | private static bool IsOdd(int number) 120 | { 121 | var res = number % 2 == 1; 122 | //Console.WriteLine(res.ToString()); 123 | return res; 124 | } 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /PipelineImplementations/Part1/MyPipeline.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace PipelineImplementations.Part1 8 | { 9 | public class MyPipeline 10 | { 11 | public bool Execute(string input) 12 | { 13 | string mostCommon = FindMostCommon(input); 14 | int characters = CountChars(mostCommon); 15 | bool isOdd = IsOdd(characters); 16 | return isOdd; 17 | } 18 | 19 | private string FindMostCommon(string input) 20 | { 21 | return input.Split(' ') 22 | .GroupBy(word => word) 23 | .OrderBy(group => group.Count()) 24 | .Last() 25 | .Key; 26 | } 27 | 28 | private int CountChars(string mostCommon) 29 | { 30 | return mostCommon.Length; 31 | } 32 | 33 | private bool IsOdd(int number) 34 | { 35 | return number % 2 == 1; 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /PipelineImplementations/Part2/TPLDataflow/TPLDataflowPipelineSimple.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using System.Threading.Tasks.Dataflow; 7 | 8 | namespace PipelineImplementations.Part2.TPLDataflow 9 | { 10 | public class TPLDataflowPipelineSimple 11 | { 12 | //private ActionBlock step1; 13 | public static TransformBlock CreatePipeline(Action resultCallback) 14 | { 15 | var step1 = new TransformBlock((sentence) => FindMostCommon(sentence)); 16 | var step2 = new TransformBlock((word) => word.Length); 17 | var step3 = new TransformBlock((length) => 18 | { 19 | Console.WriteLine(Utils.GetThreadPoolThreadsInUse()); 20 | return length % 2 == 1; 21 | }); 22 | var callBackStep = new ActionBlock(resultCallback); 23 | step1.LinkTo(step2, new DataflowLinkOptions()); 24 | step2.LinkTo(step3, new DataflowLinkOptions()); 25 | step3.LinkTo(callBackStep); 26 | return step1; 27 | } 28 | 29 | private static string FindMostCommon(string input) 30 | { 31 | return input.Split(' ') 32 | .GroupBy(word => word) 33 | .OrderBy(group => group.Count()) 34 | .Last() 35 | .Key; 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /PipelineImplementations/Part2/TPLDataflow/TPLDataflowPipelineSimpleWithOptionalRequirements.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using System.Threading.Tasks.Dataflow; 7 | 8 | namespace PipelineImplementations.Part2.TPLDataflow 9 | { 10 | public class TPLDataflowPipelineSimpleWithOptionalRequirements 11 | { 12 | //private ActionBlock step1; 13 | public static TransformBlock CreatePipeline(Action resultCallback) 14 | { 15 | var step1 = new TransformBlock((sentence) => FindMostCommon(sentence), 16 | new ExecutionDataflowBlockOptions() 17 | { 18 | MaxDegreeOfParallelism = 3, 19 | BoundedCapacity = 5, 20 | }); 21 | var step2 = new TransformBlock((word) => word.Length, 22 | new ExecutionDataflowBlockOptions() 23 | { 24 | MaxDegreeOfParallelism = 1, 25 | BoundedCapacity = 13, 26 | }); 27 | var step3 = new TransformBlock((length) =>length % 2 == 1, 28 | new ExecutionDataflowBlockOptions() 29 | { 30 | MaxDegreeOfParallelism = 11, 31 | BoundedCapacity = 6, 32 | }); 33 | var callBackStep = new ActionBlock(resultCallback); 34 | step1.LinkTo(step2, new DataflowLinkOptions()); 35 | step2.LinkTo(step3, new DataflowLinkOptions()); 36 | step3.LinkTo(callBackStep); 37 | return step1; 38 | } 39 | 40 | private static string FindMostCommon(string input) 41 | { 42 | return input.Split(' ') 43 | .GroupBy(word => word) 44 | .OrderBy(group => group.Count()) 45 | .Last() 46 | .Key; 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /PipelineImplementations/Part2/TPLDataflow/TPLDataflowPipelineWithAwaitAttempt1.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using System.Text; 3 | using System.Threading.Tasks; 4 | using System.Threading.Tasks.Dataflow; 5 | 6 | namespace PipelineImplementations.Part2.TPLDataflow 7 | { 8 | public class TC 9 | { 10 | public TC(TInput input, TaskCompletionSource tcs) 11 | { 12 | Input = input; 13 | TaskCompletionSource = tcs; 14 | } 15 | public TInput Input { get; set; } 16 | public TaskCompletionSource TaskCompletionSource{ get; set; } 17 | } 18 | 19 | public class TPLDataflowPipelineWithAwaitAttempt1 20 | { 21 | public static TransformBlock, TC> CreatePipeline() 22 | { 23 | var step1 = new TransformBlock, TC>((tc) => 24 | new TC (FindMostCommon(tc.Input), tc.TaskCompletionSource)); 25 | 26 | var step2 = new TransformBlock, TC>((tc) => 27 | new TC (tc.Input.Length, tc.TaskCompletionSource)); 28 | 29 | var step3 = new TransformBlock, TC>((tc) => 30 | new TC (tc.Input % 2 == 1, tc.TaskCompletionSource)); 31 | 32 | var setResultStep = new ActionBlock>((tc) => tc.TaskCompletionSource.SetResult(tc.Input)); 33 | 34 | step1.LinkTo(step2, new DataflowLinkOptions()); 35 | step2.LinkTo(step3, new DataflowLinkOptions()); 36 | step3.LinkTo(setResultStep, new DataflowLinkOptions()); 37 | return step1; 38 | } 39 | 40 | 41 | private static string FindMostCommon(string input) 42 | { 43 | return input.Split(' ') 44 | .GroupBy(word => word) 45 | .OrderBy(group => group.Count()) 46 | .Last() 47 | .Key; 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /PipelineImplementations/Part2/TPLDataflow/TPLPipelineWithAwaitAttempt2.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using System.Threading.Tasks.Dataflow; 6 | 7 | namespace PipelineImplementations.Part2.TPLDataflow 8 | { 9 | public class TPLPipelineWithAwaitAttempt2 10 | { 11 | private List _transformBlocks = new List(); 12 | public TPLPipelineWithAwaitAttempt2 AddStep(Func stepFunc) 13 | { 14 | var step = new TransformBlock, TC>((tc) => 15 | { 16 | try 17 | { 18 | return new TC(stepFunc(tc.Input), tc.TaskCompletionSource); 19 | } 20 | catch (Exception e) 21 | { 22 | tc.TaskCompletionSource.SetException(e); 23 | return new TC(default(TLocalOut), tc.TaskCompletionSource); 24 | } 25 | }); 26 | 27 | if (_transformBlocks.Count > 0) 28 | { 29 | var lastStep = _transformBlocks.Last(); 30 | var targetBlock = (lastStep as ISourceBlock>); 31 | targetBlock.LinkTo(step, new DataflowLinkOptions(), 32 | tc => !tc.TaskCompletionSource.Task.IsFaulted); 33 | targetBlock.LinkTo(DataflowBlock.NullTarget>(), new DataflowLinkOptions(), 34 | tc => tc.TaskCompletionSource.Task.IsFaulted); 35 | } 36 | _transformBlocks.Add(step); 37 | return this; 38 | } 39 | 40 | public TPLPipelineWithAwaitAttempt2 CreatePipeline() 41 | { 42 | var setResultStep = 43 | new ActionBlock>((tc) => tc.TaskCompletionSource.SetResult(tc.Input)); 44 | var lastStep = _transformBlocks.Last(); 45 | var setResultBlock = (lastStep as ISourceBlock>); 46 | setResultBlock.LinkTo(setResultStep); 47 | return this; 48 | } 49 | 50 | public Task Execute(TIn input) 51 | { 52 | var firstStep = _transformBlocks[0] as ITargetBlock>; 53 | var tcs = new TaskCompletionSource(); 54 | firstStep.SendAsync(new TC(input, tcs)); 55 | return tcs.Task; 56 | } 57 | } 58 | } -------------------------------------------------------------------------------- /PipelineImplementations/Part2/TPLDataflow/TPLPipelineWithAwaitFinal.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using System.Threading.Tasks.Dataflow; 6 | 7 | namespace PipelineImplementations.Part2.TPLDataflow 8 | { 9 | public class TPLPipelineWithAwaitFinal 10 | { 11 | private List _steps = new List(); 12 | public void AddStep(Func stepFunc) 13 | { 14 | var step = new TransformBlock, TC>((tc) => 15 | new TC (stepFunc(tc.Input), tc.TaskCompletionSource)); 16 | if (_steps.Count > 0) 17 | { 18 | var lastStep = _steps.Last(); 19 | var targetBlock = (lastStep as ISourceBlock>); 20 | targetBlock.LinkTo(step, new DataflowLinkOptions()); 21 | } 22 | _steps.Add(step); 23 | } 24 | 25 | public void CreatePipeline() 26 | { 27 | var setResultStep = 28 | new ActionBlock>((tc) => tc.TaskCompletionSource.SetResult(tc.Input)); 29 | var lastStep = _steps.Last(); 30 | var targetBlock = (lastStep as ISourceBlock>); 31 | targetBlock.LinkTo(setResultStep); 32 | } 33 | 34 | public Task Execute(TIn input) 35 | { 36 | var firstStep = _steps[0] as ITargetBlock>; 37 | var tcs = new TaskCompletionSource(); 38 | firstStep.SendAsync(new TC(input, tcs)); 39 | return tcs.Task; 40 | } 41 | } 42 | } -------------------------------------------------------------------------------- /PipelineImplementations/Part2/UsagePart2.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | using System.Threading.Tasks.Dataflow; 8 | using PipelineImplementations.Part2.TPLDataflow; 9 | 10 | namespace PipelineImplementations.Part2 11 | { 12 | public class UsagePart2 13 | { 14 | public static void Use() 15 | { 16 | Console.WriteLine(Utils.GetThreadPoolThreadsInUse()); 17 | //UseSimple(); 18 | //UseSimpleAsync(); 19 | //UseTCS(); 20 | UseBuilder(); 21 | } 22 | 23 | private static void UseBuilder() 24 | { 25 | var pipeline = new TPLPipelineWithAwaitAttempt2() 26 | .AddStep(sentence => FindMostCommon(sentence)) 27 | .AddStep(word => word.Length) 28 | .AddStep(length => length % 2 == 1) 29 | .CreatePipeline(); 30 | 31 | System.Threading.Tasks.Task.Run(async () => 32 | { 33 | bool res; 34 | try 35 | { 36 | res = await pipeline.Execute("The pipeline pattern is the best pattern"); 37 | Console.WriteLine(res); 38 | } 39 | catch (Exception e) 40 | { 41 | } 42 | 43 | try 44 | { 45 | res = await pipeline.Execute("The pipeline pattern is the best pattern"); 46 | Console.WriteLine(res); 47 | } 48 | catch (Exception e) 49 | { 50 | } 51 | 52 | try 53 | { 54 | res = await pipeline.Execute("abcd"); 55 | Console.WriteLine(res); 56 | } 57 | catch (Exception e) 58 | { 59 | } 60 | 61 | try 62 | { 63 | res = await pipeline.Execute("abcd"); 64 | Console.WriteLine(res); 65 | } 66 | catch (Exception e) 67 | { 68 | } 69 | 70 | res = await pipeline.Execute("The pipeline pattern is the best pattern"); 71 | Console.WriteLine(res); 72 | res = await pipeline.Execute("The pipeline pattern is the best pattern"); 73 | Console.WriteLine(res); 74 | res = await pipeline.Execute("The pipeline pattern is the best pattern"); 75 | Console.WriteLine(res); 76 | 77 | 78 | }).Wait(); 79 | } 80 | 81 | private static void UseSimple() 82 | { 83 | var pipeline = TPLDataflowPipelineSimple.CreatePipeline(resultCallback: res => 84 | { 85 | Console.WriteLine(res); 86 | Console.WriteLine(Utils.GetThreadPoolThreadsInUse()); 87 | }); 88 | for (int i = 0; i < 50; i++) 89 | { 90 | pipeline.Post("The pipeline pattern is the best pattern"); 91 | 92 | } 93 | } 94 | 95 | private static void UseSimpleAsync() 96 | { 97 | var pipeline = TPLDataflowPipelineSimple.CreatePipeline(resultCallback: res => 98 | { 99 | Console.WriteLine(res); 100 | Console.WriteLine(Utils.GetThreadPoolThreadsInUse()); 101 | }); 102 | 103 | Task.Run(async () => 104 | { 105 | for (int i = 0; i < 50; i++) 106 | { 107 | await pipeline.SendAsync("The pipeline pattern is the best pattern"); 108 | } 109 | }); 110 | } 111 | 112 | private static void UseTCS() 113 | { 114 | var pipeline = TPLDataflowPipelineWithAwaitAttempt1.CreatePipeline(); 115 | 116 | var tsk = System.Threading.Tasks.Task.Run(async () => 117 | { 118 | var tcs = new TaskCompletionSource(); 119 | var tc = new TC( 120 | "The pipeline patter is the best patter", tcs); 121 | var task = tcs.Task; 122 | await pipeline.SendAsync(tc); 123 | var result = await task; 124 | Console.WriteLine(result); 125 | //await Task.Delay(1000); 126 | }); 127 | tsk.Wait(CancellationToken.None); 128 | } 129 | 130 | private static string FindMostCommon(string input) 131 | { 132 | if (input == "abcd") 133 | throw new Exception(); 134 | 135 | return input.Split(' ') 136 | .GroupBy(word => word) 137 | .OrderBy(group => group.Count()) 138 | .Last() 139 | .Key; 140 | } 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /PipelineImplementations/Part3/Disruptor/DisruptorExample.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Runtime.InteropServices; 5 | using System.Text; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | using Disruptor; 9 | using Disruptor.Dsl; 10 | 11 | namespace PipelineImplementations.Part3.Disruptor 12 | { 13 | public class EventExample 14 | { 15 | public object Value { get; set; } 16 | } 17 | 18 | public class MyHandler : IEventHandler, IWorkHandler 19 | { 20 | public void OnEvent(EventExample data, long sequence, bool endOfBatch) 21 | { 22 | OnEvent(data); 23 | } 24 | 25 | public void OnEvent(EventExample data) 26 | { 27 | Console.WriteLine("EventExample handled: Value = {0} ", data.Value); 28 | data.Value += "Finished1"; 29 | Thread.Sleep(1); 30 | } 31 | } 32 | 33 | public class MyHandler2 : IEventHandler, IWorkHandler 34 | { 35 | public void OnEvent(EventExample data, long sequence, bool endOfBatch) 36 | { 37 | OnEvent(data); 38 | } 39 | 40 | public void OnEvent(EventExample data) 41 | { 42 | Console.WriteLine("Handling part 2: Value = {0} ", data.Value); 43 | data.Value += "Finished2"; 44 | Thread.Sleep(1); 45 | } 46 | } 47 | 48 | public class DisruptorExample 49 | { 50 | private Disruptor _disruptor; 51 | 52 | public void CreatePipeline() 53 | { 54 | _disruptor = new Disruptor(() => new EventExample(), 1024, TaskScheduler.Default, ProducerType.Multi, new BlockingSpinWaitWaitStrategy()); 55 | 56 | _disruptor.HandleEventsWithWorkerPool(new MyHandler()).HandleEventsWithWorkerPool(new IWorkHandler[]{new MyHandler2()}); 57 | _disruptor.Start(); 58 | } 59 | 60 | 61 | public void Execute(string data) 62 | { 63 | var sequence = _disruptor.RingBuffer.Next(); 64 | var disruptorEvent = _disruptor[sequence]; 65 | disruptorEvent.Value = data; 66 | _disruptor.RingBuffer.Publish(sequence); 67 | 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /PipelineImplementations/Part3/Disruptor/DisruptorSimple.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Runtime.InteropServices; 5 | using System.Text; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | using Disruptor; 9 | using Disruptor.Dsl; 10 | 11 | namespace PipelineImplementations.Part3.Disruptor 12 | { 13 | public interface ISimplePipeline 14 | { 15 | void Execute(object data); 16 | } 17 | public class DisruptorSimple : ISimplePipeline 18 | { 19 | private class StepPayload 20 | { 21 | public object Value { get; set; } 22 | } 23 | 24 | private class DelegateHandler : IWorkHandler 25 | { 26 | private readonly Func _stepFunc; 27 | 28 | public DelegateHandler(Func stepFunc) 29 | { 30 | _stepFunc = stepFunc; 31 | } 32 | 33 | public void OnEvent(StepPayload payload) 34 | { 35 | payload.Value = _stepFunc(payload.Value); 36 | } 37 | } 38 | 39 | 40 | private Disruptor _disruptor; 41 | private List _steps = new List(); 42 | public DisruptorSimple AddStep(Func stepFunc) 43 | { 44 | _steps.Add(new DelegateHandler((obj) => stepFunc((TLocalIn)obj))); 45 | return this; 46 | } 47 | 48 | public ISimplePipeline CreatePipeline() 49 | { 50 | _disruptor = new Disruptor(() => new StepPayload(), 1024, TaskScheduler.Default/*, ProducerType.Multi, new BlockingSpinWaitWaitStrategy()*/); 51 | var handlerGroup = _disruptor.HandleEventsWithWorkerPool(_steps.First()); 52 | for (int i = 1; i < _steps.Count; i++) 53 | { 54 | var step = _steps[i]; 55 | var makeStepToArray = new IWorkHandler[] {step}; 56 | handlerGroup = handlerGroup.HandleEventsWithWorkerPool(makeStepToArray); 57 | } 58 | 59 | _disruptor.Start(); 60 | return this; 61 | } 62 | 63 | 64 | public void Execute(object data) 65 | { 66 | var sequence = _disruptor.RingBuffer.Next(); 67 | var disruptorEvent = _disruptor[sequence]; 68 | disruptorEvent.Value = data; 69 | _disruptor.RingBuffer.Publish(sequence); 70 | 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /PipelineImplementations/Part3/Disruptor/DisruptorSimpleAwaitable.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Runtime.InteropServices; 5 | using System.Text; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | using Disruptor; 9 | using Disruptor.Dsl; 10 | 11 | namespace PipelineImplementations.Part3.Disruptor 12 | { 13 | 14 | public interface IPipeline 15 | { 16 | Task Execute(string data); 17 | } 18 | 19 | public class DisruptorSimpleAwaitable : IPipeline 20 | { 21 | public class StepPayload 22 | { 23 | public object Value { get; set; } 24 | public TaskCompletionSource TaskCompletionSource { get; set; } 25 | } 26 | 27 | public class DelegateHandler : IWorkHandler> 28 | { 29 | private readonly Func _stepFunc; 30 | 31 | public DelegateHandler(Func stepFunc) 32 | { 33 | _stepFunc = stepFunc; 34 | } 35 | 36 | public void OnEvent(StepPayload payload) 37 | { 38 | try 39 | { 40 | if (payload.TaskCompletionSource.Task.IsFaulted) 41 | return; 42 | payload.Value = _stepFunc(payload.Value); 43 | } 44 | catch (Exception e) 45 | { 46 | payload.TaskCompletionSource.SetException(e); 47 | } 48 | } 49 | } 50 | 51 | class SetResultHandler :IWorkHandler> 52 | { 53 | public void OnEvent(StepPayload payload) 54 | { 55 | if (payload.TaskCompletionSource.Task.IsFaulted) 56 | return; 57 | payload.TaskCompletionSource.SetResult((TOut)payload.Value); 58 | } 59 | } 60 | 61 | private Disruptor> _disruptor; 62 | private List> _steps = new List>(); 63 | public DisruptorSimpleAwaitable AddStep(Func stepFunc) 64 | { 65 | _steps.Add(new DelegateHandler((obj) => stepFunc((TLocalIn)obj))); 66 | return this; 67 | } 68 | 69 | public IPipeline CreatePipeline() 70 | { 71 | _disruptor = new Disruptor>(() => new StepPayload(), 1024, TaskScheduler.Default/*, ProducerType.Multi, new BlockingSpinWaitWaitStrategy()*/); 72 | var handlerGroup = _disruptor.HandleEventsWithWorkerPool(_steps.First()); 73 | for (int i = 1; i < _steps.Count; i++) 74 | { 75 | var step = _steps[i]; 76 | var makeStepToArray = new IWorkHandler>[] {step}; 77 | handlerGroup = handlerGroup.HandleEventsWithWorkerPool(makeStepToArray); 78 | } 79 | var setResultHandler = new SetResultHandler(); 80 | var setResultHandlerToArray = new IWorkHandler>[] {setResultHandler}; 81 | handlerGroup.HandleEventsWithWorkerPool(setResultHandlerToArray); 82 | 83 | _disruptor.Start(); 84 | return this; 85 | } 86 | 87 | 88 | public Task Execute(string data) 89 | { 90 | var sequence = _disruptor.RingBuffer.Next(); 91 | var disruptorEvent = _disruptor[sequence]; 92 | disruptorEvent.Value = data; 93 | var tcs = new TaskCompletionSource(); 94 | disruptorEvent.TaskCompletionSource = tcs; 95 | 96 | _disruptor.RingBuffer.Publish(sequence); 97 | return tcs.Task; 98 | 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /PipelineImplementations/Part3/TPLDataflowWithAsync/TPLDataflowSteppedAsync.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using System.Threading.Tasks.Dataflow; 7 | using PipelineImplementations.Part2.TPLDataflow; 8 | 9 | namespace PipelineImplementations.Part3.TPLDataflowWithAsync 10 | { 11 | public class TPLDataflowSteppedAsync 12 | { 13 | private List _steps = new List(); 14 | 15 | public void AddStep(Func stepFunc) 16 | { 17 | if (_steps.Count == 0) 18 | { 19 | var step = new TransformBlock((input) => 20 | stepFunc(input)); 21 | _steps.Add(step); 22 | } 23 | else 24 | { 25 | 26 | var lastStep = _steps.Last(); 27 | var targetBlock = (lastStep as ISourceBlock); 28 | if (targetBlock != null) 29 | { 30 | var step = new TransformBlock((input) => 31 | stepFunc(input)); 32 | targetBlock.LinkTo(step, new DataflowLinkOptions()); 33 | _steps.Add(step); 34 | } 35 | else 36 | { 37 | var step = new TransformBlock, TLocalOut>((input) => 38 | stepFunc(input.Result)); 39 | var targetBlock1 = (lastStep as ISourceBlock>); 40 | targetBlock1.LinkTo(step, new DataflowLinkOptions()); 41 | _steps.Add(step); 42 | //var step = new TransformBlock, TLocalOut>((input) => 43 | // stepFunc(input.Result)); 44 | //var targetBlock1 = lastStep as ISourceBlock>; 45 | //targetBlock1.LinkTo(step, new DataflowLinkOptions()); 46 | //_steps.Add(step); 47 | 48 | } 49 | } 50 | 51 | } 52 | 53 | public void AddStepAsync(Func> stepFunc) 54 | { 55 | if (_steps.Count == 0) 56 | { 57 | var step = new TransformBlock>(async (input) => await stepFunc(input)); 58 | _steps.Add(step); 59 | } 60 | else 61 | { 62 | var lastStep = _steps.Last(); 63 | var targetBlock = (lastStep as ISourceBlock>); 64 | if (targetBlock != null) 65 | { 66 | var step = new TransformBlock, Task>(async (input) => 67 | await stepFunc(input.Result)); 68 | 69 | targetBlock.LinkTo(step, new DataflowLinkOptions()); 70 | _steps.Add(step); 71 | } 72 | else 73 | { 74 | var targetBlock1 = (lastStep as ISourceBlock); 75 | var step = new TransformBlock>(async (input) => 76 | await stepFunc(input)); 77 | 78 | targetBlock1.LinkTo(step, new DataflowLinkOptions()); 79 | _steps.Add(step); 80 | } 81 | } 82 | 83 | } 84 | 85 | public void CreatePipeline(Action resultCallback) 86 | { 87 | var lastStep = _steps.Last(); 88 | var targetBlock = (lastStep as ISourceBlock>); 89 | if (targetBlock != null) 90 | { 91 | var callBackStep = new ActionBlock>(t => resultCallback(t.Result)); 92 | targetBlock.LinkTo(callBackStep); 93 | } 94 | else 95 | { 96 | var callBackStep1 = new ActionBlock(t => resultCallback(t)); 97 | var targetBlock1 = (lastStep as ISourceBlock); 98 | targetBlock1.LinkTo(callBackStep1); 99 | } 100 | } 101 | 102 | public void Execute(TIn input) 103 | { 104 | var firstStep = _steps[0] as ITargetBlock; 105 | firstStep.SendAsync(input); 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /PipelineImplementations/Part3/TPLDataflowWithAsync/TPLDataflowSteppedAsyncFinal2.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using System.Threading.Tasks.Dataflow; 7 | using PipelineImplementations.Part2.TPLDataflow; 8 | 9 | namespace PipelineImplementations.Part3.TPLDataflowWithAsync 10 | { 11 | public class TPLDataflowSteppedAsyncFinal2 12 | { 13 | private List<(IDataflowBlock Block, bool IsAsync)> _steps = new List<(IDataflowBlock Block, bool IsAsync)>(); 14 | public void AddStep(Func stepFunc) 15 | { 16 | if (_steps.Count == 0) 17 | { 18 | var step = new TransformBlock(stepFunc); 19 | _steps.Add((step, IsAsync: false)); 20 | } 21 | else 22 | { 23 | 24 | var lastStep = _steps.Last(); 25 | if (!lastStep.IsAsync) 26 | { 27 | var step = new TransformBlock(stepFunc); 28 | var targetBlock = (lastStep.Block as ISourceBlock); 29 | targetBlock.LinkTo(step, new DataflowLinkOptions()); 30 | _steps.Add((step, IsAsync: false)); 31 | } 32 | else 33 | { 34 | var step = new TransformBlock, TLocalOut>(async (input) => stepFunc(await input)); 35 | var targetBlock = (lastStep.Block as ISourceBlock>); 36 | targetBlock.LinkTo(step, new DataflowLinkOptions()); 37 | _steps.Add((step, IsAsync: false)); 38 | } 39 | } 40 | 41 | } 42 | 43 | public void AddStepAsync(Func> stepFunc) 44 | { 45 | if (_steps.Count == 0) 46 | { 47 | var step = 48 | new TransformBlock>(async (input) => await stepFunc(input)); 49 | _steps.Add((step, IsAsync: true)); 50 | } 51 | else 52 | { 53 | var lastStep = _steps.Last(); 54 | if (lastStep.IsAsync) 55 | { 56 | var step = new TransformBlock, Task>(async (input) => 57 | await stepFunc(await input)); 58 | var targetBlock = (lastStep.Block as ISourceBlock>); 59 | targetBlock.LinkTo(step, new DataflowLinkOptions()); 60 | _steps.Add((step, IsAsync: true)); 61 | } 62 | else 63 | { 64 | var step = new TransformBlock>(async (input) => 65 | await stepFunc(input)); 66 | var targetBlock = (lastStep.Block as ISourceBlock); 67 | targetBlock.LinkTo(step, new DataflowLinkOptions()); 68 | _steps.Add((step, IsAsync: true)); 69 | } 70 | } 71 | } 72 | 73 | public async Task CreatePipeline(Action resultCallback) 74 | { 75 | var lastStep = _steps.Last(); 76 | if (lastStep.IsAsync) 77 | { 78 | var targetBlock = (lastStep.Block as ISourceBlock>); 79 | var callBackStep = new ActionBlock>(async t => resultCallback(await t)); 80 | targetBlock.LinkTo(callBackStep); 81 | } 82 | else 83 | { 84 | var callBackStep = new ActionBlock(t => resultCallback(t)); 85 | var targetBlock = (lastStep.Block as ISourceBlock); 86 | targetBlock.LinkTo(callBackStep); 87 | } 88 | } 89 | 90 | public void Execute(TIn input) 91 | { 92 | var firstStep = _steps[0].Block as ITargetBlock; 93 | firstStep.SendAsync(input); 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /PipelineImplementations/Part3/TPLDataflowWithAsync/TPLDataflowSteppedSimple.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using System.Threading.Tasks.Dataflow; 7 | using PipelineImplementations.Part2.TPLDataflow; 8 | 9 | namespace PipelineImplementations.Part3.TPLDataflowWithAsync 10 | { 11 | public class TPLDataflowSteppedSimple 12 | { 13 | private List _steps = new List(); 14 | public void AddStep(Func stepFunc) 15 | { 16 | var step = new TransformBlock((input) => 17 | stepFunc(input)); 18 | if (_steps.Count > 0) 19 | { 20 | var lastStep = _steps.Last(); 21 | var targetBlock = (lastStep as ISourceBlock); 22 | targetBlock.LinkTo(step, new DataflowLinkOptions()); 23 | } 24 | _steps.Add(step); 25 | } 26 | 27 | public void CreatePipeline(Action resultCallback) 28 | { 29 | var callBackStep = new ActionBlock(resultCallback); 30 | var lastStep = _steps.Last(); 31 | var targetBlock = (lastStep as ISourceBlock); 32 | targetBlock.LinkTo(callBackStep); 33 | } 34 | 35 | public void Execute(TIn input) 36 | { 37 | var firstStep = _steps[0] as ITargetBlock; 38 | firstStep.SendAsync(input); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /PipelineImplementations/Part3/UsagePart3.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using PipelineImplementations.Part3.Disruptor; 7 | using PipelineImplementations.Part3.TPLDataflowWithAsync; 8 | 9 | namespace PipelineImplementations.Part3 10 | { 11 | public class UsagePart3 12 | { 13 | public static void Use() 14 | { 15 | //DisruptorExample(); 16 | DisruptorSimple(); 17 | //DisruptorAwaitable(); 18 | } 19 | 20 | private static async Task DisruptorAwaitable() 21 | { 22 | var pipeline = new DisruptorSimpleAwaitable() 23 | .AddStep(FindMostCommon) 24 | .AddStep(CountChars) 25 | .AddStep(IsOdd) 26 | .CreatePipeline(); 27 | 28 | Console.WriteLine(await pipeline.Execute("The pipeline patter is the best pattern")); 29 | } 30 | 31 | private static void DisruptorSimple() 32 | { 33 | var pipeline = new DisruptorSimple() 34 | .AddStep(FindMostCommon) 35 | .AddStep(CountChars) 36 | .AddStep(IsOdd) 37 | // This last step is kind of a result callback. We'll solve it better in a minute 38 | .AddStep((res) => 39 | { 40 | Console.WriteLine(res); 41 | return res; 42 | }) 43 | .CreatePipeline(); 44 | pipeline.Execute("The pipeline pattern is the best pattern"); 45 | } 46 | 47 | private static void Simple() 48 | { 49 | var pipeline = new TPLDataflowSteppedSimple(); 50 | pipeline.AddStep(input => FindMostCommon(input)); 51 | pipeline.AddStep(input => CountChars(input)); 52 | pipeline.AddStep(input => IsOdd(input)); 53 | pipeline.CreatePipeline(resultCallback: res => Console.WriteLine(res)); 54 | 55 | pipeline.Execute("The pipeline patter is the best patter"); 56 | 57 | 58 | } 59 | 60 | private static void SimpleAsyncFinal2() 61 | { 62 | var pipeline = new TPLDataflowSteppedAsyncFinal2(); 63 | 64 | Func> findMostCommonAsync = async (str) => await FindMostCommonAsync(str); 65 | pipeline.AddStepAsync(async input => FindMostCommon(input)); 66 | 67 | //pipeline.AddStepAsync(async (str) => await FindMostCommonAsync(str)); 68 | pipeline.AddStep(input => CountChars(input)); 69 | pipeline.AddStepAsync(async input => IsOdd(input)); 70 | 71 | 72 | pipeline.CreatePipeline(res => Console.WriteLine(res)); 73 | 74 | pipeline.Execute("The pipeline patter is the best patter"); 75 | 76 | } 77 | 78 | private static async Task SomeAsync(string input) 79 | { 80 | await Task.Delay(100); 81 | return "asdf"; 82 | } 83 | 84 | private static string FindMostCommon(string input) 85 | { 86 | return input.Split(' ') 87 | .GroupBy(word => word) 88 | .OrderBy(group => group.Count()) 89 | .Last() 90 | .Key; 91 | } 92 | 93 | private static async Task FindMostCommonAsync(string input) 94 | { 95 | await Task.Delay(10); 96 | return input.Split(' ') 97 | .GroupBy(word => word) 98 | .OrderBy(group => group.Count()) 99 | .Last() 100 | .Key; 101 | } 102 | 103 | private static int CountChars(string mostCommon) 104 | { 105 | return mostCommon.Length; 106 | } 107 | 108 | private static bool IsOdd(int number) 109 | { 110 | var res = number % 2 == 1; 111 | //Console.WriteLine(res.ToString()); 112 | return res; 113 | } 114 | 115 | 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /PipelineImplementations/PartN/IPipeline.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | 4 | namespace PipelineImplementations.PartN 5 | { 6 | public interface IPipeline : IDisposable 7 | { 8 | // TODO: use ValueTask + IValueTaskSource to avoid allocations 9 | Task Execute(TIn data); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /PipelineImplementations/PartN/IPipelineBuilder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace PipelineImplementations.PartN 4 | { 5 | public interface IPipelineBuilder 6 | { 7 | IPipelineBuilderStep Build(Func stepFunc, int workerCount); 8 | } 9 | } -------------------------------------------------------------------------------- /PipelineImplementations/PartN/IPipelineBuilderStep.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace PipelineImplementations.PartN 4 | { 5 | public interface IPipelineBuilderStep 6 | { 7 | IPipelineBuilderStep AddStep(Func stepFunc, int workerCount); 8 | IPipeline Create(); 9 | } 10 | } -------------------------------------------------------------------------------- /PipelineImplementations/PartN/Impl/DisruptorEvent.cs: -------------------------------------------------------------------------------- 1 | namespace PipelineImplementations.PartN.Impl 2 | { 3 | public class DisruptorEvent 4 | { 5 | public object Value { get; set; } 6 | public object TaskCompletionSource { get; set; } 7 | 8 | public T Read() 9 | { 10 | // TODO: use unsafe code to prevent boxing for small blittable value types 11 | return (T)Value; 12 | } 13 | 14 | public void Write(T value) 15 | { 16 | // TODO: use unsafe code to prevent boxing for small blittable value types 17 | Value = value; 18 | } 19 | } 20 | } -------------------------------------------------------------------------------- /PipelineImplementations/PartN/Impl/DisruptorPipeline.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using Disruptor.Dsl; 3 | 4 | namespace PipelineImplementations.PartN.Impl 5 | { 6 | public class DisruptorPipeline : IPipeline 7 | { 8 | private readonly Disruptor _disruptor; 9 | 10 | public DisruptorPipeline(Disruptor disruptor) 11 | { 12 | _disruptor = disruptor; 13 | _disruptor.Start(); 14 | } 15 | 16 | public Task Execute(TIn data) 17 | { 18 | // RunContinuationsAsynchronously to prevent continuation from "stealing" the releaser thread 19 | var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); 20 | 21 | var sequence = _disruptor.RingBuffer.Next(); 22 | var disruptorEvent = _disruptor[sequence]; 23 | disruptorEvent.Write(data); 24 | disruptorEvent.TaskCompletionSource = tcs; 25 | _disruptor.RingBuffer.Publish(sequence); 26 | 27 | return tcs.Task; 28 | } 29 | 30 | public void Dispose() 31 | { 32 | _disruptor.Shutdown(); 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /PipelineImplementations/PartN/Impl/DisruptorPipelineBuilder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using Disruptor; 6 | using Disruptor.Dsl; 7 | 8 | namespace PipelineImplementations.PartN.Impl 9 | { 10 | public class DisruptorPipelineBuilder : IPipelineBuilder 11 | { 12 | public IPipelineBuilderStep Build(Func stepFunc, int workerCount) 13 | { 14 | var state = new State(); 15 | 16 | return state.AddStep(stepFunc, workerCount); 17 | } 18 | 19 | private interface IStep : IWorkHandler, IEventHandler 20 | { 21 | bool IsWorkerPool { get; } 22 | 23 | IWorkHandler[] AsWorkerPool(); 24 | } 25 | 26 | private class Step : IPipelineBuilderStep, IStep 27 | { 28 | private readonly Func _stepFunc; 29 | private readonly int _threadCount; 30 | 31 | public Step(Func stepFunc, int threadCount) 32 | { 33 | _stepFunc = stepFunc; 34 | _threadCount = threadCount; 35 | } 36 | 37 | public State State { get; set; } 38 | 39 | public bool IsWorkerPool => _threadCount != 1; 40 | 41 | public IWorkHandler[] AsWorkerPool() 42 | => Enumerable.Repeat(this, _threadCount).Cast>().ToArray(); 43 | 44 | public IPipelineBuilderStep AddStep(Func stepFunc, int workerCount) 45 | { 46 | return State.AddStep(stepFunc, workerCount); 47 | } 48 | 49 | public IPipeline Create() 50 | { 51 | return State.Create(); 52 | } 53 | 54 | public void OnEvent(DisruptorEvent evt) 55 | { 56 | evt.Write(_stepFunc.Invoke(evt.Read())); 57 | } 58 | 59 | public void OnEvent(DisruptorEvent data, long sequence, bool endOfBatch) 60 | { 61 | OnEvent(data); 62 | } 63 | } 64 | 65 | private class State 66 | { 67 | private readonly List _steps = new List(); 68 | 69 | public IPipelineBuilderStep AddStep(Func stepFunc, int workerCount) 70 | { 71 | var step = new Step(stepFunc, workerCount); 72 | 73 | _steps.Add(step); 74 | step.State = this; 75 | 76 | return step; 77 | } 78 | 79 | public IPipeline Create() 80 | { 81 | var disruptor = new Disruptor(() => new DisruptorEvent(), 1024, TaskScheduler.Default, ProducerType.Multi, new BlockingSpinWaitWaitStrategy()); 82 | 83 | var firstStep = _steps.First(); 84 | var currentGroup = firstStep.IsWorkerPool 85 | ? disruptor.HandleEventsWithWorkerPool(firstStep.AsWorkerPool()) 86 | : disruptor.HandleEventsWith(firstStep); 87 | 88 | foreach (var step in _steps.Skip(1)) 89 | { 90 | currentGroup = step.IsWorkerPool 91 | ? currentGroup.HandleEventsWithWorkerPool(step.AsWorkerPool()) 92 | : currentGroup.HandleEventsWith(step); 93 | } 94 | 95 | currentGroup.HandleEventsWith(new Releaser()); 96 | 97 | return new DisruptorPipeline(disruptor); 98 | } 99 | } 100 | 101 | private class Releaser : IEventHandler 102 | { 103 | public void OnEvent(DisruptorEvent data, long sequence, bool endOfBatch) 104 | { 105 | var tcs = (TaskCompletionSource)data.TaskCompletionSource; 106 | tcs.SetResult(data.Read()); 107 | } 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /PipelineImplementations/PartN/UsagePartN.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using PipelineImplementations.PartN.Impl; 7 | 8 | namespace PipelineImplementations.PartN 9 | { 10 | public static class UsagePartN 11 | { 12 | public static async Task Use1() 13 | { 14 | var builder = new DisruptorPipelineBuilder(); 15 | 16 | var pipeline = builder.Build(FindMostCommon, 2) 17 | .AddStep(x => x.Length, 2) 18 | .AddStep(x => x % 2 == 1, 2) 19 | .Create(); 20 | 21 | Console.WriteLine(await pipeline.Execute("The pipeline pattern is the best pattern")); 22 | Console.WriteLine(await pipeline.Execute("The pipeline pattern is the best pattern")); 23 | Console.WriteLine(await pipeline.Execute("The pipeline pattern is the best pattern")); 24 | Console.WriteLine(await pipeline.Execute("The pipeline patter is the best patter")); 25 | Console.WriteLine(await pipeline.Execute("The pipeline pattern is the best pattern")); 26 | 27 | pipeline.Dispose(); 28 | } 29 | 30 | public static async Task Use2() 31 | { 32 | var builder = new DisruptorPipelineBuilder(); 33 | var results = new HashSet(); 34 | 35 | var pipeline = builder.Build(x => x.Max(), 2) 36 | .AddStep(x => new string(x, 20), 2) 37 | .AddStep(x => $"[{x}]", 1) 38 | .AddStep(x => results.Add(x), 1) 39 | .Create(); 40 | 41 | for (var i = 0; i < 1_000_000; i++) 42 | { 43 | _ = pipeline.Execute(i.ToString()); 44 | } 45 | 46 | await pipeline.Execute("X"); 47 | 48 | Console.WriteLine("Completed!"); 49 | 50 | foreach (var result in results) 51 | { 52 | Console.WriteLine(result); 53 | } 54 | 55 | pipeline.Dispose(); 56 | } 57 | 58 | private static string FindMostCommon(string input) 59 | { 60 | return input.Split(' ') 61 | .GroupBy(word => word) 62 | .OrderBy(group => group.Count()) 63 | .Last() 64 | .Key; 65 | } 66 | 67 | private static int CountChars(string mostCommon) 68 | { 69 | return mostCommon.Length; 70 | } 71 | 72 | private static bool IsOdd(int number) 73 | { 74 | var res = number % 2 == 1; 75 | //Console.WriteLine(res.ToString()); 76 | return res; 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /PipelineImplementations/PipelineImplementations.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {0EA3DBCB-2E0F-4670-861F-871D4626AB84} 8 | Exe 9 | PipelineImplementations 10 | PipelineImplementations 11 | v4.6.1 12 | 512 13 | true 14 | true 15 | 16 | 17 | 18 | AnyCPU 19 | true 20 | full 21 | false 22 | bin\Debug\ 23 | DEBUG;TRACE 24 | prompt 25 | 4 26 | 7.1 27 | 28 | 29 | AnyCPU 30 | pdbonly 31 | true 32 | bin\Release\ 33 | TRACE 34 | prompt 35 | 4 36 | 7.1 37 | 38 | 39 | 40 | ..\packages\Disruptor.3.4.2\lib\net452\Disruptor.dll 41 | 42 | 43 | 44 | 45 | ..\packages\System.Runtime.CompilerServices.Unsafe.4.5.2\lib\netstandard2.0\System.Runtime.CompilerServices.Unsafe.dll 46 | 47 | 48 | ..\packages\System.Threading.Tasks.Dataflow.4.9.0\lib\netstandard2.0\System.Threading.Tasks.Dataflow.dll 49 | 50 | 51 | ..\packages\System.Threading.Tasks.Extensions.4.5.3\lib\netstandard2.0\System.Threading.Tasks.Extensions.dll 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | -------------------------------------------------------------------------------- /PipelineImplementations/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using PipelineImplementations.Part1; 7 | using PipelineImplementations.Part1.BlockingCollection; 8 | using PipelineImplementations.Part2; 9 | using PipelineImplementations.Part3; 10 | using PipelineImplementations.PartN; 11 | 12 | namespace PipelineImplementations 13 | { 14 | public static class Program 15 | { 16 | public static async Task Main(string[] args) 17 | { 18 | //string input = "The pipeline pattern is the best pattern"; 19 | //var pipeline = new MyPipeline(); 20 | //Console.Write(pipeline.Execute(input)); // Returns 'True' because 'pattern' is the most common, with 7 characters and it's indeed an odd number 21 | //UsagePart1.Use(); 22 | 23 | //await UsagePartN.Use2(); 24 | UsagePart3.Use(); 25 | 26 | Console.WriteLine("Press any key to exit..."); 27 | Console.ReadKey(); 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /PipelineImplementations/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("PipelineImplementations")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("PipelineImplementations")] 13 | [assembly: AssemblyCopyright("Copyright © 2019")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("0ea3dbcb-2e0f-4670-861f-871d4626ab84")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /PipelineImplementations/Utils.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | 8 | namespace PipelineImplementations 9 | { 10 | public class Utils 11 | { 12 | public static int GetThreadPoolThreadsInUse() 13 | { 14 | int max, max2; 15 | ThreadPool.GetMaxThreads(out max, out max2); 16 | int available, available2; 17 | ThreadPool.GetAvailableThreads(out available, out available2); 18 | int running = max - available; 19 | return running; 20 | } 21 | 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /PipelineImplementations/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | --------------------------------------------------------------------------------