├── .gitattributes ├── .gitignore ├── DataflowPipelineBuilder.sln ├── DataflowPipelineBuilder ├── Assembly.cs ├── BlockExtensions.cs ├── Builder.cs ├── BuilderExtensions.cs ├── DataflowPipelineBuilder.csproj ├── ForkBuilder.cs ├── IBuilder.cs ├── IForkBuilder.cs ├── MiddleBuilder.cs ├── StartBuilder.cs ├── TaskExtensions.cs └── WrapperBlock.cs ├── LICENSE ├── README.md ├── SimpleDataflowPipelineBuilder ├── BlocksExtensions.cs └── SimpleDataflowPipelineBuilder.csproj └── Test ├── BuilderTest.cs └── Test.csproj /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.userosscache 8 | *.sln.docstates 9 | 10 | # User-specific files (MonoDevelop/Xamarin Studio) 11 | *.userprefs 12 | 13 | # Build results 14 | [Dd]ebug/ 15 | [Dd]ebugPublic/ 16 | [Rr]elease/ 17 | [Rr]eleases/ 18 | x64/ 19 | x86/ 20 | bld/ 21 | [Bb]in/ 22 | [Oo]bj/ 23 | [Ll]og/ 24 | 25 | # Visual Studio 2015 cache/options directory 26 | .vs/ 27 | # Uncomment if you have tasks that create the project's static files in wwwroot 28 | #wwwroot/ 29 | 30 | # MSTest test Results 31 | [Tt]est[Rr]esult*/ 32 | [Bb]uild[Ll]og.* 33 | 34 | # NUNIT 35 | *.VisualState.xml 36 | TestResult.xml 37 | 38 | # Build Results of an ATL Project 39 | [Dd]ebugPS/ 40 | [Rr]eleasePS/ 41 | dlldata.c 42 | 43 | # DNX 44 | project.lock.json 45 | project.fragment.lock.json 46 | artifacts/ 47 | 48 | *_i.c 49 | *_p.c 50 | *_i.h 51 | *.ilk 52 | *.meta 53 | *.obj 54 | *.pch 55 | *.pdb 56 | *.pgc 57 | *.pgd 58 | *.rsp 59 | *.sbr 60 | *.tlb 61 | *.tli 62 | *.tlh 63 | *.tmp 64 | *.tmp_proj 65 | *.log 66 | *.vspscc 67 | *.vssscc 68 | .builds 69 | *.pidb 70 | *.svclog 71 | *.scc 72 | 73 | # Chutzpah Test files 74 | _Chutzpah* 75 | 76 | # Visual C++ cache files 77 | ipch/ 78 | *.aps 79 | *.ncb 80 | *.opendb 81 | *.opensdf 82 | *.sdf 83 | *.cachefile 84 | *.VC.db 85 | *.VC.VC.opendb 86 | 87 | # Visual Studio profiler 88 | *.psess 89 | *.vsp 90 | *.vspx 91 | *.sap 92 | 93 | # TFS 2012 Local Workspace 94 | $tf/ 95 | 96 | # Guidance Automation Toolkit 97 | *.gpState 98 | 99 | # ReSharper is a .NET coding add-in 100 | _ReSharper*/ 101 | *.[Rr]e[Ss]harper 102 | *.DotSettings.user 103 | 104 | # JustCode is a .NET coding add-in 105 | .JustCode 106 | 107 | # TeamCity is a build add-in 108 | _TeamCity* 109 | 110 | # DotCover is a Code Coverage Tool 111 | *.dotCover 112 | 113 | # NCrunch 114 | _NCrunch_* 115 | .*crunch*.local.xml 116 | nCrunchTemp_* 117 | 118 | # MightyMoose 119 | *.mm.* 120 | AutoTest.Net/ 121 | 122 | # Web workbench (sass) 123 | .sass-cache/ 124 | 125 | # Installshield output folder 126 | [Ee]xpress/ 127 | 128 | # DocProject is a documentation generator add-in 129 | DocProject/buildhelp/ 130 | DocProject/Help/*.HxT 131 | DocProject/Help/*.HxC 132 | DocProject/Help/*.hhc 133 | DocProject/Help/*.hhk 134 | DocProject/Help/*.hhp 135 | DocProject/Help/Html2 136 | DocProject/Help/html 137 | 138 | # Click-Once directory 139 | publish/ 140 | 141 | # Publish Web Output 142 | *.[Pp]ublish.xml 143 | *.azurePubxml 144 | # TODO: Comment the next line if you want to checkin your web deploy settings 145 | # but database connection strings (with potential passwords) will be unencrypted 146 | #*.pubxml 147 | *.publishproj 148 | 149 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 150 | # checkin your Azure Web App publish settings, but sensitive information contained 151 | # in these scripts will be unencrypted 152 | PublishScripts/ 153 | 154 | # NuGet Packages 155 | *.nupkg 156 | # The packages folder can be ignored because of Package Restore 157 | **/packages/* 158 | # except build/, which is used as an MSBuild target. 159 | !**/packages/build/ 160 | # Uncomment if necessary however generally it will be regenerated when needed 161 | #!**/packages/repositories.config 162 | # NuGet v3's project.json files produces more ignoreable files 163 | *.nuget.props 164 | *.nuget.targets 165 | 166 | # Microsoft Azure Build Output 167 | csx/ 168 | *.build.csdef 169 | 170 | # Microsoft Azure Emulator 171 | ecf/ 172 | rcf/ 173 | 174 | # Windows Store app package directories and files 175 | AppPackages/ 176 | BundleArtifacts/ 177 | Package.StoreAssociation.xml 178 | _pkginfo.txt 179 | 180 | # Visual Studio cache files 181 | # files ending in .cache can be ignored 182 | *.[Cc]ache 183 | # but keep track of directories ending in .cache 184 | !*.[Cc]ache/ 185 | 186 | # Others 187 | ClientBin/ 188 | ~$* 189 | *~ 190 | *.dbmdl 191 | *.dbproj.schemaview 192 | *.jfm 193 | *.pfx 194 | *.publishsettings 195 | node_modules/ 196 | orleans.codegen.cs 197 | 198 | # Since there are multiple workflows, uncomment next line to ignore bower_components 199 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 200 | #bower_components/ 201 | 202 | # RIA/Silverlight projects 203 | Generated_Code/ 204 | 205 | # Backup & report files from converting an old project file 206 | # to a newer Visual Studio version. Backup files are not needed, 207 | # because we have git ;-) 208 | _UpgradeReport_Files/ 209 | Backup*/ 210 | UpgradeLog*.XML 211 | UpgradeLog*.htm 212 | 213 | # SQL Server files 214 | *.mdf 215 | *.ldf 216 | 217 | # Business Intelligence projects 218 | *.rdl.data 219 | *.bim.layout 220 | *.bim_*.settings 221 | 222 | # Microsoft Fakes 223 | FakesAssemblies/ 224 | 225 | # GhostDoc plugin setting file 226 | *.GhostDoc.xml 227 | 228 | # Node.js Tools for Visual Studio 229 | .ntvs_analysis.dat 230 | 231 | # Visual Studio 6 build log 232 | *.plg 233 | 234 | # Visual Studio 6 workspace options file 235 | *.opt 236 | 237 | # Visual Studio LightSwitch build output 238 | **/*.HTMLClient/GeneratedArtifacts 239 | **/*.DesktopClient/GeneratedArtifacts 240 | **/*.DesktopClient/ModelManifest.xml 241 | **/*.Server/GeneratedArtifacts 242 | **/*.Server/ModelManifest.xml 243 | _Pvt_Extensions 244 | 245 | # Paket dependency manager 246 | .paket/paket.exe 247 | paket-files/ 248 | 249 | # FAKE - F# Make 250 | .fake/ 251 | 252 | # JetBrains Rider 253 | .idea/ 254 | *.sln.iml 255 | 256 | # CodeRush 257 | .cr/ 258 | 259 | # Python Tools for Visual Studio (PTVS) 260 | __pycache__/ 261 | *.pyc -------------------------------------------------------------------------------- /DataflowPipelineBuilder.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.28010.2003 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DataflowPipelineBuilder", "DataflowPipelineBuilder\DataflowPipelineBuilder.csproj", "{1A36BFE3-D89E-4EED-8350-4BE42FBB0D1F}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Test", "Test\Test.csproj", "{7A3130A2-E41B-46A2-A3FC-0C20BB42C853}" 9 | EndProject 10 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SimpleDataflowPipelineBuilder", "SimpleDataflowPipelineBuilder\SimpleDataflowPipelineBuilder.csproj", "{1A913534-CBB7-4F17-96BD-8F128F9FA4CD}" 11 | EndProject 12 | Global 13 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 14 | Debug|Any CPU = Debug|Any CPU 15 | Release|Any CPU = Release|Any CPU 16 | EndGlobalSection 17 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 18 | {1A36BFE3-D89E-4EED-8350-4BE42FBB0D1F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 19 | {1A36BFE3-D89E-4EED-8350-4BE42FBB0D1F}.Debug|Any CPU.Build.0 = Debug|Any CPU 20 | {1A36BFE3-D89E-4EED-8350-4BE42FBB0D1F}.Release|Any CPU.ActiveCfg = Release|Any CPU 21 | {1A36BFE3-D89E-4EED-8350-4BE42FBB0D1F}.Release|Any CPU.Build.0 = Release|Any CPU 22 | {7A3130A2-E41B-46A2-A3FC-0C20BB42C853}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 23 | {7A3130A2-E41B-46A2-A3FC-0C20BB42C853}.Debug|Any CPU.Build.0 = Debug|Any CPU 24 | {7A3130A2-E41B-46A2-A3FC-0C20BB42C853}.Release|Any CPU.ActiveCfg = Release|Any CPU 25 | {7A3130A2-E41B-46A2-A3FC-0C20BB42C853}.Release|Any CPU.Build.0 = Release|Any CPU 26 | {1A913534-CBB7-4F17-96BD-8F128F9FA4CD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 27 | {1A913534-CBB7-4F17-96BD-8F128F9FA4CD}.Debug|Any CPU.Build.0 = Debug|Any CPU 28 | {1A913534-CBB7-4F17-96BD-8F128F9FA4CD}.Release|Any CPU.ActiveCfg = Release|Any CPU 29 | {1A913534-CBB7-4F17-96BD-8F128F9FA4CD}.Release|Any CPU.Build.0 = Release|Any CPU 30 | EndGlobalSection 31 | GlobalSection(SolutionProperties) = preSolution 32 | HideSolutionNode = FALSE 33 | EndGlobalSection 34 | GlobalSection(ExtensibilityGlobals) = postSolution 35 | SolutionGuid = {9FE7B00D-B3C0-4C9D-B10E-1383651C12CC} 36 | EndGlobalSection 37 | EndGlobal 38 | -------------------------------------------------------------------------------- /DataflowPipelineBuilder/Assembly.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.CompilerServices; 2 | 3 | [assembly:InternalsVisibleTo("Test")] -------------------------------------------------------------------------------- /DataflowPipelineBuilder/BlockExtensions.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Threading.Tasks; 3 | using System.Threading.Tasks.Dataflow; 4 | 5 | namespace DataflowPipelineBuilder 6 | { 7 | public static class BlockExtensions 8 | { 9 | public static async Task> ReceiveAllAsync(this ISourceBlock source) 10 | { 11 | var batch = 12 | new BatchBlock(int.MaxValue); 13 | 14 | source.LinkTo(batch, new DataflowLinkOptions { PropagateCompletion = true }); 15 | 16 | return await batch.ReceiveAsync().ConfigureAwait(false); 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /DataflowPipelineBuilder/Builder.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks.Dataflow; 2 | 3 | namespace DataflowPipelineBuilder 4 | { 5 | public class Builder 6 | { 7 | public IBuilder Create(IPropagatorBlock target) => 8 | new StartBuilder(target); 9 | 10 | public IBuilder Create() => 11 | new StartBuilder(new BufferBlock()); 12 | } 13 | } -------------------------------------------------------------------------------- /DataflowPipelineBuilder/BuilderExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using System.Threading.Tasks.Dataflow; 6 | 7 | namespace DataflowPipelineBuilder 8 | { 9 | public static class BuilderExtensions 10 | { 11 | public static IBuilder> Batch 12 | ( 13 | this IBuilder builder, 14 | int size 15 | ) => builder.Then(new BatchBlock(size)); 16 | 17 | public static IBuilder SelectMany 18 | ( 19 | this IBuilder> builder 20 | ) => builder.Then(new TransformManyBlock, T>(enumerable => enumerable)); 21 | 22 | public static IBuilder Select 23 | ( 24 | this IBuilder builder, 25 | Func selector 26 | ) => builder.Then(new TransformBlock(selector)); 27 | 28 | // This will buffer all the data and call complete 29 | // what happens if at this stage the other blocks 30 | // are not linked yet? 31 | // Change the test BuilderTest.Fork() to use this 32 | // and you will get only one output, fix the issue. 33 | // We could potentially keep the information in the 34 | // builder and once people call End() then we complete 35 | // the initial source. 36 | public static IBuilder FromEnumerable(this Builder builder, IEnumerable source) 37 | { 38 | var buffer = new BufferBlock(); 39 | 40 | Task.Run(() => Task.WhenAll(source.Select(buffer.SendAsync)) 41 | .Then(() => buffer.Complete())); 42 | 43 | return builder.Create(buffer); 44 | } 45 | 46 | public static IBuilder> 47 | Fork 48 | ( 49 | this IBuilder source, 50 | Func, IBuilder> leftBranch, 51 | Func, IBuilder> rightBranch 52 | ) => source.Fork() 53 | .Then(leftBranch, rightBranch); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /DataflowPipelineBuilder/DataflowPipelineBuilder.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | latest 6 | true 7 | Stefano d'Antonio 8 | Stefano d'Antonio 9 | Stefano d'Antonio 10 | https://github.com/UnoSD/DataflowPipelineBuilder 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /DataflowPipelineBuilder/ForkBuilder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks.Dataflow; 3 | 4 | namespace DataflowPipelineBuilder 5 | { 6 | public class ForkBuilder : IForkBuilder 7 | { 8 | readonly ITargetBlock _start; 9 | readonly ISourceBlock _current; 10 | 11 | public ForkBuilder(ITargetBlock start, ISourceBlock current) 12 | { 13 | _start = start; 14 | _current = current; 15 | } 16 | 17 | public IBuilder> 18 | Then 19 | ( 20 | Func, IBuilder> leftBranch, 21 | Func, IBuilder> rightBranch 22 | ) 23 | { 24 | var broadcastBlock = new BroadcastBlock(i => i); 25 | 26 | _current.LinkTo(broadcastBlock, new DataflowLinkOptions { PropagateCompletion = true }); 27 | 28 | var broadcastBuilder = new MiddleBuilder(_start, broadcastBlock); 29 | 30 | var left = leftBranch(broadcastBuilder); 31 | var right = rightBranch(broadcastBuilder); 32 | 33 | var join = new JoinBlock(); 34 | 35 | left.End().LinkTo(join.Target1, new DataflowLinkOptions { PropagateCompletion = true }); 36 | right.End().LinkTo(join.Target2, new DataflowLinkOptions { PropagateCompletion = true }); 37 | 38 | return new MiddleBuilder>(_start, join); 39 | } 40 | } 41 | } -------------------------------------------------------------------------------- /DataflowPipelineBuilder/IBuilder.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks.Dataflow; 2 | 3 | namespace DataflowPipelineBuilder 4 | { 5 | public interface IBuilder 6 | { 7 | IBuilder Then(IPropagatorBlock block); 8 | 9 | IForkBuilder Fork(); 10 | 11 | IPropagatorBlock End(); 12 | } 13 | } -------------------------------------------------------------------------------- /DataflowPipelineBuilder/IForkBuilder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace DataflowPipelineBuilder 4 | { 5 | public interface IForkBuilder 6 | { 7 | IBuilder> Then 8 | ( 9 | Func, IBuilder> leftBranch, 10 | Func, IBuilder> rightBranch 11 | ); 12 | } 13 | } -------------------------------------------------------------------------------- /DataflowPipelineBuilder/MiddleBuilder.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks.Dataflow; 2 | 3 | namespace DataflowPipelineBuilder 4 | { 5 | public class MiddleBuilder : IBuilder 6 | { 7 | readonly ITargetBlock _start; 8 | readonly ISourceBlock _current; 9 | 10 | internal MiddleBuilder(ITargetBlock start, ISourceBlock current) 11 | { 12 | _start = start; 13 | _current = current; 14 | } 15 | 16 | public IBuilder Then(IPropagatorBlock block) 17 | { 18 | _current.LinkTo(block, new DataflowLinkOptions { PropagateCompletion = true }); 19 | 20 | return new MiddleBuilder(_start, block); 21 | } 22 | 23 | public IForkBuilder Fork() => new ForkBuilder(_start, _current); 24 | 25 | public IPropagatorBlock End() => 26 | new WrapperBlock(_start, _current); 27 | } 28 | } -------------------------------------------------------------------------------- /DataflowPipelineBuilder/StartBuilder.cs: -------------------------------------------------------------------------------- 1 |  2 | using System.Threading.Tasks.Dataflow; 3 | 4 | namespace DataflowPipelineBuilder 5 | { 6 | public class StartBuilder : IBuilder 7 | { 8 | readonly IPropagatorBlock _start; 9 | 10 | internal StartBuilder(IPropagatorBlock start) => _start = start; 11 | 12 | public IBuilder Then(IPropagatorBlock block) 13 | { 14 | _start.LinkTo(block, new DataflowLinkOptions { PropagateCompletion = true }); 15 | 16 | return new MiddleBuilder(_start, block); 17 | } 18 | 19 | public IForkBuilder Fork() => 20 | new ForkBuilder(_start, _start); 21 | 22 | public IPropagatorBlock End() => _start; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /DataflowPipelineBuilder/TaskExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Threading.Tasks; 4 | 5 | namespace DataflowPipelineBuilder 6 | { 7 | static class TaskExtensions 8 | { 9 | internal static async Task Then(this Task task, Action action) 10 | { 11 | await task.ConfigureAwait(false); 12 | 13 | action(); 14 | } 15 | 16 | internal static Task WhenAll(this IEnumerable> tasks) => 17 | Task.WhenAll(tasks); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /DataflowPipelineBuilder/WrapperBlock.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using System.Threading.Tasks.Dataflow; 4 | 5 | namespace DataflowPipelineBuilder 6 | { 7 | class WrapperBlock : IPropagatorBlock 8 | { 9 | readonly ISourceBlock _output; 10 | readonly ITargetBlock _input; 11 | 12 | public WrapperBlock(ITargetBlock input, ISourceBlock output) 13 | { 14 | _output = output; 15 | _input = input; 16 | } 17 | 18 | public DataflowMessageStatus OfferMessage(DataflowMessageHeader messageHeader, TSource messageValue, ISourceBlock source, bool consumeToAccept) => 19 | _input.OfferMessage(messageHeader, messageValue, source, consumeToAccept); 20 | 21 | public void Complete() => _input.Complete(); 22 | 23 | public void Fault(Exception exception) => _input.Fault(exception); 24 | 25 | public Task Completion => _output.Completion; 26 | 27 | public IDisposable LinkTo(ITargetBlock target, DataflowLinkOptions linkOptions) => 28 | _output.LinkTo(target, linkOptions); 29 | 30 | public TTarget ConsumeMessage(DataflowMessageHeader messageHeader, ITargetBlock target, out bool messageConsumed) => 31 | _output.ConsumeMessage(messageHeader, target, out messageConsumed); 32 | 33 | public bool ReserveMessage(DataflowMessageHeader messageHeader, ITargetBlock target) => 34 | _output.ReserveMessage(messageHeader, target); 35 | 36 | public void ReleaseReservation(DataflowMessageHeader messageHeader, ITargetBlock target) => 37 | _output.ReleaseReservation(messageHeader, target); 38 | } 39 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DataflowPipelineBuilder 2 | Build TPL Dataflow pipelines using a fluent API and helpful extensions 3 | 4 | # Rationale 5 | TPL Dataflow is a powerful library, but the code is far less readable without the help of a diagram, this library should help visualize the pipeline in code, too. See the last example (Simple extensions) for the pipeline in code that reseables a diagram. 6 | 7 | # NuGet 8 | https://www.nuget.org/packages/DataflowPipelineBuilder 9 | 10 | # Using LINQ-like extension syntax 11 | 12 | ```csharp 13 | var builder = 14 | new Builder(); 15 | 16 | var pipelineBlock = 17 | builder.FromEnumerable(Enumerable.Repeat("{ }", 10)) 18 | .Select(JObject.Parse) 19 | .Batch(3) 20 | .SelectMany() 21 | .Select(j => j) 22 | .Batch(3) 23 | .End(); 24 | 25 | var actual = 26 | await pipelineBlock.ReceiveAllAsync(); 27 | 28 | Assert.That(actual.Count, Is.EqualTo(4)); 29 | Assert.That(actual.Count(b => b.Count == 3), Is.EqualTo(3)); 30 | Assert.That(actual.Count(b => b.Count == 1), Is.EqualTo(1)); 31 | ``` 32 | # Using blocks 33 | 34 | ```csharp 35 | var builder = 36 | new Builder(); 37 | 38 | var pipelineBlock = 39 | builder.Create(new BufferBlock()) 40 | .Then(new TransformBlock(json => JObject.Parse(json))) 41 | .Then(new BatchBlock(3)) 42 | .End(); 43 | 44 | await Task.WhenAll(Enumerable.Repeat("{ }", 10) 45 | .Select(pipelineBlock.SendAsync)); 46 | 47 | pipelineBlock.Complete(); 48 | 49 | var actual = 50 | await pipelineBlock.ReceiveAllAsync(); 51 | 52 | Assert.That(actual.Count, Is.EqualTo(4)); 53 | Assert.That(actual.Count(b => b.Length == 3), Is.EqualTo(3)); 54 | Assert.That(actual.Count(b => b.Length == 1), Is.EqualTo(1)); 55 | ``` 56 | # Fork and join 57 | 58 | ```csharp 59 | var builder = 60 | new Builder(); 61 | 62 | var pipelineBlock = 63 | builder.Create() 64 | .Select(JObject.Parse) 65 | .Fork(source => source.Select(s => s.Properties().First().Name), 66 | source => source.Select(s => s.Properties().First().Value.Value())) 67 | .End(); 68 | 69 | await 70 | Enumerable.Range(1, 3) 71 | .Select(index => $@"{{ ""value{index}"": {index} }}") 72 | .Select(pipelineBlock.SendAsync) 73 | .WhenAll() 74 | .Then(() => pipelineBlock.Complete()); 75 | 76 | var actual = 77 | await pipelineBlock.ReceiveAllAsync(); 78 | 79 | Assert.That(actual.Count, Is.EqualTo(3)); 80 | Assert.That(actual.Select(t => t.Item1), Is.EquivalentTo(new [] { "value1", "value2", "value3" })); 81 | Assert.That(actual.Select(t => t.Item2), Is.EquivalentTo(new [] { 1, 2, 3 })); 82 | ``` 83 | # Simple extensions 84 | 85 | Alternatively, the project `SimpleDataflowPipelineBuilder` contains extension methods applying to the dataflow blocks directly without the help of a builder (this doesn't support added logging and custom options applied to all links, but it's simple and works well in most cases). 86 | 87 | ```csharp 88 | var input = BufferBlock(token); 89 | 90 | var output = 91 | input.Then(ParseDataBlock()) 92 | .ForkJoinThen(source => source.Then(BatchBlock()) 93 | .Then(ApiCallToExternalResourceBlock()) 94 | .Then(SelectManyBlock()), 95 | source => source.Then(ApiCallToExternalResourceThatDoesNotAcceptBatches())) 96 | .Then(JoinResultsBlock()) 97 | .Then(WriteToStorageBlock()); 98 | ``` 99 | -------------------------------------------------------------------------------- /SimpleDataflowPipelineBuilder/BlocksExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks.Dataflow; 3 | 4 | namespace SimpleDataflowPipelineBuilder 5 | { 6 | public static class BlocksExtensions 7 | { 8 | static readonly DataflowLinkOptions LinkOptions = 9 | new DataflowLinkOptions { PropagateCompletion = true }; 10 | 11 | public static ISourceBlock Then 12 | ( 13 | this ISourceBlock source, 14 | IPropagatorBlock target 15 | ) 16 | { 17 | source.LinkTo(target, LinkOptions); 18 | 19 | return target; 20 | } 21 | 22 | public static IDataflowBlock Then 23 | ( 24 | this ISourceBlock source, 25 | ITargetBlock target 26 | ) 27 | { 28 | source.LinkTo(target, LinkOptions); 29 | 30 | return target; 31 | } 32 | 33 | public static ISourceBlock> ForkJoinThen 34 | ( 35 | this ISourceBlock source, 36 | Func, ISourceBlock> left, 37 | Func, ISourceBlock> right 38 | ) 39 | { 40 | var fork = 41 | new BroadcastBlock(data => data, 42 | new DataflowBlockOptions 43 | { 44 | NameFormat = "Fork {1}" 45 | }); 46 | 47 | source.LinkTo(fork, LinkOptions); 48 | 49 | var join = new JoinBlock(); 50 | 51 | left(fork).LinkTo(join.Target1, LinkOptions); 52 | 53 | right(fork).LinkTo(join.Target2, LinkOptions); 54 | 55 | return join; 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /SimpleDataflowPipelineBuilder/SimpleDataflowPipelineBuilder.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /Test/BuilderTest.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using System.Threading.Tasks; 3 | using System.Threading.Tasks.Dataflow; 4 | using DataflowPipelineBuilder; 5 | using Newtonsoft.Json.Linq; 6 | using NUnit.Framework; 7 | 8 | namespace Test 9 | { 10 | [TestFixture] 11 | class BuilderTest 12 | { 13 | [Test] 14 | public async Task MixedPipeline() 15 | { 16 | var builder = 17 | new Builder(); 18 | 19 | var pipelineBlock = 20 | builder.FromEnumerable(Enumerable.Repeat("{ }", 10)) 21 | .Select(JObject.Parse) 22 | .Batch(3) 23 | .SelectMany() 24 | .Select(j => j) 25 | .Batch(3) 26 | .End(); 27 | 28 | var actual = 29 | await pipelineBlock.ReceiveAllAsync(); 30 | 31 | Assert.That(actual.Count, Is.EqualTo(4)); 32 | Assert.That(actual.Count(b => b.Count == 3), Is.EqualTo(3)); 33 | Assert.That(actual.Count(b => b.Count == 1), Is.EqualTo(1)); 34 | } 35 | 36 | [Test] 37 | public async Task Fork() 38 | { 39 | var builder = 40 | new Builder(); 41 | 42 | var pipelineBlock = 43 | builder.Create() 44 | .Select(JObject.Parse) 45 | .Fork(source => source.Select(s => s.Properties().First().Name), 46 | source => source.Select(s => s.Properties().First().Value.Value())) 47 | .End(); 48 | 49 | await 50 | Enumerable.Range(1, 3) 51 | .Select(index => $@"{{ ""value{index}"": {index} }}") 52 | .Select(pipelineBlock.SendAsync) 53 | .WhenAll() 54 | .Then(() => pipelineBlock.Complete()); 55 | 56 | var actual = 57 | await pipelineBlock.ReceiveAllAsync(); 58 | 59 | Assert.That(actual.Count, Is.EqualTo(3)); 60 | Assert.That(actual.Select(t => t.Item1), Is.EquivalentTo(new [] { "value1", "value2", "value3" })); 61 | Assert.That(actual.Select(t => t.Item2), Is.EquivalentTo(new [] { 1, 2, 3 })); 62 | } 63 | 64 | [Test] 65 | public async Task MixedPipelineBlocks() 66 | { 67 | var builder = 68 | new Builder(); 69 | 70 | var pipelineBlock = 71 | builder.Create(new BufferBlock()) 72 | .Then(new TransformBlock(json => JObject.Parse(json))) 73 | .Then(new BatchBlock(3)) 74 | .End(); 75 | 76 | await Task.WhenAll(Enumerable.Repeat("{ }", 10) 77 | .Select(pipelineBlock.SendAsync)); 78 | 79 | pipelineBlock.Complete(); 80 | 81 | var actual = 82 | await pipelineBlock.ReceiveAllAsync(); 83 | 84 | Assert.That(actual.Count, Is.EqualTo(4)); 85 | Assert.That(actual.Count(b => b.Length == 3), Is.EqualTo(3)); 86 | Assert.That(actual.Count(b => b.Length == 1), Is.EqualTo(1)); 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /Test/Test.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.1 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | --------------------------------------------------------------------------------