├── .gitignore ├── .nuget ├── NuGet.Config ├── NuGet.exe └── NuGet.targets ├── .vscode ├── launch.json └── tasks.json ├── Gridsum.DataflowEx.Demo ├── AggregatorFlow.cs ├── App.config ├── CircularFlow.cs ├── ComplexIntFlow.cs ├── ForStackoverflow │ └── CompletionDemo1.cs ├── Gridsum.DataflowEx.Demo.csproj ├── LineAggregatorFlow.cs ├── LocalDBUtil.cs ├── MyLogger.cs ├── NLog.config ├── NLog.xsd ├── Order.cs ├── PeopleFlow.cs ├── Program.cs └── SlowFlow.cs ├── Gridsum.DataflowEx.Test ├── App.config ├── BlockExtensionTest.cs ├── DatabaseTests │ ├── BulkDataReaderTest.cs │ ├── BulkInserterTest.cs │ ├── DbColumnPathTest.cs │ ├── ExternalMappingTest.cs │ ├── GSProduct.cs │ ├── NestClassTest.cs │ ├── ReferenceLoopDepthTest.cs │ └── TestBootstrapper.cs ├── DataflowFaultTest.cs ├── DataflowTest.cs ├── Demo │ ├── Order.cs │ ├── PeopleFlow.cs │ └── SlowFlow.cs ├── ETL │ ├── TestDataJoiner.cs │ └── TestDataJoiner2.cs ├── Gridsum.DataflowEx.Test.csproj ├── TestAutoComplete.cs ├── TestBlockBehaviors.cs ├── TestDataBroadcaster.cs ├── TestFlowBlocking.cs ├── TestImmutable.cs ├── TestLogReader.cs ├── TestStatisticsRecorder.cs ├── TestStringCondition.cs └── TestUtils.cs ├── Gridsum.DataflowEx.TestHelper ├── App.config ├── Gridsum.DataflowEx.TestHelper.csproj └── Program.cs ├── Gridsum.DataflowEx.sln ├── Gridsum.DataflowEx ├── AutoCompletion │ ├── AutoCompleteWrapper.cs │ ├── ControlItem.cs │ ├── ControlType.cs │ ├── HeartbeatNode.cs │ ├── IRingNode.cs │ ├── ITracableItem.cs │ ├── RingMonitor.cs │ └── TracableItemBase.cs ├── Blocks │ └── TransformAndLinkPropagator.cs ├── DataBroadcaster.cs ├── DataDispatcher.cs ├── Databases │ ├── BulkDataReader.cs │ ├── DBColumnMapping.cs │ ├── DBColumnPath.cs │ ├── DbBulkInserter.cs │ ├── DbBulkInserterBase.cs │ ├── EagerDbBulkInserter.cs │ ├── InvalidDBColumnMappingException.cs │ ├── MultiDbBulkInserter.cs │ ├── NoNullCheckAttribute.cs │ ├── PostBulkInsertDelegate.cs │ ├── TargetTable.cs │ └── TypeAccessor.cs ├── Dataflow.cs ├── DataflowBlockExtensions.cs ├── DataflowMerger.cs ├── DataflowOptions.cs ├── DataflowUtils.cs ├── ETL │ ├── ByteArrayEqualityComparer.cs │ ├── DbDataJoiner.cs │ ├── IDimRow.cs │ ├── PartialDimRow.cs │ └── RowCache.cs ├── Exceptions │ ├── NoChildRegisteredException.cs │ ├── PostToBlockFailedException.cs │ ├── PropagatedException.cs │ └── TaskEx.cs ├── Gridsum.DataflowEx.csproj ├── IDataflow.cs ├── IDataflowDependency.cs ├── ImmutableUtils.cs ├── IntHolder.cs ├── LogHelper.cs ├── LogReader.cs ├── PatternMatch │ ├── IMatchCondition.cs │ ├── IMatchable.cs │ ├── MatchConditionBase.cs │ ├── MatchNoneCondition.cs │ ├── MatchType.cs │ ├── MultiMatcher.cs │ ├── StringMatchCondition.cs │ └── UrlStringMatchCondition.cs ├── PropagatorDataflow.cs ├── StatisticsRecorder.cs ├── TargetDataflow.cs └── Utils.cs ├── License.txt ├── README.md ├── global.json └── images ├── MyLoggerDemo.png ├── dbbulkinserter_screenshot1.png ├── dbbulkinserter_screenshot2.png ├── lookupDemo1.png └── lookupDemo2.png /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # Nuget packages 5 | *.nupkg 6 | 7 | # User-specific files 8 | *.suo 9 | *.user 10 | *.sln.docstates 11 | 12 | # Build results 13 | [Dd]ebug/ 14 | [Dd]ebugPublic/ 15 | [Rr]elease/ 16 | x64/ 17 | build/ 18 | bld/ 19 | [Bb]in/ 20 | [Oo]bj/ 21 | 22 | # MSTest test Results 23 | [Tt]est[Rr]esult*/ 24 | [Bb]uild[Ll]og.* 25 | 26 | #NUNIT 27 | *.VisualState.xml 28 | TestResult.xml 29 | 30 | # Build Results of an ATL Project 31 | [Dd]ebugPS/ 32 | [Rr]eleasePS/ 33 | dlldata.c 34 | 35 | *_i.c 36 | *_p.c 37 | *_i.h 38 | *.ilk 39 | *.meta 40 | *.obj 41 | *.pch 42 | *.pdb 43 | *.pgc 44 | *.pgd 45 | *.rsp 46 | *.sbr 47 | *.tlb 48 | *.tli 49 | *.tlh 50 | *.tmp 51 | *.tmp_proj 52 | *.log 53 | *.vspscc 54 | *.vssscc 55 | .builds 56 | *.pidb 57 | *.svclog 58 | *.scc 59 | 60 | # Chutzpah Test files 61 | _Chutzpah* 62 | 63 | # Visual C++ cache files 64 | ipch/ 65 | *.aps 66 | *.ncb 67 | *.opensdf 68 | *.sdf 69 | *.cachefile 70 | 71 | # Visual Studio profiler 72 | *.psess 73 | *.vsp 74 | *.vspx 75 | 76 | # TFS 2012 Local Workspace 77 | $tf/ 78 | 79 | # Guidance Automation Toolkit 80 | *.gpState 81 | 82 | # ReSharper is a .NET coding add-in 83 | _ReSharper*/ 84 | *.[Rr]e[Ss]harper 85 | *.DotSettings.user 86 | 87 | # JustCode is a .NET coding addin-in 88 | .JustCode 89 | 90 | # TeamCity is a build add-in 91 | _TeamCity* 92 | 93 | # DotCover is a Code Coverage Tool 94 | *.dotCover 95 | 96 | # NCrunch 97 | *.ncrunch* 98 | _NCrunch_* 99 | .*crunch*.local.xml 100 | 101 | # MightyMoose 102 | *.mm.* 103 | AutoTest.Net/ 104 | 105 | # Web workbench (sass) 106 | .sass-cache/ 107 | 108 | # Installshield output folder 109 | [Ee]xpress/ 110 | 111 | # DocProject is a documentation generator add-in 112 | DocProject/buildhelp/ 113 | DocProject/Help/*.HxT 114 | DocProject/Help/*.HxC 115 | DocProject/Help/*.hhc 116 | DocProject/Help/*.hhk 117 | DocProject/Help/*.hhp 118 | DocProject/Help/Html2 119 | DocProject/Help/html 120 | 121 | # Click-Once directory 122 | publish/ 123 | 124 | # Publish Web Output 125 | *.[Pp]ublish.xml 126 | *.azurePubxml 127 | 128 | # NuGet Packages Directory 129 | packages/ 130 | ## TODO: If the tool you use requires repositories.config uncomment the next line 131 | #!packages/repositories.config 132 | 133 | # Enable "build/" folder in the NuGet Packages folder since NuGet packages use it for MSBuild targets 134 | # This line needs to be after the ignore of the build folder (and the packages folder if the line above has been uncommented) 135 | !packages/build/ 136 | 137 | # Windows Azure Build Output 138 | csx/ 139 | *.build.csdef 140 | 141 | # Windows Store app package directory 142 | AppPackages/ 143 | 144 | # Others 145 | sql/ 146 | *.Cache 147 | ClientBin/ 148 | [Ss]tyle[Cc]op.* 149 | ~$* 150 | *~ 151 | *.dbmdl 152 | *.dbproj.schemaview 153 | *.pfx 154 | *.publishsettings 155 | node_modules/ 156 | 157 | # RIA/Silverlight projects 158 | Generated_Code/ 159 | 160 | # Backup & report files from converting an old project file to a newer 161 | # Visual Studio version. Backup files are not needed, because we have git ;-) 162 | _UpgradeReport_Files/ 163 | Backup*/ 164 | UpgradeLog*.XML 165 | UpgradeLog*.htm 166 | 167 | # SQL Server files 168 | *.mdf 169 | *.ldf 170 | 171 | # Business Intelligence projects 172 | *.rdl.data 173 | *.bim.layout 174 | *.bim_*.settings 175 | 176 | # Microsoft Fakes 177 | FakesAssemblies/ 178 | /.vs/Gridsum.DataflowEx/v15/sqlite3/storage.ide 179 | .vs/config/applicationhost.config 180 | -------------------------------------------------------------------------------- /.nuget/NuGet.Config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.nuget/NuGet.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gridsum/DataflowEx/552b0b45eeb4a513c4079a10d0bc57df4950448f/.nuget/NuGet.exe -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to find out which attributes exist for C# debugging 3 | // Use hover for the description of the existing attributes 4 | // For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": ".NET Core Launch (console)", 9 | "type": "coreclr", 10 | "request": "launch", 11 | "preLaunchTask": "build", 12 | // If you have changed target frameworks, make sure to update the program path. 13 | "program": "${workspaceRoot}/Gridsum.DataflowEx.Test/bin/Debug/netcoreapp2.0/Gridsum.DataflowEx.Test.dll", 14 | "args": [], 15 | "cwd": "${workspaceRoot}/Gridsum.DataflowEx.Test", 16 | // For more information about the 'console' field, see https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md#console-terminal-window 17 | "console": "internalConsole", 18 | "stopAtEntry": false, 19 | "internalConsoleOptions": "openOnSessionStart" 20 | }, 21 | { 22 | "name": ".NET Core Attach", 23 | "type": "coreclr", 24 | "request": "attach", 25 | "processId": "${command:pickProcess}" 26 | } 27 | ] 28 | } -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.1.0", 3 | "command": "dotnet", 4 | "isShellCommand": true, 5 | "args": [], 6 | "tasks": [ 7 | { 8 | "taskName": "build", 9 | "args": [ 10 | "${workspaceRoot}/Gridsum.DataflowEx.Test/Gridsum.DataflowEx.Test.csproj" 11 | ], 12 | "isBuildCommand": true, 13 | "problemMatcher": "$msCompile" 14 | } 15 | ] 16 | } -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Demo/AggregatorFlow.cs: -------------------------------------------------------------------------------- 1 | namespace Gridsum.DataflowEx.Demo 2 | { 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Threading.Tasks.Dataflow; 6 | 7 | using Gridsum.DataflowEx; 8 | 9 | public class AggregatorFlow : Dataflow 10 | { 11 | //Blocks 12 | private TransformBlock> _splitter; 13 | private ActionBlock> _aggregater; 14 | 15 | //Data 16 | private Dictionary _dict; 17 | 18 | public AggregatorFlow() : base(DataflowOptions.Default) 19 | { 20 | this._splitter = new TransformBlock>((Func>)this.Split); 21 | this._dict = new Dictionary(); 22 | this._aggregater = new ActionBlock>((Action>)this.Aggregate); 23 | 24 | //Block linking 25 | this._splitter.LinkTo(this._aggregater, new DataflowLinkOptions() { PropagateCompletion = true }); 26 | 27 | /* IMPORTANT */ 28 | this.RegisterChild(this._splitter); 29 | this.RegisterChild(this._aggregater); 30 | } 31 | 32 | protected virtual void Aggregate(KeyValuePair pair) 33 | { 34 | int oldValue; 35 | this._dict[pair.Key] = this._dict.TryGetValue(pair.Key, out oldValue) ? oldValue + pair.Value : pair.Value; 36 | } 37 | 38 | protected virtual KeyValuePair Split(string input) 39 | { 40 | string[] splitted = input.Split('='); 41 | return new KeyValuePair(splitted[0], int.Parse(splitted[1])); 42 | } 43 | 44 | public override ITargetBlock InputBlock { get { return this._splitter; } } 45 | 46 | public IDictionary Result { get { return this._dict; } } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Demo/App.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 |
6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Demo/CircularFlow.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.Demo 8 | { 9 | using System.Threading.Tasks.Dataflow; 10 | 11 | using Gridsum.DataflowEx.AutoCompletion; 12 | 13 | public class CircularFlow : Dataflow 14 | { 15 | private Dataflow _buffer; 16 | 17 | public CircularFlow(DataflowOptions dataflowOptions) : base(dataflowOptions) 18 | { 19 | //a no-op node to demonstrate the usage of preTask param in RegisterChildRing 20 | _buffer = new BufferBlock().ToDataflow(name: "NoOpBuffer"); 21 | 22 | var heater = new TransformManyDataflow(async i => 23 | { 24 | await Task.Delay(200); 25 | Console.WriteLine("Heated to {0}", i + 1); 26 | return new [] {i + 1}; 27 | }, dataflowOptions); 28 | heater.Name = "Heater"; 29 | 30 | var cooler = new TransformManyDataflow(async i => 31 | { 32 | await Task.Delay(200); 33 | int cooled = i - 2; 34 | Console.WriteLine("Cooled to {0}", cooled); 35 | 36 | if (cooled < 0) //time to stop 37 | { 38 | return Enumerable.Empty(); 39 | } 40 | 41 | return new [] {cooled}; 42 | }, dataflowOptions); 43 | cooler.Name = "Cooler"; 44 | 45 | var heartbeat = new HeartbeatNode(dataflowOptions) {Name = "HeartBeat"}; 46 | 47 | _buffer.LinkTo(heater); 48 | 49 | //circular 50 | heater.LinkTo(cooler); 51 | cooler.LinkTo(heartbeat); 52 | heartbeat.LinkTo(heater); 53 | 54 | RegisterChildren(_buffer, heater, cooler, heartbeat); 55 | 56 | //ring registration 57 | RegisterChildRing(_buffer.CompletionTask, heater, cooler, heartbeat); 58 | } 59 | 60 | public override ITargetBlock InputBlock { get { return _buffer.InputBlock; } } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Demo/ComplexIntFlow.cs: -------------------------------------------------------------------------------- 1 | namespace Gridsum.DataflowEx.Demo 2 | { 3 | using System; 4 | using System.Threading.Tasks; 5 | using System.Threading.Tasks.Dataflow; 6 | using Gridsum.DataflowEx; 7 | 8 | public class ComplexIntFlow : Dataflow 9 | { 10 | private ITargetBlock _headBlock; 11 | public ComplexIntFlow() : base(DataflowOptions.Default) 12 | { 13 | Dataflow node2 = DataflowUtils.FromDelegate(i => i); 14 | Dataflow node3 = DataflowUtils.FromDelegate(i => i * -1); 15 | 16 | Dataflow node1 = DataflowUtils.FromDelegate( 17 | i => { 18 | if (i % 2 == 0) { node2.Post(i); } 19 | else { node3.Post(i); } 20 | return 999; 21 | }); 22 | 23 | Dataflow printer = DataflowUtils.FromDelegate(Console.WriteLine); 24 | 25 | node1.Name = "node1"; 26 | node2.Name = "node2"; 27 | node3.Name = "node3"; 28 | printer.Name = "printer"; 29 | 30 | this.RegisterChild(node1); 31 | this.RegisterChild(node2); 32 | this.RegisterChild(node3); 33 | this.RegisterChild(printer, t => 34 | { 35 | if (t.Status == TaskStatus.RanToCompletion) 36 | Console.WriteLine("Printer done!"); 37 | }); 38 | 39 | node1.LinkTo(printer); 40 | node2.LinkTo(printer); 41 | node3.LinkTo(printer); 42 | 43 | //Completion propagation: node1 ---> node2 44 | node2.RegisterDependency(node1); 45 | //Completion propagation: node1 + node2 ---> node3 46 | node3.RegisterDependency(node1); 47 | node3.RegisterDependency(node2); 48 | 49 | this._headBlock = node1.InputBlock; 50 | } 51 | 52 | public override ITargetBlock InputBlock { get { return this._headBlock; } } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Demo/ForStackoverflow/CompletionDemo1.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.Demo.ForStackoverflow 8 | { 9 | using System.Threading; 10 | using System.Threading.Tasks.Dataflow; 11 | 12 | /// 13 | /// For http://stackoverflow.com/questions/13510094/tpl-dataflow-guarantee-completion-only-when-all-source-data-blocks-completed 14 | /// 15 | public class CompletionDemo1 16 | { 17 | private Dataflow broadCaster; 18 | private TransformBlock transformBlock1; 19 | private TransformBlock transformBlock2; 20 | private Dataflow processor; 21 | 22 | public CompletionDemo1() 23 | { 24 | broadCaster = new BroadcastBlock( 25 | i => 26 | { 27 | return i; 28 | }).ToDataflow(); 29 | 30 | transformBlock1 = new TransformBlock( 31 | i => 32 | { 33 | Console.WriteLine("1 input count: " + transformBlock1.InputCount); 34 | Thread.Sleep(50); 35 | return ("1_" + i); 36 | }); 37 | 38 | transformBlock2 = new TransformBlock( 39 | i => 40 | { 41 | Console.WriteLine("2 input count: " + transformBlock2.InputCount); 42 | Thread.Sleep(20); 43 | return ("2_" + i); 44 | }); 45 | 46 | processor = new ActionBlock( 47 | i => 48 | { 49 | Console.WriteLine(i); 50 | }).ToDataflow(); 51 | 52 | /** rather than TPL linking 53 | broadCastBlock.LinkTo(transformBlock1, new DataflowLinkOptions { PropagateCompletion = true }); 54 | broadCastBlock.LinkTo(transformBlock2, new DataflowLinkOptions { PropagateCompletion = true }); 55 | transformBlock1.LinkTo(processorBlock, new DataflowLinkOptions { PropagateCompletion = true }); 56 | transformBlock2.LinkTo(processorBlock, new DataflowLinkOptions { PropagateCompletion = true }); 57 | **/ 58 | 59 | //Use DataflowEx linking 60 | var transform1 = transformBlock1.ToDataflow(); 61 | var transform2 = transformBlock2.ToDataflow(); 62 | 63 | broadCaster.LinkTo(transform1); 64 | broadCaster.LinkTo(transform2); 65 | transform1.LinkTo(processor); 66 | transform2.LinkTo(processor); 67 | } 68 | 69 | public void Start() 70 | { 71 | const int numElements = 100; 72 | 73 | for (int i = 1; i <= numElements; i++) 74 | { 75 | broadCaster.SendAsync(i); 76 | } 77 | 78 | //mark completion 79 | broadCaster.Complete(); 80 | 81 | processor.CompletionTask.Wait(); 82 | 83 | Console.WriteLine("Finished"); 84 | Console.ReadLine(); 85 | } 86 | 87 | /* 88 | public static void Main() 89 | { 90 | var demo = new CompletionDemo1(); 91 | demo.Start(); 92 | } 93 | */ 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Demo/Gridsum.DataflowEx.Demo.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.0 5 | 6 | Exe 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Demo/LineAggregatorFlow.cs: -------------------------------------------------------------------------------- 1 | namespace Gridsum.DataflowEx.Demo 2 | { 3 | using System.Threading.Tasks.Dataflow; 4 | using Gridsum.DataflowEx; 5 | 6 | public class LineAggregatorFlow : Dataflow 7 | { 8 | private Dataflow _lineProcessor; 9 | private AggregatorFlow _aggregator; 10 | public LineAggregatorFlow() : base(DataflowOptions.Default) 11 | { 12 | this._lineProcessor = new TransformManyBlock(line => line.Split(' ')).ToDataflow(); 13 | this._aggregator = new AggregatorFlow(); 14 | this._lineProcessor.LinkTo(this._aggregator); 15 | this.RegisterChild(this._lineProcessor); 16 | this.RegisterChild(this._aggregator); 17 | } 18 | 19 | public override ITargetBlock InputBlock { get { return this._lineProcessor.InputBlock; } } 20 | public int this[string key] { get { return this._aggregator.Result[key]; } } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Demo/LocalDBUtil.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.Demo 8 | { 9 | using System.Data.SqlClient; 10 | using System.IO; 11 | using System.Reflection; 12 | 13 | //Borrowed from https://social.msdn.microsoft.com/Forums/sqlserver/en-US/268c3411-102a-4272-b305-b14e29604313/localdb-create-connect-to-database-programmatically-?forum=sqlsetupandupgrade 14 | public static class LocalDB 15 | { 16 | public const string DB_DIRECTORY = "Data"; 17 | 18 | [Obsolete] 19 | public static SqlConnection GetLocalDB(string dbName, bool deleteIfExists = false) 20 | { 21 | try 22 | { 23 | string outputFolder = Path.Combine( 24 | Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), 25 | DB_DIRECTORY); 26 | string mdfFilename = dbName + ".mdf"; 27 | string dbFileName = Path.Combine(outputFolder, mdfFilename); 28 | string logFileName = Path.Combine(outputFolder, String.Format("{0}_log.ldf", dbName)); 29 | // Create Data Directory If It Doesn't Already Exist. 30 | if (!Directory.Exists(outputFolder)) 31 | { 32 | Directory.CreateDirectory(outputFolder); 33 | } 34 | 35 | // If the file exists, and we want to delete old data, remove it here and create a new database. 36 | if (File.Exists(dbFileName) && deleteIfExists) 37 | { 38 | if (File.Exists(logFileName)) File.Delete(logFileName); 39 | File.Delete(dbFileName); 40 | CreateDatabase(dbName, dbFileName); 41 | } 42 | // If the database does not already exist, create it. 43 | else if (!File.Exists(dbFileName)) 44 | { 45 | CreateDatabase(dbName, dbFileName); 46 | } 47 | 48 | // Open newly created, or old database. 49 | string connectionString = 50 | String.Format( 51 | @"Data Source=(LocalDB)\v11.0;AttachDBFileName={1};Initial Catalog={0};Integrated Security=True;", 52 | dbName, 53 | dbFileName); 54 | SqlConnection connection = new SqlConnection(connectionString); 55 | connection.Open(); 56 | return connection; 57 | } 58 | catch 59 | { 60 | throw; 61 | } 62 | } 63 | 64 | private static bool CreateDatabase(string dbName, string dbFileName) 65 | { 66 | try 67 | { 68 | string connectionString = 69 | String.Format(@"Data Source=(LocalDB)\v11.0;Initial Catalog=master;Integrated Security=True"); 70 | using (var connection = new SqlConnection(connectionString)) 71 | { 72 | connection.Open(); 73 | SqlCommand cmd = connection.CreateCommand(); 74 | 75 | 76 | DetachDatabase(dbName); 77 | 78 | cmd.CommandText = String.Format( 79 | "CREATE DATABASE {0} ON (NAME = N'{0}', FILENAME = '{1}')", 80 | dbName, 81 | dbFileName); 82 | cmd.ExecuteNonQuery(); 83 | } 84 | 85 | if (File.Exists(dbFileName)) return true; 86 | else return false; 87 | } 88 | catch 89 | { 90 | throw; 91 | } 92 | } 93 | 94 | private static bool DetachDatabase(string dbName) 95 | { 96 | try 97 | { 98 | string connectionString = 99 | String.Format(@"Data Source=(LocalDB)\v11.0;Initial Catalog=master;Integrated Security=True"); 100 | using (var connection = new SqlConnection(connectionString)) 101 | { 102 | connection.Open(); 103 | SqlCommand cmd = connection.CreateCommand(); 104 | cmd.CommandText = String.Format("exec sp_detach_db '{0}'", dbName); 105 | cmd.ExecuteNonQuery(); 106 | 107 | return true; 108 | } 109 | } 110 | catch 111 | { 112 | return false; 113 | } 114 | } 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Demo/MyLogger.cs: -------------------------------------------------------------------------------- 1 | using Common.Logging; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | 8 | namespace Gridsum.DataflowEx.Demo 9 | { 10 | using System.IO; 11 | using System.Threading.Tasks.Dataflow; 12 | 13 | public class MyLog 14 | { 15 | public LogLevel Level { get; set; } 16 | public string Message { get; set; } 17 | } 18 | 19 | /// 20 | /// Logger that accepts logs and dispatch them to appropriate dynamically created log writer 21 | /// 22 | public class MyLogger : DataDispatcher 23 | { 24 | public MyLogger() : base(log => log.Level) 25 | { 26 | } 27 | 28 | /// 29 | /// This function will only be called once for each distinct dispatchKey (the first time) 30 | /// 31 | protected override Dataflow CreateChildFlow(LogLevel dispatchKey) 32 | { 33 | //dynamically create a log writer by the dispatchKey (i.e. the log level) 34 | var writer = new LogWriter(string.Format(@".\MyLogger-{0}.log", dispatchKey)); 35 | 36 | //no need to call RegisterChild(writer) here as DataDispatcher will call automatically 37 | return writer; 38 | } 39 | } 40 | 41 | /// 42 | /// Log writer node for a single destination file 43 | /// 44 | internal class LogWriter : Dataflow 45 | { 46 | private readonly ActionBlock m_writerBlock; 47 | private readonly StreamWriter m_writer; 48 | 49 | public LogWriter(string fileName) : base(DataflowOptions.Default) 50 | { 51 | this.m_writer = new StreamWriter(new FileStream(fileName, FileMode.Append)); 52 | 53 | m_writerBlock = new ActionBlock(log => m_writer.WriteLine("[{0}] {1}", log.Level, log.Message)); 54 | 55 | RegisterChild(m_writerBlock); 56 | } 57 | 58 | public override ITargetBlock InputBlock { get { return m_writerBlock; } } 59 | 60 | protected override void CleanUp(Exception e) 61 | { 62 | base.CleanUp(e); 63 | m_writer.Flush(); 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Demo/NLog.config: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Demo/Order.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.Demo 8 | { 9 | using System.Linq.Expressions; 10 | 11 | using Gridsum.DataflowEx.Databases; 12 | using Gridsum.DataflowEx.ETL; 13 | 14 | public class Order 15 | { 16 | [DBColumnMapping("OrderTarget", "Date", null)] 17 | public DateTime OrderDate { get; set; } 18 | 19 | [DBColumnMapping("OrderTarget", "Value", 0.0f)] 20 | public float? OrderValue { get; set; } 21 | 22 | [DBColumnPath(DBColumnPathOptions.DoNotGenerateNullCheck)] 23 | public Person Customer { get; set; } 24 | } 25 | 26 | public class OrderEx : Order 27 | { 28 | public OrderEx() 29 | { 30 | this.OrderDate = DateTime.Now; 31 | } 32 | 33 | // This is the field we want to populate 34 | [DBColumnMapping("LookupDemo", "ProductKey")] 35 | [DBColumnMapping("OrderTarget", "ProductKey")] 36 | public long? ProductKey { get; set; } 37 | 38 | public Product Product { get; set; } 39 | } 40 | 41 | public class Product 42 | { 43 | public Product(string category, string name) 44 | { 45 | this.Category = category; 46 | this.Name = name; 47 | } 48 | 49 | [DBColumnMapping("LookupDemo", "Category")] 50 | public string Category { get; private set; } 51 | 52 | [DBColumnMapping("LookupDemo", "ProductName")] 53 | public string Name { get; private set; } 54 | 55 | [DBColumnMapping("LookupDemo", "ProductFullName")] 56 | public string FullName { get { return string.Format("{0}-{1}", Category, Name); } } 57 | } 58 | 59 | public class ProductLookupFlow : DbDataJoiner 60 | { 61 | public ProductLookupFlow(TargetTable dimTableTarget, int batchSize = 8192, int cacheSize = 1048576) 62 | : base( 63 | //Tells DbDataJoiner the join condition is 'OrderEx.Product.FullName = TABLE.ProductFullName' 64 | o => o.Product.FullName, 65 | dimTableTarget, DataflowOptions.Default, batchSize, cacheSize) 66 | { 67 | } 68 | 69 | protected override void OnSuccessfulLookup(OrderEx input, IDimRow rowInDimTable) 70 | { 71 | //what we want is just the key column of the dimension row 72 | input.ProductKey = rowInDimTable.AutoIncrementKey; 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Demo/PeopleFlow.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.Demo 8 | { 9 | using System.Threading; 10 | using System.Threading.Tasks.Dataflow; 11 | 12 | using Gridsum.DataflowEx.Databases; 13 | 14 | using Newtonsoft.Json; 15 | 16 | public class Person : IEventProvider 17 | { 18 | [DBColumnMapping("PersonTarget", "NameCol", "N/A", ColumnMappingOption.Mandatory)] 19 | [DBColumnMapping("OrderTarget", "CustomerName", "Unknown Customer", ColumnMappingOption.Optional)] 20 | public string Name { get; set; } 21 | 22 | [DBColumnMapping("PersonTarget", "AgeCol", -1, ColumnMappingOption.Optional)] 23 | [DBColumnMapping("OrderTarget", "CustomerAge", -1, ColumnMappingOption.Optional)] 24 | public int? Age { get; set; } 25 | 26 | public DataflowEvent GetEvent() 27 | { 28 | if (Age > 70) 29 | { 30 | return new DataflowEvent("OldPerson"); 31 | } 32 | else 33 | { 34 | //returning empty so it will not be recorded as an event 35 | return DataflowEvent.Empty; 36 | } 37 | } 38 | } 39 | 40 | public class PeopleFlow : Dataflow 41 | { 42 | private TransformBlock m_converter; 43 | private TransformBlock m_recorder; 44 | private StatisticsRecorder m_peopleRecorder; 45 | 46 | public PeopleFlow(DataflowOptions dataflowOptions) 47 | : base(dataflowOptions) 48 | { 49 | m_peopleRecorder = new StatisticsRecorder(this) { Name = "PeopleRecorder"}; 50 | 51 | m_converter = new TransformBlock(s => JsonConvert.DeserializeObject(s)); 52 | m_recorder = new TransformBlock( 53 | p => 54 | { 55 | //record every person 56 | m_peopleRecorder.Record(p); 57 | return p; 58 | }); 59 | 60 | m_converter.LinkTo(m_recorder, new DataflowLinkOptions() { PropagateCompletion = true}); 61 | 62 | RegisterChild(m_converter); 63 | RegisterChild(m_recorder); 64 | } 65 | 66 | public override ITargetBlock InputBlock { get { return m_converter; } } 67 | public override ISourceBlock OutputBlock { get { return m_recorder; } } 68 | public StatisticsRecorder PeopleRecorder { get { return m_peopleRecorder; } } 69 | } 70 | } 71 | 72 | 73 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Demo/SlowFlow.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.Demo 8 | { 9 | using System.Threading; 10 | using System.Threading.Tasks.Dataflow; 11 | 12 | public class SlowFlow : Dataflow 13 | { 14 | private Dataflow _splitter; 15 | private Dataflow _printer; 16 | 17 | public SlowFlow(DataflowOptions dataflowOptions) 18 | : base(dataflowOptions) 19 | { 20 | _splitter = new TransformManyBlock(new Func>(this.SlowSplit), 21 | dataflowOptions.ToExecutionBlockOption()) 22 | .ToDataflow(dataflowOptions, "SlowSplitter"); 23 | 24 | _printer = new ActionBlock(c => Console.WriteLine(c), 25 | dataflowOptions.ToExecutionBlockOption()) 26 | .ToDataflow(dataflowOptions, "Printer"); 27 | 28 | RegisterChild(_splitter); 29 | RegisterChild(_printer); 30 | 31 | _splitter.LinkTo(_printer); 32 | } 33 | 34 | private IEnumerable SlowSplit(string s) 35 | { 36 | foreach (var c in s) 37 | { 38 | Thread.Sleep(1000); //slow down 39 | yield return c; 40 | } 41 | } 42 | 43 | public override ITargetBlock InputBlock { get { return _splitter.InputBlock; } } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/App.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 |
7 | 8 |
9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/BlockExtensionTest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Threading; 4 | using System.Threading.Tasks; 5 | using System.Threading.Tasks.Dataflow; 6 | using Microsoft.VisualStudio.TestTools.UnitTesting; 7 | 8 | namespace Gridsum.DataflowEx.Test 9 | { 10 | [TestClass] 11 | public class BlockExtensionTest 12 | { 13 | [TestMethod] 14 | public async Task TestBlockBufferCount() 15 | { 16 | var block1 = new TransformBlock(i => 2 * i); 17 | var block2 = new TransformManyBlock(i => new [] { i }); 18 | var block3 = new ActionBlock(i => { Thread.Sleep(1000); }); 19 | 20 | block1.Post(0); 21 | block2.Post(0); 22 | block2.Post(0); 23 | block3.Post(0); 24 | block3.Post(0); 25 | block3.Post(0); 26 | 27 | await Task.Delay(200); 28 | 29 | Assert.AreEqual(1, block1.GetBufferCount().Total()); 30 | Assert.AreEqual(2, block2.GetBufferCount().Total()); 31 | Assert.AreEqual(2, block3.GetBufferCount().Total()); 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/DatabaseTests/DbColumnPathTest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.Test.DatabaseTests 8 | { 9 | using Dapper; 10 | 11 | using Gridsum.DataflowEx.Databases; 12 | using Gridsum.DataflowEx.Test.Demo; 13 | 14 | using Microsoft.VisualStudio.TestTools.UnitTesting; 15 | 16 | [TestClass] 17 | public class DbColumnPathTest 18 | { 19 | public class PersonEx 20 | { 21 | [DBColumnMapping("PersonTarget", "NameCol", "N/A", ColumnMappingOption.Optional)] 22 | public string Name { get; set; } 23 | 24 | [DBColumnMapping("PersonTarget", "AgeCol", -1, ColumnMappingOption.Optional)] 25 | public int? Age { get; set; } 26 | 27 | [DBColumnPath(DBColumnPathOptions.DoNotGenerateNullCheck)] 28 | [DBColumnPath(DBColumnPathOptions.DoNotExpand)] 29 | public Person Me { get; set; } 30 | } 31 | 32 | public class PersonEx2 33 | { 34 | [DBColumnMapping("PersonTarget", "NameCol", "N/A", ColumnMappingOption.Optional)] 35 | public string Name { get; set; } 36 | 37 | [DBColumnMapping("PersonTarget", "AgeCol", -1, ColumnMappingOption.Optional)] 38 | public int? Age { get; set; } 39 | 40 | [DBColumnPath(DBColumnPathOptions.DoNotGenerateNullCheck)] 41 | public Person Me { get; set; } 42 | } 43 | 44 | public class Person 45 | { 46 | [DBColumnMapping("PersonTarget", "NameCol", "N/A", ColumnMappingOption.Overwrite)] 47 | public string Name { get; set; } 48 | 49 | [DBColumnMapping("PersonTarget", "AgeCol", -1, ColumnMappingOption.Overwrite)] 50 | public int? Age { get; set; } 51 | } 52 | 53 | //should access level 1 54 | [TestMethod] 55 | public async Task TestDoNoExpand() 56 | { 57 | string connStr = ExternalMappingTest.CreatePeopleTable(4); 58 | 59 | var dbInserter = new DbBulkInserter(connStr, "dbo.People", DataflowOptions.Default, "PersonTarget"); 60 | 61 | dbInserter.Post(new PersonEx { Age = 10, Name = "Aaron", Me = new Person { Age = 20, Name = "Bob" } }); 62 | await dbInserter.SignalAndWaitForCompletionAsync(); 63 | 64 | using (var conn = TestUtils.GetLocalDB("ExternalMappingTest4")) 65 | { 66 | Assert.AreEqual(1, conn.ExecuteScalar("select count(*) from dbo.People")); 67 | Assert.AreEqual(1, conn.ExecuteScalar("select count(*) from dbo.People where NameCol = 'Aaron'")); 68 | Assert.AreEqual(1, conn.ExecuteScalar("select count(*) from dbo.People where AgeCol = 10")); 69 | } 70 | } 71 | 72 | //should access level 2 73 | [TestMethod] 74 | public async Task TestDoNoExpand2() 75 | { 76 | string connStr = ExternalMappingTest.CreatePeopleTable(4); 77 | 78 | var dbInserter = new DbBulkInserter(connStr, "dbo.People", DataflowOptions.Default, "PersonTarget"); 79 | 80 | dbInserter.Post(new PersonEx2 { Age = 10, Name = "Aaron", Me = new Person { Age = 20, Name = "Bob" } }); 81 | await dbInserter.SignalAndWaitForCompletionAsync(); 82 | 83 | using (var conn = TestUtils.GetLocalDB("ExternalMappingTest4")) 84 | { 85 | Assert.AreEqual(1, conn.ExecuteScalar("select count(*) from dbo.People")); 86 | Assert.AreEqual(1, conn.ExecuteScalar("select count(*) from dbo.People where NameCol = 'Bob'")); 87 | Assert.AreEqual(1, conn.ExecuteScalar("select count(*) from dbo.People where AgeCol = 20")); 88 | } 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/DatabaseTests/GSProduct.cs: -------------------------------------------------------------------------------- 1 | namespace Gridsum.DataflowEx.Test.DatabaseTests 2 | { 3 | class GSProduct 4 | { 5 | public const string WD = ("WD"); 6 | public const string CD = ("CD"); 7 | public const string SEMD = ("SEMD"); 8 | public const string MD = ("MD"); 9 | public const string VD = ("VD"); 10 | public const string SD = ("SD"); 11 | public const string SEOD = ("SEOD"); 12 | public const string AD = ("AD"); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/DatabaseTests/NestClassTest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Data.SqlClient; 3 | using System.Linq; 4 | using Gridsum.DataflowEx.Databases; 5 | using Microsoft.EntityFrameworkCore; 6 | using Microsoft.VisualStudio.TestTools.UnitTesting; 7 | 8 | namespace Gridsum.DataflowEx.Test.DatabaseTests 9 | { 10 | public class A 11 | { 12 | public int Id { get; set; } 13 | 14 | [DBColumnMapping("NestClassTest", "Value1", "-")] 15 | public string Value1 { get; set; } 16 | } 17 | 18 | public class AEF : A 19 | { 20 | public string Value2 { get; set; } 21 | } 22 | 23 | public class ABulk : A 24 | { 25 | public B B { get; set; } 26 | } 27 | 28 | public class B 29 | { 30 | [DBColumnMapping("NestClassTest", "Value2", "--")] 31 | public string Value2 { get; set; } 32 | } 33 | 34 | [TestClass] 35 | public class NestClassTest 36 | { 37 | [TestMethod] 38 | public void TestNestClass() 39 | { 40 | var connectString = TestUtils.GetLocalDBConnectionString(); 41 | var context = new NestContext(connectString); 42 | context.NestInits.Add(new NestInit()); 43 | context.SaveChanges(); 44 | 45 | var a1 = new ABulk {}; 46 | var a2 = new ABulk {B = new B { } }; 47 | var a3 = new ABulk {B = new B { Value2 = "a" } }; 48 | var a4 = new ABulk {Value1 = "a", B = new B { Value2 = "a" } }; 49 | 50 | using (var connection = new SqlConnection(connectString)) 51 | { 52 | connection.Open(); 53 | var aTypeAccessor = TypeAccessorManager.GetAccessorForTable(new TargetTable("NestClassTest", connectString, "dbo.AAs")); 54 | var aReader = new BulkDataReader(aTypeAccessor, new[] { a1, a2, a3, a4 }); 55 | using (var aCopy = new SqlBulkCopy(connection)) 56 | { 57 | foreach (var map in aReader.ColumnMappings) 58 | { 59 | aCopy.ColumnMappings.Add(map); 60 | } 61 | aCopy.DestinationTableName = "dbo.AAs"; 62 | aCopy.WriteToServer(aReader); 63 | } 64 | } 65 | 66 | Assert.AreEqual(1, context.AAs.FirstOrDefault().Id); 67 | Assert.AreEqual(1, context.AAs.Count(a => a.Value1 == "a")); 68 | Assert.AreEqual(3, context.AAs.Count(a => a.Value1 == "-")); 69 | Assert.AreEqual(2, context.AAs.Count(a => a.Value2 == "a")); 70 | Assert.AreEqual(2, context.AAs.Count(a => a.Value2 == "--")); 71 | } 72 | } 73 | 74 | public class NestContext : DbContextBase 75 | { 76 | public NestContext(string config) 77 | : base(config) 78 | { 79 | } 80 | 81 | public DbSet NestInits { get; set; } 82 | public DbSet AAs { get; set; } 83 | } 84 | 85 | public class NestInit 86 | { 87 | public int Id { get; set; } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/DatabaseTests/ReferenceLoopDepthTest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Data.SqlClient; 3 | using System.Linq; 4 | using Gridsum.DataflowEx.Databases; 5 | using Microsoft.EntityFrameworkCore; 6 | using Microsoft.VisualStudio.TestTools.UnitTesting; 7 | 8 | namespace Gridsum.DataflowEx.Test.DatabaseTests 9 | { 10 | public class LoopBase 11 | { 12 | public int Id { get; set; } 13 | public string Name { get; set; } 14 | } 15 | 16 | public class LoopA : LoopBase 17 | { 18 | public LoopB LoopB { get; set; } 19 | } 20 | 21 | public class LoopB : LoopBase 22 | { 23 | public LoopC LoopC { get; set; } 24 | } 25 | 26 | public class LoopC 27 | { 28 | public LoopA LoopA { get; set; } 29 | } 30 | /// 31 | /// 主要测试当对象引用有闭环时，以及多个属性同时对应到数据库列时。是否能正常 32 | /// 33 | 34 | [TestClass] 35 | public class ReferenceLoopDepthTest 36 | { 37 | [TestMethod] 38 | public void TestReferenceLoopDepth() 39 | { 40 | var connectString = TestUtils.GetLocalDBConnectionString(); 41 | //init database 42 | var context = new LoopContext(connectString); 43 | context.LoopInits.Add(new LoopInit()); 44 | context.SaveChanges(); 45 | 46 | var loopA = new LoopA 47 | { 48 | Name = "loopA", 49 | LoopB = new LoopB 50 | { 51 | Name = "loopB", 52 | LoopC = new LoopC 53 | { 54 | LoopA = new LoopA 55 | { 56 | Name = "loopA2" 57 | } 58 | } 59 | } 60 | }; 61 | 62 | using (var connection = new SqlConnection(connectString)) 63 | { 64 | connection.Open(); 65 | 66 | var tableName = "dbo.LoopEntities"; 67 | var accessor = TypeAccessorManager.GetAccessorForTable(new TargetTable(GSProduct.AD, connectString, tableName)); 68 | var reader = new BulkDataReader(accessor, new[] { loopA }); 69 | using (var bulkCopy = new SqlBulkCopy(connection)) 70 | { 71 | foreach (SqlBulkCopyColumnMapping map in reader.ColumnMappings) 72 | { 73 | bulkCopy.ColumnMappings.Add(map); 74 | } 75 | bulkCopy.DestinationTableName = tableName; 76 | 77 | bulkCopy.WriteToServer(reader); 78 | } 79 | } 80 | 81 | Assert.AreEqual("loopA", context.LoopEntities.FirstOrDefault().Name); 82 | 83 | } 84 | } 85 | 86 | 87 | public class LoopContext : DbContextBase 88 | { 89 | public LoopContext(string con) : base(con) { } 90 | 91 | public DbSet LoopInits { get; set; } 92 | 93 | public DbSet LoopEntities { get; set; } 94 | } 95 | 96 | public class LoopInit 97 | { 98 | public int Id { get; set; } 99 | } 100 | 101 | public class LoopEntity 102 | { 103 | public int Id { get; set; } 104 | 105 | public string Name { get; set; } 106 | 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/DatabaseTests/TestBootstrapper.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.VisualStudio.TestTools.UnitTesting; 3 | using Docker.DotNet; 4 | using Docker.DotNet.Models; 5 | using System.Threading; 6 | using System.Collections.Generic; 7 | using System.Diagnostics; 8 | using System.Text; 9 | 10 | namespace Gridsum.DataflowEx.Test.DatabaseTests 11 | { 12 | [TestClass] 13 | public class TestBootstrapper 14 | { 15 | internal static string s_containerName = "mssql-for-dataflowex-test"; 16 | internal static string s_saPassword = "UpLow123-+"; 17 | internal static string s_imageName = "microsoft/mssql-server-linux:2017-GA"; 18 | internal static string s_runningSqlServerContainerID; 19 | internal static Process s_sqlserverDockerProcess; 20 | 21 | [AssemblyInitialize] 22 | public static void BootSqlServerWithDockerCli(TestContext tc) 23 | { 24 | BootSqlServerWithDockerCli(tc, s_containerName); 25 | } 26 | 27 | public static void BootSqlServerWithDockerCli(TestContext tc, string containerName) 28 | { 29 | //using (var pullProcess = ExecuteCommand("docker", $"pull {s_imageName}", ".", Console.WriteLine, Console.WriteLine)) 30 | //{ 31 | // pullProcess.WaitForExit(); 32 | //} 33 | 34 | s_sqlserverDockerProcess = ExecuteCommand("docker", $"run --name \"{containerName}\" -e \"ACCEPT_EULA=Y\" -e \"SA_PASSWORD={s_saPassword}\" -a stdin -a stdout -a stderr -p 1433:1433 {s_imageName}", ".", Console.WriteLine, Console.WriteLine); 35 | Thread.Sleep(20 * 1000); //todo: improve this 36 | } 37 | 38 | [AssemblyCleanup] 39 | public static void ShutdownSqlServerWithDockerCli() 40 | { 41 | ShutdownSqlServerWithDockerCli(true, s_containerName); 42 | } 43 | 44 | public static void ShutdownSqlServerWithDockerCli(bool removeAfterShutDown, string containerName) 45 | { 46 | using (var process = ExecuteCommand("docker", $"stop \"{containerName}\"", ".", Console.WriteLine, Console.WriteLine)) 47 | { 48 | process.WaitForExit(); 49 | } 50 | 51 | s_sqlserverDockerProcess.Dispose(); 52 | 53 | if (removeAfterShutDown) 54 | { 55 | using (var process2 = ExecuteCommand("docker", $"rm \"{containerName}\"", ".", Console.WriteLine, Console.WriteLine)) 56 | { 57 | process2.WaitForExit(); 58 | } 59 | } 60 | } 61 | 62 | #region docker client approach (not working) 63 | public static void BootSqlServerWithDockerClient(TestContext tc) 64 | { 65 | DockerClient client = new DockerClientConfiguration(new Uri("npipe://./pipe/docker_engine")).CreateClient(); 66 | var response = client.Containers.CreateContainerAsync(new CreateContainerParameters { 67 | Image = "microsoft/mssql-server-linux", 68 | AttachStderr = true, 69 | AttachStdin = true, 70 | AttachStdout = true, 71 | Env = new[] { "ACCEPT_EULA=Y", $"SA_PASSWORD={s_saPassword}" }, 72 | ExposedPorts = new Dictionary() { 73 | { "1433/tcp", new EmptyStruct() } 74 | }, 75 | HostConfig = new HostConfig 76 | { 77 | PortBindings = new Dictionary> { 78 | { 79 | "1433/tcp", new List { 80 | new PortBinding { HostPort = 1433.ToString() } 81 | } 82 | } 83 | } 84 | } 85 | 86 | }).Result; 87 | s_runningSqlServerContainerID = response.ID; 88 | client.Containers.StartContainerAsync(s_runningSqlServerContainerID, new ContainerStartParameters { }).Wait(); 89 | } 90 | 91 | public static void ShutdownSqlServerDockerImage() 92 | { 93 | DockerClient client = new DockerClientConfiguration(new Uri("npipe://./pipe/docker_engine")).CreateClient(); 94 | client.Containers.StopContainerAsync(s_runningSqlServerContainerID, new ContainerStopParameters { 95 | WaitBeforeKillSeconds = 60 96 | }, new CancellationToken()).Wait(); 97 | } 98 | #endregion 99 | 100 | static Process ExecuteCommand(string executable, string arguments, string workingDirectory, Action output, Action error) 101 | { 102 | var process = new Process(); 103 | process.StartInfo.FileName = executable; 104 | process.StartInfo.Arguments = arguments; 105 | process.StartInfo.WorkingDirectory = workingDirectory; 106 | process.StartInfo.UseShellExecute = false; 107 | process.StartInfo.CreateNoWindow = true; 108 | process.StartInfo.RedirectStandardOutput = true; 109 | process.StartInfo.RedirectStandardError = true; 110 | process.StartInfo.StandardOutputEncoding = Encoding.UTF8; 111 | process.StartInfo.StandardErrorEncoding = Encoding.UTF8; 112 | 113 | process.OutputDataReceived += (sender, e) => 114 | { 115 | output(e.Data); 116 | }; 117 | 118 | process.ErrorDataReceived += (sender, e) => 119 | { 120 | error(e.Data); 121 | }; 122 | 123 | process.Start(); 124 | 125 | process.BeginOutputReadLine(); 126 | process.BeginErrorReadLine(); 127 | 128 | return process; 129 | } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/DataflowFaultTest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.VisualStudio.TestTools.UnitTesting; 3 | 4 | namespace Gridsum.DataflowEx.Test 5 | { 6 | using System.Threading.Tasks; 7 | using System.Threading.Tasks.Dataflow; 8 | 9 | using Gridsum.DataflowEx.Exceptions; 10 | 11 | [TestClass] 12 | public class DataflowFaultTest 13 | { 14 | public class MyDataflow : Dataflow 15 | { 16 | public Dataflow m_block1; 17 | public Dataflow m_block2; 18 | public int cleanUpSignal = 0; 19 | 20 | public MyDataflow(): base(DataflowOptions.Default) 21 | { 22 | m_block1 = new BufferBlock().ToDataflow(name: "innerflow1"); 23 | m_block2 = new BatchBlock(100).ToDataflow(name: "innerflow2"); 24 | m_block1.LinkTo(m_block2); 25 | RegisterChild(this.m_block1); 26 | RegisterChild(this.m_block2); 27 | } 28 | 29 | public override ITargetBlock InputBlock 30 | { 31 | get 32 | { 33 | return this.m_block1.InputBlock; 34 | } 35 | } 36 | 37 | protected override void CleanUp(Exception dataflowException) 38 | { 39 | if (dataflowException == null) 40 | { 41 | cleanUpSignal = 1; 42 | } 43 | else 44 | { 45 | cleanUpSignal = -1; 46 | } 47 | } 48 | } 49 | 50 | [TestMethod] 51 | public async Task TestCleanUp() 52 | { 53 | var f = new MyDataflow(); 54 | await f.SignalAndWaitForCompletionAsync(); 55 | Assert.AreEqual(1, f.cleanUpSignal); 56 | } 57 | 58 | [TestMethod] 59 | public async Task TestCleanUpOnFault() 60 | { 61 | var f = new MyDataflow() {Name = "dataflow1"}; 62 | f.m_block2.RegisterChild(new BufferBlock()); 63 | 64 | Task.Run( 65 | async () => 66 | { 67 | await Task.Delay(200); 68 | (f.m_block2.InputBlock).Fault(new InvalidOperationException()); 69 | }); 70 | 71 | var f2 = new Dataflow(DataflowOptions.Default) {Name = "dataflow2"}; 72 | f2.RegisterChild(f.m_block1); 73 | f2.RegisterChild(f.m_block2); 74 | 75 | var f3 = new Dataflow(DataflowOptions.Default) { Name = "dataflow3" }; 76 | f3.RegisterChild(f); 77 | f3.RegisterChild(f2); 78 | 79 | await this.AssertFlowError(f); 80 | 81 | Assert.AreEqual(-1, f.cleanUpSignal); 82 | 83 | await this.AssertFlowError(f2); 84 | 85 | await this.AssertFlowError(f3); 86 | } 87 | 88 | public async Task AssertFlowError(Dataflow f) where T : Exception 89 | { 90 | try 91 | { 92 | await f.CompletionTask; 93 | } 94 | catch (Exception e) 95 | { 96 | Assert.IsTrue(TaskEx.UnwrapWithPriority(e as AggregateException) is T); 97 | } 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/Demo/Order.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.Test.Demo 8 | { 9 | using System.Linq.Expressions; 10 | 11 | using Gridsum.DataflowEx.Databases; 12 | using Gridsum.DataflowEx.ETL; 13 | 14 | public class Order 15 | { 16 | [DBColumnMapping("OrderTarget", "Date", null)] 17 | public DateTime OrderDate { get; set; } 18 | 19 | [DBColumnMapping("OrderTarget", "Value", 0.0f)] 20 | public float? OrderValue { get; set; } 21 | 22 | [DBColumnPath(DBColumnPathOptions.DoNotGenerateNullCheck)] 23 | public Person Customer { get; set; } 24 | } 25 | 26 | public class OrderEx : Order 27 | { 28 | public OrderEx() 29 | { 30 | this.OrderDate = DateTime.Now; 31 | } 32 | 33 | // This is the field we want to populate 34 | [DBColumnMapping("LookupDemo", "ProductKey")] 35 | [DBColumnMapping("OrderTarget", "ProductKey")] 36 | public long? ProductKey { get; set; } 37 | 38 | public Product Product { get; set; } 39 | } 40 | 41 | public class Product 42 | { 43 | public Product(string category, string name) 44 | { 45 | this.Category = category; 46 | this.Name = name; 47 | } 48 | 49 | [DBColumnMapping("LookupDemo", "Category")] 50 | public string Category { get; private set; } 51 | 52 | [DBColumnMapping("LookupDemo", "ProductName")] 53 | public string Name { get; private set; } 54 | 55 | [DBColumnMapping("LookupDemo", "ProductFullName")] 56 | public string FullName { get { return string.Format("{0}-{1}", Category, Name); } } 57 | } 58 | 59 | public class ProductLookupFlow : DbDataJoiner 60 | { 61 | public ProductLookupFlow(TargetTable dimTableTarget, int batchSize = 8192, int cacheSize = 1048576) 62 | : base( 63 | //Tells DbDataJoiner the join condition is 'OrderEx.Product.FullName = TABLE.ProductFullName' 64 | o => o.Product.FullName, 65 | dimTableTarget, DataflowOptions.Default, batchSize, cacheSize) 66 | { 67 | } 68 | 69 | protected override void OnSuccessfulLookup(OrderEx input, IDimRow rowInDimTable) 70 | { 71 | //what we want is just the key column of the dimension row 72 | input.ProductKey = rowInDimTable.AutoIncrementKey; 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/Demo/PeopleFlow.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.Test.Demo 8 | { 9 | using System.Threading; 10 | using System.Threading.Tasks.Dataflow; 11 | 12 | using Gridsum.DataflowEx.Databases; 13 | 14 | using Newtonsoft.Json; 15 | 16 | public class Person : IEventProvider 17 | { 18 | [DBColumnMapping("PersonTarget", "NameCol", "N/A", ColumnMappingOption.Mandatory)] 19 | [DBColumnMapping("OrderTarget", "CustomerName", "Unknown Customer", ColumnMappingOption.Optional)] 20 | public string Name { get; set; } 21 | 22 | [DBColumnMapping("PersonTarget", "AgeCol", -1, ColumnMappingOption.Optional)] 23 | [DBColumnMapping("OrderTarget", "CustomerAge", -1, ColumnMappingOption.Optional)] 24 | public int? Age { get; set; } 25 | 26 | public DataflowEvent GetEvent() 27 | { 28 | if (Age > 70) 29 | { 30 | return new DataflowEvent("OldPerson"); 31 | } 32 | else 33 | { 34 | //returning empty so it will not be recorded as an event 35 | return DataflowEvent.Empty; 36 | } 37 | } 38 | } 39 | 40 | public class PeopleFlow : Dataflow 41 | { 42 | private TransformBlock m_converter; 43 | private TransformBlock m_recorder; 44 | private StatisticsRecorder m_peopleRecorder; 45 | 46 | public PeopleFlow(DataflowOptions dataflowOptions) 47 | : base(dataflowOptions) 48 | { 49 | m_peopleRecorder = new StatisticsRecorder(this) { Name = "PeopleRecorder"}; 50 | 51 | m_converter = new TransformBlock(s => JsonConvert.DeserializeObject(s)); 52 | m_recorder = new TransformBlock( 53 | p => 54 | { 55 | //record every person 56 | m_peopleRecorder.Record(p); 57 | return p; 58 | }); 59 | 60 | m_converter.LinkTo(m_recorder, new DataflowLinkOptions() { PropagateCompletion = true}); 61 | 62 | RegisterChild(m_converter); 63 | RegisterChild(m_recorder); 64 | } 65 | 66 | public override ITargetBlock InputBlock { get { return m_converter; } } 67 | public override ISourceBlock OutputBlock { get { return m_recorder; } } 68 | public StatisticsRecorder PeopleRecorder { get { return m_peopleRecorder; } } 69 | } 70 | } 71 | 72 | 73 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/Demo/SlowFlow.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.Test.Demo 8 | { 9 | using System.Threading; 10 | using System.Threading.Tasks.Dataflow; 11 | 12 | public class SlowFlow : Dataflow 13 | { 14 | private Dataflow _splitter; 15 | private Dataflow _printer; 16 | 17 | public SlowFlow(DataflowOptions dataflowOptions) 18 | : base(dataflowOptions) 19 | { 20 | _splitter = new TransformManyBlock(new Func>(this.SlowSplit), 21 | dataflowOptions.ToExecutionBlockOption()) 22 | .ToDataflow(dataflowOptions, "SlowSplitter"); 23 | 24 | _printer = new ActionBlock(c => Console.WriteLine(c), 25 | dataflowOptions.ToExecutionBlockOption()) 26 | .ToDataflow(dataflowOptions, "Printer"); 27 | 28 | RegisterChild(_splitter); 29 | RegisterChild(_printer); 30 | 31 | _splitter.LinkTo(_printer); 32 | } 33 | 34 | private IEnumerable SlowSplit(string s) 35 | { 36 | foreach (var c in s) 37 | { 38 | Thread.Sleep(1000); //slow down 39 | yield return c; 40 | } 41 | } 42 | 43 | public override ITargetBlock InputBlock { get { return _splitter.InputBlock; } } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/ETL/TestDataJoiner.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.Test.ETL 8 | { 9 | using System.Data; 10 | using System.Diagnostics; 11 | using System.Linq.Expressions; 12 | using System.Threading.Tasks.Dataflow; 13 | 14 | using Gridsum.DataflowEx.Databases; 15 | using Gridsum.DataflowEx.ETL; 16 | using Gridsum.DataflowEx.Test.DatabaseTests; 17 | using Microsoft.EntityFrameworkCore; 18 | using Microsoft.VisualStudio.TestTools.UnitTesting; 19 | 20 | [TestClass] 21 | public class TestDataJoiner 22 | { 23 | class JoinerWithAsserter : DbDataJoiner 24 | { 25 | public int Count; 26 | 27 | public JoinerWithAsserter(Expression> joinOn, TargetTable dimTableTarget, int batchSize) 28 | : base(joinOn, dimTableTarget, DataflowOptions.Default, batchSize) 29 | { 30 | this.Count = 0; 31 | } 32 | 33 | protected override void OnSuccessfulLookup(Trunk input, IDimRow rowInDimTable) 34 | { 35 | Assert.AreEqual(input.Pointer, rowInDimTable.JoinOn); 36 | //Assert.AreEqual("Str" + input.Pointer, rowInDimTable["StrValue"]); 37 | this.Count++; 38 | } 39 | } 40 | 41 | [TestMethod] 42 | public async Task TestDataJoinerJoining() 43 | { 44 | var connectString = TestUtils.GetLocalDBConnectionString("TestDataJoinerJoining"); 45 | var context = new InsertContext(connectString); 46 | 47 | for (int i = 0; i < 10; i++) 48 | { 49 | context.Leafs.Add(new Leaf() {IntValue = i, Key = i.ToString(), StrValue = "Str" + i}); 50 | } 51 | 52 | context.SaveChanges(); 53 | 54 | var joiner = new JoinerWithAsserter( 55 | t => t.Pointer, 56 | new TargetTable(Leaf.DimLeaf, connectString, "Leafs"), 57 | 2); 58 | 59 | joiner.DataflowOptions.MonitorInterval = TimeSpan.FromSeconds(3); 60 | joiner.LinkLeftToNull(); 61 | 62 | var dataArray = new[] 63 | { 64 | new Trunk() { Pointer = "3" }, 65 | new Trunk() { Pointer = "5" }, 66 | new Trunk() { Pointer = "7" }, 67 | new Trunk() { Pointer = "7" }, 68 | new Trunk() { Pointer = "9" }, 69 | new Trunk() { Pointer = "99", Leaf = new Leaf() { StrValue = "Str99" } }, 70 | new Trunk() { Pointer = "99", Leaf = new Leaf() { StrValue = "Str99" } }, 71 | new Trunk() { Pointer = "99", Leaf = new Leaf() { StrValue = "Str99" } }, 72 | new Trunk() { Pointer = "99", Leaf = new Leaf() { StrValue = "Str99" } }, 73 | new Trunk() { Pointer = "99", Leaf = new Leaf() { StrValue = "Str99" } }, 74 | new Trunk() { Pointer = "99", Leaf = new Leaf() { StrValue = "Str99" } }, 75 | new Trunk() { Pointer = "99", Leaf = new Leaf() { StrValue = "Str99" } }, 76 | new Trunk() { Pointer = "99", Leaf = new Leaf() { StrValue = "Str99" } }, 77 | new Trunk() { Pointer = "99", Leaf = new Leaf() { StrValue = "Str99" } }, 78 | new Trunk() { Pointer = "99", Leaf = new Leaf() { StrValue = "Str99" } }, 79 | new Trunk() { Pointer = "99", Leaf = new Leaf() { StrValue = "Str99" } }, 80 | new Trunk() { Pointer = "99", Leaf = new Leaf() { StrValue = "Str99" } }, 81 | new Trunk() { Pointer = "99", Leaf = new Leaf() { StrValue = "Str99" } }, 82 | new Trunk() { Pointer = "99", Leaf = new Leaf() { StrValue = "Str99" } }, 83 | }; 84 | 85 | await joiner.ProcessAsync(dataArray); 86 | 87 | Assert.AreEqual(dataArray.Length, joiner.Count); 88 | } 89 | } 90 | 91 | public class Trunk 92 | { 93 | public int Id { get; set; } 94 | 95 | [DBColumnMapping(Leaf.DimLeaf, "Key")] 96 | public string Pointer { get; set; } 97 | 98 | public string Name { get; set; } 99 | public Leaf Leaf { get; set; } 100 | } 101 | 102 | public class Leaf 103 | { 104 | public const string DimLeaf = "DimLeaf"; 105 | 106 | public int Id { get; set; } 107 | 108 | public string Key { get; set; } 109 | 110 | [DBColumnMapping(DimLeaf)] 111 | public string StrValue { get; set; } 112 | 113 | [DBColumnMapping(DimLeaf)] 114 | public int IntValue { get; set; } 115 | } 116 | 117 | public class InsertContext : DbContextBase 118 | { 119 | public InsertContext(string conn) 120 | : base(conn) 121 | { 122 | } 123 | 124 | public DbSet Trunks { get; set; } 125 | public DbSet Leafs { get; set; } 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/ETL/TestDataJoiner2.cs: -------------------------------------------------------------------------------- 1 | namespace Gridsum.DataflowEx.Test.ETL.Test2 2 | { 3 | using System; 4 | using System.Data; 5 | using System.Linq; 6 | using System.Linq.Expressions; 7 | using System.Text; 8 | using System.Threading.Tasks; 9 | 10 | using Gridsum.DataflowEx.Databases; 11 | using Gridsum.DataflowEx.ETL; 12 | using Microsoft.EntityFrameworkCore; 13 | using Microsoft.VisualStudio.TestTools.UnitTesting; 14 | 15 | [TestClass] 16 | public class TestDataJoiner2 17 | { 18 | class JoinerWithAsserter : DbDataJoiner 19 | { 20 | public int Count; 21 | 22 | public JoinerWithAsserter(Expression> joinOn, TargetTable dimTableTarget, int batchSize) 23 | : base(joinOn, dimTableTarget, DataflowOptions.Default, batchSize) 24 | { 25 | this.Count = 0; 26 | } 27 | 28 | protected override void OnSuccessfulLookup(Trunk input, IDimRow rowInDimTable) 29 | { 30 | Assert.IsTrue(TestUtils.ArrayEqual(input.Pointer, rowInDimTable.JoinOn)); 31 | //Assert.AreEqual("Str" + Encoding.UTF8.GetString(input.Pointer), rowInDimTable["StrValue"]); 32 | this.Count++; 33 | } 34 | } 35 | 36 | [TestMethod] 37 | public async Task TestDataJoinerJoining() 38 | { 39 | var connectString = TestUtils.GetLocalDBConnectionString("TestDataJoinerJoining"); 40 | var context = new InsertContext(connectString); 41 | 42 | for (int i = 0; i < 10; i++) 43 | { 44 | context.Leafs.Add(new Leaf() { IntValue = i, Key = i.ToString().ToByteArray(), StrValue = "Str" + i }); 45 | } 46 | 47 | context.SaveChanges(); 48 | 49 | var joiner = new JoinerWithAsserter( 50 | t => t.Pointer, 51 | new TargetTable(Leaf.DimLeaf, connectString, "Leafs"), 52 | 2); 53 | 54 | joiner.DataflowOptions.MonitorInterval = TimeSpan.FromSeconds(3); 55 | 56 | joiner.LinkLeftToNull(); 57 | 58 | var dataArray = new[] 59 | { 60 | new Trunk() { Pointer = "3".ToByteArray() }, 61 | new Trunk() { Pointer = "5".ToByteArray() }, 62 | new Trunk() { Pointer = "7".ToByteArray() }, 63 | new Trunk() { Pointer = "7".ToByteArray() }, 64 | new Trunk() { Pointer = "9".ToByteArray() }, 65 | new Trunk() { Pointer = "99".ToByteArray(), Leaf = new Leaf() { StrValue = "Str99" } }, 66 | new Trunk() { Pointer = "99".ToByteArray(), Leaf = new Leaf() { StrValue = "Str99" } }, 67 | new Trunk() { Pointer = "99".ToByteArray(), Leaf = new Leaf() { StrValue = "Str99" } }, 68 | new Trunk() { Pointer = "99".ToByteArray(), Leaf = new Leaf() { StrValue = "Str99" } }, 69 | new Trunk() { Pointer = "99".ToByteArray(), Leaf = new Leaf() { StrValue = "Str99" } }, 70 | new Trunk() { Pointer = "99".ToByteArray(), Leaf = new Leaf() { StrValue = "Str99" } }, 71 | new Trunk() { Pointer = "99".ToByteArray(), Leaf = new Leaf() { StrValue = "Str99" } }, 72 | new Trunk() { Pointer = "99".ToByteArray(), Leaf = new Leaf() { StrValue = "Str99" } }, 73 | new Trunk() { Pointer = "99".ToByteArray(), Leaf = new Leaf() { StrValue = "Str99" } }, 74 | new Trunk() { Pointer = "99".ToByteArray(), Leaf = new Leaf() { StrValue = "Str99" } }, 75 | new Trunk() { Pointer = "99".ToByteArray(), Leaf = new Leaf() { StrValue = "Str99" } }, 76 | new Trunk() { Pointer = "99".ToByteArray(), Leaf = new Leaf() { StrValue = "Str99" } }, 77 | new Trunk() { Pointer = "99".ToByteArray(), Leaf = new Leaf() { StrValue = "Str99" } }, 78 | }; 79 | 80 | await joiner.ProcessAsync(dataArray); 81 | 82 | Assert.AreEqual(dataArray.Length, joiner.Count); 83 | 84 | Assert.AreEqual(11, context.Leafs.Count()); 85 | } 86 | } 87 | 88 | 89 | public class Trunk 90 | { 91 | public int Id { get; set; } 92 | 93 | [DBColumnMapping(Leaf.DimLeaf, "Key")] 94 | public byte[] Pointer { get; set; } 95 | 96 | public string Name { get; set; } 97 | public Leaf Leaf { get; set; } 98 | } 99 | 100 | public class Leaf 101 | { 102 | public const string DimLeaf = "DimLeaf"; 103 | 104 | public int Id { get; set; } 105 | 106 | public byte[] Key { get; set; } 107 | 108 | [DBColumnMapping(DimLeaf)] 109 | public string StrValue { get; set; } 110 | 111 | [DBColumnMapping(DimLeaf)] 112 | public int IntValue { get; set; } 113 | } 114 | 115 | public class InsertContext : DbContextBase 116 | { 117 | public InsertContext(string conn) 118 | : base(conn) 119 | { 120 | } 121 | 122 | public DbSet Trunks { get; set; } 123 | public DbSet Leafs { get; set; } 124 | } 125 | } -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/Gridsum.DataflowEx.Test.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp2.0 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/TestAutoComplete.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Threading; 4 | using System.Threading.Tasks; 5 | using System.Threading.Tasks.Dataflow; 6 | using Gridsum.DataflowEx.AutoCompletion; 7 | using Microsoft.VisualStudio.TestTools.UnitTesting; 8 | 9 | namespace Gridsum.DataflowEx.Test 10 | { 11 | [TestClass] 12 | public class TestAutoComplete 13 | { 14 | public class Int : TracableItemBase 15 | { 16 | public int Val { get; set; } 17 | } 18 | 19 | [TestMethod] 20 | public async Task TestAutoComplete1() 21 | { 22 | var block = new TransformManyBlock(i => 23 | { 24 | int j = i.Val + 1; 25 | Console.WriteLine("block2: i = {0}, j = {1}", i.Val, j); 26 | Thread.Sleep(500); 27 | if (j < 10) return new[] { new Int { Val = j } }; 28 | else return Enumerable.Empty(); 29 | }); 30 | 31 | var dataflow1 = DataflowUtils.FromDelegate(i => i); 32 | var dataflow2 = DataflowUtils.FromBlock(block).AutoComplete(TimeSpan.FromSeconds(1)); 33 | 34 | dataflow1.GoTo(dataflow2).GoTo(dataflow1); 35 | 36 | dataflow1.InputBlock.Post(new Int() { Val = 1 }); 37 | Assert.IsTrue(await dataflow2.CompletionTask.FinishesIn(TimeSpan.FromSeconds(10))); 38 | } 39 | 40 | [TestMethod] 41 | public async Task TestAutoComplete2() 42 | { 43 | var block2 = new TransformManyBlock(i => 44 | { 45 | return Enumerable.Empty(); 46 | }); 47 | 48 | var dataflow1 = DataflowUtils.FromDelegate(i => new[] { i }); //preserve the guid 49 | var dataflow2 = DataflowUtils.FromBlock(block2).AutoComplete(TimeSpan.FromSeconds(1)); 50 | 51 | dataflow1.GoTo(dataflow2).GoTo(dataflow1); 52 | 53 | dataflow1.InputBlock.Post(new Int() { Val = 1 }); 54 | Assert.IsTrue(await dataflow2.CompletionTask.FinishesIn(TimeSpan.FromSeconds(2))); 55 | Assert.IsTrue(dataflow2.Name.EndsWith("AutoComplete")); 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/TestDataBroadcaster.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using System.Threading.Tasks.Dataflow; 7 | using Gridsum.DataflowEx.Exceptions; 8 | using Microsoft.VisualStudio.TestTools.UnitTesting; 9 | 10 | namespace Gridsum.DataflowEx.Test 11 | { 12 | [TestClass] 13 | public class TestDataBroadcaster 14 | { 15 | [TestMethod] 16 | public async Task TestDataBroadcaster1() 17 | { 18 | var random = new Random(); 19 | var dataCopier = new DataBroadcaster(); 20 | 21 | int sum1 = 0; 22 | int sum2 = 0; 23 | 24 | var action1 = new ActionBlock(i => sum1 = sum1 + i); 25 | var action2 = new ActionBlock(i => sum2 = sum2 + i); 26 | 27 | dataCopier.LinkTo(DataflowUtils.FromBlock(action1)); 28 | dataCopier.LinkTo(DataflowUtils.FromBlock(action2)); 29 | 30 | for (int j = 0; j < 1000; j++) 31 | { 32 | dataCopier.InputBlock.Post((int) (random.NextDouble()*10000)); 33 | } 34 | 35 | dataCopier.Complete(); 36 | 37 | await TaskEx.AwaitableWhenAll(action1.Completion, action2.Completion); 38 | 39 | Console.WriteLine("sum1 = {0} , sum2 = {1}", sum1, sum2); 40 | Assert.AreEqual(sum1, sum2); 41 | } 42 | 43 | [TestMethod] 44 | public async Task TestDataBroadcaster2() 45 | { 46 | var random = new Random(); 47 | var buffer = new BufferBlock(); 48 | 49 | int sum1 = 0; 50 | int sum2 = 0; 51 | int sum3 = 0; 52 | 53 | var action1 = new ActionBlock(i => sum1 = sum1 + i); 54 | var action2 = new ActionBlock(i => sum2 = sum2 + i); 55 | var action3 = new ActionBlock(i => sum3 = sum3 + i); 56 | 57 | buffer.ToDataflow().LinkToMultiple(new []{action1, action2, action3}.Select(a => a.ToDataflow()).ToArray()); 58 | 59 | for (int j = 0; j < 1000; j++) 60 | { 61 | buffer.Post((int)(random.NextDouble() * 10000)); 62 | } 63 | 64 | buffer.Complete(); 65 | 66 | await TaskEx.AwaitableWhenAll(action1.Completion, action2.Completion, action3.Completion); 67 | 68 | Console.WriteLine("sum1 = {0} , sum2 = {1}", sum1, sum2); 69 | Assert.AreEqual(sum1, sum2); 70 | Assert.AreEqual(sum1, sum3); 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/TestFlowBlocking.cs: -------------------------------------------------------------------------------- 1 | namespace Gridsum.DataflowEx.Test 2 | { 3 | using System; 4 | using System.Threading.Tasks; 5 | 6 | using Gridsum.DataflowEx.Test.Demo; 7 | 8 | using Microsoft.VisualStudio.TestTools.UnitTesting; 9 | 10 | [TestClass] 11 | public class TestFlowBlocking 12 | { 13 | [TestMethod] 14 | public async Task TestBlocking1() 15 | { 16 | var slowFlow = new SlowFlow(new DataflowOptions 17 | { 18 | FlowMonitorEnabled = true, 19 | MonitorInterval = TimeSpan.FromSeconds(5), 20 | RecommendedCapacity = 2 21 | }); 22 | 23 | Assert.IsTrue(slowFlow.Post("a")); 24 | Assert.IsTrue(slowFlow.Post("a")); 25 | Assert.IsFalse(slowFlow.Post("a")); 26 | Assert.IsFalse(slowFlow.Post("a")); 27 | Assert.IsFalse(slowFlow.Post("a")); 28 | Assert.IsFalse(slowFlow.Post("a")); 29 | 30 | await Task.Delay(2000); 31 | 32 | Assert.IsTrue(slowFlow.Post("a")); 33 | } 34 | 35 | [TestMethod] 36 | public async Task TestBlocking2() 37 | { 38 | var slowFlow = new SlowFlow(new DataflowOptions 39 | { 40 | FlowMonitorEnabled = true, 41 | MonitorInterval = TimeSpan.FromSeconds(5), 42 | RecommendedCapacity = 2 43 | }); 44 | 45 | await slowFlow.SendAsync("a"); 46 | await slowFlow.SendAsync("a"); 47 | await slowFlow.SendAsync("a"); 48 | await slowFlow.SendAsync("a"); 49 | await slowFlow.SendAsync("a"); 50 | await slowFlow.SendAsync("a"); 51 | 52 | //Assert.IsTrue(slowFlow.Post("a")); 53 | } 54 | } 55 | } -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/TestImmutable.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Immutable; 3 | using System.Linq; 4 | using Microsoft.VisualStudio.TestTools.UnitTesting; 5 | 6 | namespace Gridsum.DataflowEx.Test 7 | { 8 | using System.Threading.Tasks; 9 | 10 | [TestClass] 11 | public class TestImmutable 12 | { 13 | [TestMethod] 14 | public void TestImmutableList() 15 | { 16 | var l = ImmutableList.Create(1, 3, 2); 17 | 18 | Assert.AreEqual(1, l.First()); 19 | Assert.AreEqual(2, l.Last()); 20 | 21 | l = l.Add(1); 22 | 23 | Assert.AreEqual(1, l.Last()); 24 | } 25 | 26 | [TestMethod] 27 | public void TestListAdd() 28 | { 29 | const int c = 1000000; 30 | var l = ImmutableList.Empty; 31 | 32 | Parallel.For(0, c, i => { l = l.Add(i); }); 33 | 34 | Assert.IsTrue(l.Count < c); 35 | Console.WriteLine(l.Count); 36 | } 37 | 38 | [TestMethod] 39 | public void TestListAdd2() 40 | { 41 | const int c = 1000000; 42 | var l = ImmutableList.Empty; 43 | 44 | Parallel.For(0, c, i => ImmutableUtils.AddOptimistically(ref l, i)); 45 | 46 | Assert.IsTrue(l.Count == c); 47 | Console.WriteLine(l.Count); 48 | } 49 | 50 | [TestMethod] 51 | public void TestSetAdd() 52 | { 53 | var iset = ImmutableHashSet.Create(1,2,3); 54 | 55 | Assert.IsFalse(ImmutableUtils.TryAddOptimistically(ref iset, 3)); 56 | Assert.IsTrue(ImmutableUtils.TryAddOptimistically(ref iset, 4)); 57 | } 58 | 59 | [TestMethod] 60 | public void TestSetAdd2() 61 | { 62 | const int c = 1000000; 63 | var l = ImmutableHashSet.Empty; 64 | 65 | Parallel.For(0, c, i => Assert.IsTrue(ImmutableUtils.TryAddOptimistically(ref l, i))); 66 | Parallel.For(0, c, i => Assert.IsFalse(ImmutableUtils.TryAddOptimistically(ref l, i))); 67 | 68 | Assert.IsTrue(l.Count == c); 69 | Console.WriteLine(l.Count); 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/TestLogReader.cs: -------------------------------------------------------------------------------- 1 | namespace Gridsum.DataflowEx.Test 2 | { 3 | using System; 4 | using System.Collections.Generic; 5 | using System.IO; 6 | using System.Linq; 7 | using System.Threading; 8 | using System.Threading.Tasks; 9 | using System.Threading.Tasks.Dataflow; 10 | using System.Xml.Serialization; 11 | 12 | using Microsoft.VisualStudio.TestTools.UnitTesting; 13 | 14 | /// 15 | /// Summary description for TestLogReader 16 | /// 17 | [TestClass] 18 | public class TestLogReader 19 | { 20 | #region Public Methods and Operators 21 | 22 | [TestMethod] 23 | public async Task TestMethod1() 24 | { 25 | var logReader = new SimpleLogReader(DataflowOptions.Default); 26 | 27 | long count = await logReader.ProcessAsync(new[] { "a", "a b", "a b c", "a b c d" }, false); 28 | long count2 = await logReader.ProcessAsync(new[] { "a", "a b", "a b c", "a b c d" }); 29 | 30 | Assert.AreEqual(4 * 2, count + count2); 31 | Assert.AreEqual(4 * 2, logReader.Recorder[new DataflowEvent("a")]); 32 | Assert.AreEqual(3 * 2, logReader.Recorder[new DataflowEvent("b")]); 33 | Assert.AreEqual(2 * 2, logReader.Recorder[new DataflowEvent("c")]); 34 | Assert.AreEqual(1 * 2, logReader.Recorder[new DataflowEvent("d")]); 35 | } 36 | 37 | [TestMethod] 38 | public async Task TestMethod2() 39 | { 40 | var logReader = new SimpleLogReader(DataflowOptions.Default); 41 | 42 | try 43 | { 44 | long count = await logReader.ProcessAsync(this.GetStringSequence()); 45 | Assert.Fail("shouldn't arrive here"); 46 | } 47 | catch (Exception e) 48 | { 49 | Assert.AreEqual(typeof(OperationCanceledException), e.GetType()); 50 | } 51 | } 52 | #endregion 53 | 54 | IEnumerable GetStringSequence() 55 | { 56 | yield return "a"; 57 | yield return "b"; 58 | yield return "ERROR"; 59 | Thread.Sleep(5000); 60 | yield return "c"; 61 | Thread.Sleep(5000); 62 | yield return "d"; 63 | } 64 | } 65 | 66 | public class SimpleLogReader : LogReader 67 | { 68 | #region Fields 69 | 70 | private readonly TransformManyBlock m_parsingBlock; 71 | 72 | private readonly StatisticsRecorder m_recorder; 73 | 74 | private ActionBlock m_recordBlock; 75 | 76 | static readonly char[] Splitor = { ' ' }; 77 | 78 | #endregion 79 | 80 | #region Constructors and Destructors 81 | 82 | public SimpleLogReader(DataflowOptions dataflowOptions) 83 | : base(dataflowOptions) 84 | { 85 | this.m_parsingBlock = new TransformManyBlock( 86 | s => 87 | { 88 | if (string.IsNullOrEmpty(s)) 89 | { 90 | return Enumerable.Empty(); 91 | } 92 | 93 | if (s == "ERROR") 94 | { 95 | throw new InvalidDataException("evil data :)"); 96 | } 97 | 98 | return s.Split(Splitor); 99 | }); 100 | 101 | this.m_recorder = new StatisticsRecorder(this); 102 | this.m_recordBlock = new ActionBlock( 103 | s => 104 | { 105 | if (s == "ERROR") 106 | { 107 | throw new InvalidDataException("evil data :)"); 108 | } 109 | 110 | this.m_recorder.RecordEvent(s); 111 | }); 112 | 113 | var df1 = DataflowUtils.FromBlock(m_parsingBlock); 114 | var df2 = DataflowUtils.FromBlock(m_recordBlock); 115 | df1.LinkTo(df2); 116 | 117 | RegisterChild(df1); 118 | RegisterChild(df2); 119 | } 120 | 121 | #endregion 122 | 123 | #region Public Properties 124 | 125 | public override ITargetBlock InputBlock 126 | { 127 | get 128 | { 129 | return this.m_parsingBlock; 130 | } 131 | } 132 | 133 | public override StatisticsRecorder Recorder 134 | { 135 | get 136 | { 137 | return this.m_recorder; 138 | } 139 | } 140 | 141 | #endregion 142 | } 143 | } -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/TestStatisticsRecorder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.VisualStudio.TestTools.UnitTesting; 3 | 4 | namespace Gridsum.DataflowEx.Test 5 | { 6 | [TestClass] 7 | public class TestStatisticsRecorder 8 | { 9 | [TestMethod] 10 | public void TestMethod1() 11 | { 12 | var recorder = new StatisticsRecorder(); 13 | recorder.Record("a"); 14 | recorder.Record("a"); 15 | recorder.Record("a"); 16 | recorder.Record(1); 17 | recorder.Record(1); 18 | recorder.Record(1); 19 | recorder.Record(1); 20 | recorder.Record(new EventHolder()); 21 | recorder.Record(new EventHolder()); 22 | recorder.Record(new EventHolder()); 23 | recorder.RecordEvent("ev1"); 24 | recorder.RecordEvent("ev1"); 25 | recorder.RecordEvent("ev1"); 26 | recorder.RecordEvent("ev1"); 27 | recorder.RecordEvent("ev1"); 28 | recorder.RecordEvent("ev2", "1"); 29 | recorder.RecordEvent("ev2", "2"); 30 | recorder.RecordEvent("ev3", "1"); 31 | recorder.RecordEvent("ev3", "1"); 32 | recorder.RecordEvent("ev3", "1"); 33 | 34 | Assert.AreEqual(3, recorder[typeof(string)]); 35 | Assert.AreEqual(3, recorder[typeof(EventHolder)]); 36 | Assert.AreEqual(4, recorder[typeof(int)]); 37 | Assert.AreEqual(5 + 3, recorder["ev1"]); 38 | Assert.AreEqual(2, recorder["ev2"]); 39 | Assert.AreEqual(3, recorder[new DataflowEvent("ev3", "1")]); 40 | 41 | Assert.AreEqual("a-b", new DataflowEvent("a", "b").ToString()); 42 | Assert.AreEqual("a", new DataflowEvent("a").ToString()); 43 | } 44 | 45 | internal class EventHolder : IEventProvider 46 | { 47 | public DataflowEvent GetEvent() 48 | { 49 | return new DataflowEvent("ev1"); 50 | } 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/TestStringCondition.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Gridsum.DataflowEx.PatternMatch; 3 | using Microsoft.VisualStudio.TestTools.UnitTesting; 4 | 5 | namespace Gridsum.DataflowEx.Test 6 | { 7 | [TestClass] 8 | public class TestStringCondition 9 | { 10 | [TestMethod] 11 | public void TestStringCondition1() 12 | { 13 | var cond = new StringMatchCondition("abc"); 14 | Assert.IsTrue(cond.Matches("abc")); 15 | Assert.IsFalse(cond.Matches("abc1")); 16 | } 17 | 18 | [TestMethod] 19 | public void TestStringCondition2() 20 | { 21 | var cond = new StringMatchCondition("abc", MatchType.BeginsWith); 22 | Assert.IsTrue(cond.Matches("abc")); 23 | Assert.IsTrue(cond.Matches("abc1")); 24 | Assert.IsFalse(cond.Matches("1abc")); 25 | } 26 | 27 | [TestMethod] 28 | public void TestStringCondition3() 29 | { 30 | var cond = new StringMatchCondition("abc", MatchType.Contains); 31 | Assert.IsTrue(cond.Matches("abc")); 32 | Assert.IsTrue(cond.Matches("abc1")); 33 | Assert.IsTrue(cond.Matches("1abc")); 34 | Assert.IsFalse(cond.Matches("abd")); 35 | } 36 | 37 | [TestMethod] 38 | public void TestStringCondition4() 39 | { 40 | var cond = new StringMatchCondition(".*ab[cd].*", MatchType.RegexMatch); 41 | Assert.IsTrue(cond.Matches("abc")); 42 | Assert.IsTrue(cond.Matches("abc1")); 43 | Assert.IsTrue(cond.Matches("1abc")); 44 | Assert.IsTrue(cond.Matches("abd")); 45 | Assert.IsFalse(cond.Matches("abef")); 46 | } 47 | 48 | [TestMethod] 49 | public void TestUrlStringMatchCondition() 50 | { 51 | var cond = new UrlStringMatchCondition(@"((http)|(https)://)?www.gridsum.com/.*\.txt$", MatchType.RegexMatch, true); 52 | Assert.IsTrue(cond.Matches("www.gridsum.com/1.txt?a=b")); 53 | Assert.IsTrue(cond.Matches("http://www.gridsum.com/2.txt#section2")); 54 | Assert.IsTrue(cond.Matches("https://www.gridsum.com/3.txt")); 55 | Assert.IsFalse(cond.Matches("https://www.gridsum.com/3.txta")); 56 | } 57 | 58 | [TestMethod] 59 | public void TestUrlStringMatchCondition2() 60 | { 61 | var cond = new UrlStringMatchCondition(@"((http)|(https)://)?www.gridsum.com/.*\.txt$", MatchType.RegexMatch, false); 62 | Assert.IsFalse(cond.Matches("www.gridsum.com/1.txt?a=b")); 63 | Assert.IsFalse(cond.Matches("http://www.gridsum.com/2.txt#section2")); 64 | Assert.IsTrue(cond.Matches("https://www.gridsum.com/3.txt")); 65 | Assert.IsFalse(cond.Matches("https://www.gridsum.com/3.txta")); 66 | } 67 | 68 | [TestMethod] 69 | public void TestExactMatch() 70 | { 71 | var cond1 = new StringMatchCondition("1", MatchType.Contains); 72 | var cond2 = new StringMatchCondition("2", MatchType.Contains); 73 | var cond3 = new StringMatchCondition("3", MatchType.Contains); 74 | 75 | var multi = new MultiMatchCondition(new[] { cond1, cond2, cond3 }); 76 | 77 | Assert.IsTrue(multi.Matches("432")); 78 | Assert.IsTrue(multi.Matches("123")); 79 | Assert.AreEqual(cond2, multi.MatchesExact("432")); 80 | Assert.AreEqual(cond1, multi.MatchesExact("123")); 81 | 82 | } 83 | 84 | [TestMethod] 85 | public void TestExactMatch2() 86 | { 87 | var cond1 = new StringMatchable("1", MatchType.Contains); 88 | var cond2 = new StringMatchable("2", MatchType.Contains); 89 | var cond3 = new StringMatchable("3", MatchType.EndsWith); 90 | 91 | var multi = new MultiMatcher(new[] { cond1, cond2, cond3 }); 92 | 93 | Assert.IsTrue(multi.Matches("432")); 94 | Assert.IsTrue(multi.Matches("123")); 95 | Assert.AreEqual(cond2.Condition, multi.MatchesExact("432")); 96 | Assert.AreEqual(cond1.Condition, multi.MatchesExact("123")); 97 | 98 | StringMatchable m; 99 | IMatchCondition c; 100 | Assert.IsTrue(multi.TryMatchExact("9873", out m, out c)); 101 | Assert.AreEqual(cond3, m); 102 | Assert.AreEqual(cond3.Condition, c); 103 | } 104 | 105 | internal class StringMatchable : IMatchable 106 | { 107 | private readonly string m_pattern; 108 | private UrlStringMatchCondition m_cond; 109 | 110 | public StringMatchable(string pattern, MatchType matchType) 111 | { 112 | this.m_pattern = pattern; 113 | m_cond = new UrlStringMatchCondition(pattern, matchType, false); 114 | } 115 | 116 | public string Pattern 117 | { 118 | get 119 | { 120 | return m_pattern; 121 | } 122 | } 123 | 124 | public IMatchCondition Condition 125 | { 126 | get 127 | { 128 | return m_cond; 129 | } 130 | } 131 | } 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.Test/TestUtils.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.Test 8 | { 9 | using DatabaseTests; 10 | using System.Data.SqlClient; 11 | using System.Diagnostics; 12 | using System.Reflection; 13 | using Microsoft.EntityFrameworkCore; 14 | 15 | public class DbContextBase : DbContext where T : DbContext 16 | { 17 | public DbContextBase(string connectString) 18 | : base(new DbContextOptionsBuilder().UseSqlServer(connectString).Options) 19 | { 20 | this.ConnectionString = connectString; 21 | this.Database.EnsureDeleted(); 22 | this.Database.EnsureCreated(); 23 | } 24 | 25 | public string ConnectionString { get; private set; } 26 | 27 | 28 | } 29 | 30 | public static class TestUtils 31 | { 32 | public static async Task FinishesIn(this Task t,TimeSpan ts) 33 | { 34 | var timeOutTask = Task.Delay(ts); 35 | return t == await Task.WhenAny(t, timeOutTask); 36 | } 37 | 38 | public static SqlConnection GetLocalDB(string dbName) 39 | { 40 | using (var conn = new SqlConnection(GetLocalDBConnectionString("master", false))) 41 | { 42 | conn.Open(); 43 | var command = conn.CreateCommand(); 44 | command.CommandText = $"IF db_id('DataflowEx-TestDB-{dbName}') is null BEGIN CREATE DATABASE [DataflowEx-TestDB-{dbName}] END;"; 45 | command.ExecuteNonQuery(); 46 | } 47 | 48 | var connToDBName = new SqlConnection(GetLocalDBConnectionString(dbName)); 49 | connToDBName.Open(); 50 | return connToDBName; 51 | } 52 | 53 | public static string GetLocalDBConnectionString(string dbName = null, bool addPrefix = true) 54 | { 55 | if (dbName == null) 56 | { 57 | var callingMethod = new StackTrace().GetFrame(1).GetMethod() as MethodInfo; 58 | dbName = callingMethod.DeclaringType.Name + "-" + callingMethod.Name; 59 | } 60 | 61 | SqlConnectionStringBuilder builder = new SqlConnectionStringBuilder(); 62 | builder.DataSource = "localhost"; // update me 63 | builder.UserID = "sa"; // update me 64 | builder.Password = TestBootstrapper.s_saPassword; 65 | builder.InitialCatalog = addPrefix ? $"DataflowEx-TestDB-{dbName}" : dbName; 66 | return builder.ConnectionString; 67 | } 68 | 69 | public static byte[] ToByteArray(this string s) 70 | { 71 | return Encoding.UTF8.GetBytes(s); 72 | } 73 | 74 | public static bool ArrayEqual(byte[] a, byte[] b) 75 | { 76 | if (a.Length == b.Length) 77 | { 78 | for (int i = 0; i < a.Length; i++) 79 | { 80 | if (a[i] != b[i]) return false; 81 | } 82 | 83 | return true; 84 | } 85 | else 86 | { 87 | return false; 88 | } 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.TestHelper/App.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 |
6 | 7 | 8 |
9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.TestHelper/Gridsum.DataflowEx.TestHelper.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp2.0 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.TestHelper/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.TestHelper 8 | { 9 | using Gridsum.DataflowEx.Test.ETL; 10 | 11 | class Program 12 | { 13 | static void Main(string[] args) 14 | { 15 | new TestDataJoiner().TestDataJoinerJoining().Wait(); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 2013 4 | VisualStudioVersion = 12.0.30501.0 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Gridsum.DataflowEx", "Gridsum.DataflowEx\Gridsum.DataflowEx.csproj", "{A4FD6288-AE2E-4C28-B7D4-D317D1601E86}" 7 | EndProject 8 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{77014771-4A9D-4A59-AB9D-B587668F5233}" 9 | ProjectSection(SolutionItems) = preProject 10 | .nuget\NuGet.Config = .nuget\NuGet.Config 11 | .nuget\NuGet.exe = .nuget\NuGet.exe 12 | .nuget\NuGet.targets = .nuget\NuGet.targets 13 | EndProjectSection 14 | EndProject 15 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Gridsum.DataflowEx.Test", "Gridsum.DataflowEx.Test\Gridsum.DataflowEx.Test.csproj", "{E5089AD2-0BE8-48E8-B361-A4C38B016C76}" 16 | EndProject 17 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Gridsum.DataflowEx.TestHelper", "Gridsum.DataflowEx.TestHelper\Gridsum.DataflowEx.TestHelper.csproj", "{69654373-6EC8-4005-8441-6F3B8368AEF8}" 18 | EndProject 19 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Gridsum.DataflowEx.Demo", "Gridsum.DataflowEx.Demo\Gridsum.DataflowEx.Demo.csproj", "{6DE0770B-8795-4F64-9AEE-4BD2550009FB}" 20 | EndProject 21 | Global 22 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 23 | Debug|Any CPU = Debug|Any CPU 24 | Release|Any CPU = Release|Any CPU 25 | EndGlobalSection 26 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 27 | {A4FD6288-AE2E-4C28-B7D4-D317D1601E86}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 28 | {A4FD6288-AE2E-4C28-B7D4-D317D1601E86}.Debug|Any CPU.Build.0 = Debug|Any CPU 29 | {A4FD6288-AE2E-4C28-B7D4-D317D1601E86}.Release|Any CPU.ActiveCfg = Release|Any CPU 30 | {A4FD6288-AE2E-4C28-B7D4-D317D1601E86}.Release|Any CPU.Build.0 = Release|Any CPU 31 | {E5089AD2-0BE8-48E8-B361-A4C38B016C76}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 32 | {E5089AD2-0BE8-48E8-B361-A4C38B016C76}.Debug|Any CPU.Build.0 = Debug|Any CPU 33 | {E5089AD2-0BE8-48E8-B361-A4C38B016C76}.Release|Any CPU.ActiveCfg = Release|Any CPU 34 | {E5089AD2-0BE8-48E8-B361-A4C38B016C76}.Release|Any CPU.Build.0 = Release|Any CPU 35 | {69654373-6EC8-4005-8441-6F3B8368AEF8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 36 | {69654373-6EC8-4005-8441-6F3B8368AEF8}.Debug|Any CPU.Build.0 = Debug|Any CPU 37 | {69654373-6EC8-4005-8441-6F3B8368AEF8}.Release|Any CPU.ActiveCfg = Release|Any CPU 38 | {69654373-6EC8-4005-8441-6F3B8368AEF8}.Release|Any CPU.Build.0 = Release|Any CPU 39 | {6DE0770B-8795-4F64-9AEE-4BD2550009FB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 40 | {6DE0770B-8795-4F64-9AEE-4BD2550009FB}.Debug|Any CPU.Build.0 = Debug|Any CPU 41 | {6DE0770B-8795-4F64-9AEE-4BD2550009FB}.Release|Any CPU.ActiveCfg = Release|Any CPU 42 | {6DE0770B-8795-4F64-9AEE-4BD2550009FB}.Release|Any CPU.Build.0 = Release|Any CPU 43 | EndGlobalSection 44 | GlobalSection(SolutionProperties) = preSolution 45 | HideSolutionNode = FALSE 46 | EndGlobalSection 47 | EndGlobal 48 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/AutoCompletion/AutoCompleteWrapper.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Timers; 6 | using System.Threading.Tasks; 7 | using System.Threading.Tasks.Dataflow; 8 | 9 | namespace Gridsum.DataflowEx.AutoCompletion 10 | { 11 | public class AutoCompleteWrapper : Dataflow 12 | where TIn : ITracableItem 13 | where TOut : ITracableItem 14 | { 15 | private readonly TimeSpan m_processTimeout; 16 | private readonly Timer m_timer; 17 | private Guid? m_last; 18 | private Dataflow m_before; 19 | private Dataflow m_after; 20 | private Dataflow m_Dataflow; 21 | 22 | public AutoCompleteWrapper(Dataflow dataflow, TimeSpan processTimeout, DataflowOptions options) : base(options) 23 | { 24 | m_Dataflow = dataflow; 25 | m_processTimeout = processTimeout; 26 | m_timer = new Timer(); 27 | m_timer.Interval = m_processTimeout.TotalMilliseconds; 28 | m_timer.Elapsed += OnTimerElapsed; 29 | 30 | var before = new TransformBlock(@in => 31 | { 32 | if (m_last == null || @in.UniqueId == m_last.Value) 33 | { 34 | //The last one is back, so there is nothing else in the pipeline. 35 | //Set a timer: if nothing new produced when timer expires, the whole loop ends. 36 | m_timer.Start(); 37 | } 38 | return @in; 39 | }); 40 | 41 | m_before = DataflowUtils.FromBlock(before); 42 | 43 | var after = new TransformBlock(@out => 44 | { 45 | if (@out.UniqueId != Guid.Empty) 46 | { 47 | m_last = @out.UniqueId; 48 | m_timer.Stop(); 49 | } 50 | else 51 | { 52 | LogHelper.Logger.WarnFormat("Empty guid found in output. You may have forgotten to set it."); 53 | } 54 | 55 | return @out; 56 | }); 57 | 58 | m_after = DataflowUtils.FromBlock(after); 59 | 60 | m_before.GoTo(dataflow).GoTo(m_after); 61 | 62 | RegisterChild(m_before); 63 | RegisterChild(dataflow); 64 | RegisterChild(m_after); 65 | } 66 | 67 | void OnTimerElapsed(object sender, ElapsedEventArgs e) 68 | { 69 | LogHelper.Logger.InfoFormat("Auto complete timer elapsed. Shutting down the inner dataflow ({0})..", m_Dataflow.FullName); 70 | 71 | m_before.Complete(); //pass completion down to the chain 72 | } 73 | 74 | public override ISourceBlock OutputBlock 75 | { 76 | get { return m_after.OutputBlock; } 77 | } 78 | 79 | public override ITargetBlock InputBlock 80 | { 81 | get { return m_before.InputBlock; } 82 | } 83 | 84 | public override string Name 85 | { 86 | get 87 | { 88 | return string.Format("{0}-AutoComplete", m_Dataflow.Name); 89 | } 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/AutoCompletion/ControlItem.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.AutoCompletion 8 | { 9 | public class ControlItem : ITracableItem 10 | { 11 | public Guid UniqueId 12 | { 13 | get; 14 | set; 15 | } 16 | 17 | public ControlType Type { get; set; } 18 | } 19 | 20 | //todo: ISpiderComponent1Item -> SpiderRequest, Control 21 | //todo: ISpiderComponent2Item -> SpiderResponse, Control 22 | 23 | 24 | } 25 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/AutoCompletion/ControlType.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | 6 | namespace Gridsum.DataflowEx.AutoCompletion 7 | { 8 | public enum ControlType 9 | { 10 | BeforeLast, 11 | AfterLast 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/AutoCompletion/HeartbeatNode.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using Gridsum.DataflowEx; 7 | using System.Threading.Tasks.Dataflow; 8 | 9 | namespace Gridsum.DataflowEx.AutoCompletion 10 | { 11 | /// 12 | /// A dataflow node used in ring completion detection infrastructure. 13 | /// 14 | public interface IHeartbeatNode : IRingNode 15 | { 16 | long ProcessedItemCount { get; } 17 | 18 | bool NoHeartbeatDuring(Action action); 19 | Task NoHeartbeatDuring(Func action); 20 | } 21 | 22 | /// 23 | /// A default implementation of IHeartbeatNode, used in ring completion detection. 24 | /// 25 | public class HeartbeatNode : Dataflow, IHeartbeatNode 26 | { 27 | private long m_beats; 28 | private TransformBlock m_block; 29 | 30 | public HeartbeatNode(DataflowOptions options) : base(options) 31 | { 32 | m_beats = 0; 33 | 34 | Func f = arg => 35 | { 36 | IsBusy = true; 37 | m_beats++; 38 | IsBusy = false; 39 | return arg; 40 | }; 41 | 42 | m_block = new TransformBlock(f, options.ToExecutionBlockOption()); 43 | RegisterChild(m_block); 44 | } 45 | 46 | public override ITargetBlock InputBlock 47 | { 48 | get 49 | { 50 | return m_block; 51 | } 52 | } 53 | 54 | public override ISourceBlock OutputBlock 55 | { 56 | get 57 | { 58 | return m_block; 59 | } 60 | } 61 | 62 | public long ProcessedItemCount 63 | { 64 | get 65 | { 66 | return m_beats; 67 | } 68 | } 69 | 70 | public bool NoHeartbeatDuring(Action action) 71 | { 72 | long before = m_beats; 73 | action(); 74 | long after = m_beats; 75 | return after == before; 76 | } 77 | 78 | public async Task NoHeartbeatDuring(Func action) 79 | { 80 | long before = m_beats; 81 | await action().ConfigureAwait(false); 82 | long after = m_beats; 83 | return after == before; 84 | } 85 | 86 | public bool IsBusy { get; private set; } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/AutoCompletion/IRingNode.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.AutoCompletion 8 | { 9 | public interface IRingNode : IDataflow 10 | { 11 | bool IsBusy { get; } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/AutoCompletion/ITracableItem.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.AutoCompletion 8 | { 9 | public interface ITracableItem 10 | { 11 | Guid UniqueId { get; } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/AutoCompletion/RingMonitor.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.AutoCompletion 8 | { 9 | public class RingMonitor 10 | { 11 | private readonly Dataflow m_host; 12 | private readonly IRingNode[] m_ringNodes; 13 | private readonly Lazy m_lazyDisplayName; 14 | private IHeartbeatNode m_hb; 15 | 16 | public RingMonitor(Dataflow host, IRingNode[] ringNodes) 17 | { 18 | if (ringNodes.Length == 0) 19 | { 20 | throw new ArgumentException("The child ring contains nothing"); 21 | } 22 | 23 | m_hb = ringNodes.OfType().FirstOrDefault(); 24 | if (m_hb == null) 25 | { 26 | throw new ArgumentException("A ring must contain at least one IHeartbeatNode"); 27 | } 28 | 29 | var nonChild = ringNodes.FirstOrDefault(r => { return !host.IsMyChild(r); }); 30 | if (nonChild != null) 31 | { 32 | throw new ArgumentException("The child ring contains a non-child: " + nonChild.FullName); 33 | } 34 | 35 | this.m_host = host; 36 | this.m_ringNodes = ringNodes; 37 | 38 | m_lazyDisplayName = new Lazy(this.GetDisplayName); 39 | } 40 | 41 | private string GetDisplayName() 42 | { 43 | string ringString = string.Join("=>", m_ringNodes.Select(n => n.Name)); 44 | return string.Format("({0})", ringString); 45 | } 46 | 47 | public string DisplayName 48 | { 49 | get 50 | { 51 | return m_lazyDisplayName.Value; 52 | } 53 | } 54 | 55 | public async Task StartMonitoring(Task preTask) 56 | { 57 | LogHelper.Logger.InfoFormat("{0} A ring is set up: {1}", m_host.FullName, DisplayName); 58 | await preTask.ConfigureAwait(false); 59 | LogHelper.Logger.InfoFormat("{0} Ring pretask done. Starting check loop for {1}", m_host.FullName, DisplayName); 60 | 61 | while (true) 62 | { 63 | await Task.Delay(m_host.m_dataflowOptions.MonitorInterval).ConfigureAwait(false); 64 | 65 | bool empty = false; 66 | 67 | if (m_hb.NoHeartbeatDuring(() => { empty = this.IsRingEmpty(); }) && empty) 68 | { 69 | LogHelper.Logger.DebugFormat("{0} 1st level empty ring check passed for {1}", m_host.FullName, this.DisplayName); 70 | 71 | bool noHeartbeat = await m_hb.NoHeartbeatDuring( 72 | async () => 73 | { 74 | //trigger batch 75 | foreach (var batchedFlow in m_ringNodes.OfType()) 76 | { 77 | batchedFlow.TriggerBatch(); 78 | } 79 | 80 | await Task.Delay(m_host.m_dataflowOptions.MonitorInterval).ConfigureAwait(false); 81 | 82 | empty = this.IsRingEmpty(); 83 | }).ConfigureAwait(false); 84 | 85 | if (noHeartbeat && empty) 86 | { 87 | LogHelper.Logger.DebugFormat("{0} 2nd level empty ring check passed for {1}", m_host.FullName, this.DisplayName); 88 | 89 | m_hb.Complete(); //start the completion domino :) 90 | break; 91 | } 92 | else 93 | { 94 | //batched flow dumped some hidden elements to the flow, go back to the loop 95 | LogHelper.Logger.DebugFormat("{0} New elements entered the ring by batched flows ({1}). Will fall back to 1st level empty ring check.", m_host.FullName, this.DisplayName); 96 | } 97 | } 98 | } 99 | 100 | LogHelper.Logger.InfoFormat("{0} Ring completion detected! Completion triggered on heartbeat node: {1}", m_host.FullName, DisplayName); 101 | } 102 | 103 | private bool IsRingEmpty() 104 | { 105 | var hb = m_ringNodes.OfType().First(); 106 | 107 | if (m_ringNodes.All(r => !r.IsBusy)) 108 | { 109 | long before = hb.ProcessedItemCount; 110 | int buffered = m_ringNodes.Sum(r => r.BufferedCount) + m_ringNodes[0].BufferedCount; 111 | long after = hb.ProcessedItemCount; 112 | 113 | if (buffered == 0 && before == after) 114 | { 115 | return true; 116 | } 117 | } 118 | 119 | return false; 120 | } 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/AutoCompletion/TracableItemBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Gridsum.DataflowEx.AutoCompletion 4 | { 5 | public class TracableItemBase : ITracableItem 6 | { 7 | public TracableItemBase() : this(Guid.NewGuid()) 8 | { 9 | } 10 | 11 | public TracableItemBase(Guid guid) 12 | { 13 | this.UniqueId = guid; 14 | } 15 | 16 | public Guid UniqueId { get; private set; } 17 | } 18 | } -------------------------------------------------------------------------------- /Gridsum.DataflowEx/Blocks/TransformAndLinkPropagator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using System.Threading.Tasks.Dataflow; 7 | 8 | namespace Gridsum.DataflowEx.Blocks 9 | { 10 | internal sealed class TransformAndLinkPropagator : IPropagatorBlock, ITargetBlock, ISourceBlock, IDataflowBlock 11 | { 12 | private readonly ISourceBlock m_source; 13 | private readonly ITargetBlock m_target; 14 | private readonly Predicate m_userProvidedPredicate; 15 | private readonly Func m_transform; 16 | 17 | Task IDataflowBlock.Completion 18 | { 19 | get 20 | { 21 | return this.m_source.Completion; 22 | } 23 | } 24 | 25 | internal TransformAndLinkPropagator(ISourceBlock source, ITargetBlock target, Predicate predicate, Func transform) 26 | { 27 | this.m_source = source; 28 | this.m_target = target; 29 | this.m_transform = transform; 30 | this.m_userProvidedPredicate = predicate; 31 | } 32 | 33 | private bool RunPredicate(TIn item) 34 | { 35 | return this.m_userProvidedPredicate(item); 36 | } 37 | 38 | DataflowMessageStatus ITargetBlock.OfferMessage(DataflowMessageHeader messageHeader, TIn messageValue, ISourceBlock source, bool consumeToAccept) 39 | { 40 | if (!messageHeader.IsValid) 41 | throw new ArgumentException("message header is invalid", "messageHeader"); 42 | if (source == null) 43 | throw new ArgumentNullException("source"); 44 | if (this.RunPredicate(messageValue)) 45 | return this.m_target.OfferMessage(messageHeader, m_transform(messageValue), (ISourceBlock)this, consumeToAccept); 46 | else 47 | return DataflowMessageStatus.Declined; 48 | } 49 | 50 | TOut ISourceBlock.ConsumeMessage(DataflowMessageHeader messageHeader, ITargetBlock target, out bool messageConsumed) 51 | { 52 | TIn raw = this.m_source.ConsumeMessage(messageHeader, (ITargetBlock)this, out messageConsumed); 53 | return m_transform(raw); 54 | } 55 | 56 | bool ISourceBlock.ReserveMessage(DataflowMessageHeader messageHeader, ITargetBlock target) 57 | { 58 | return this.m_source.ReserveMessage(messageHeader, (ITargetBlock)this); 59 | } 60 | 61 | void ISourceBlock.ReleaseReservation(DataflowMessageHeader messageHeader, ITargetBlock target) 62 | { 63 | this.m_source.ReleaseReservation(messageHeader, (ITargetBlock)this); 64 | } 65 | 66 | void IDataflowBlock.Complete() 67 | { 68 | this.m_target.Complete(); 69 | } 70 | 71 | void IDataflowBlock.Fault(Exception exception) 72 | { 73 | this.m_target.Fault(exception); 74 | } 75 | 76 | IDisposable ISourceBlock.LinkTo(ITargetBlock target, DataflowLinkOptions linkOptions) 77 | { 78 | throw new NotSupportedException(); 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/DataBroadcaster.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using System.Threading.Tasks.Dataflow; 7 | 8 | namespace Gridsum.DataflowEx 9 | { 10 | using System.Collections.Immutable; 11 | 12 | /// 13 | /// BroadcastBlock in TPL Dataflow only pushes latest data (if destination is full) and causes data loss. 14 | /// That's why we need this DataBroadcaster which preserves a 100% same copy of the data stream 15 | /// 16 | /// The input and output type of the data flow 17 | public class DataBroadcaster : Dataflow 18 | { 19 | //todo: fix race condition 20 | private ImmutableList> m_copyBuffers; 21 | 22 | private readonly TransformBlock m_transformBlock; 23 | 24 | /// 25 | /// Construct an DataBroadcaster instance 26 | /// 27 | public DataBroadcaster() : this(DataflowOptions.Default) {} 28 | 29 | /// 30 | /// Construct an DataBroadcaster instance 31 | /// 32 | /// the option of this dataflow 33 | public DataBroadcaster(DataflowOptions dataflowOptions) : this(null, dataflowOptions) {} 34 | 35 | /// 36 | /// Construct an DataBroadcaster instance 37 | /// 38 | /// The copy function when broadcasting 39 | /// the option of this dataflow 40 | public DataBroadcaster(Func copyFunc, DataflowOptions dataflowOptions) : base(dataflowOptions) 41 | { 42 | m_copyBuffers = ImmutableList>.Empty; 43 | 44 | m_transformBlock = new TransformBlock( 45 | async arg => 46 | { 47 | T copy = copyFunc == null ? arg : copyFunc(arg); 48 | foreach (var buffer in m_copyBuffers) 49 | { 50 | await buffer.SendAsync(copy).ConfigureAwait(false); 51 | } 52 | return arg; 53 | }, dataflowOptions.ToExecutionBlockOption()); 54 | 55 | RegisterChild(m_transformBlock); 56 | } 57 | 58 | /// 59 | /// See 60 | /// 61 | public override ITargetBlock InputBlock 62 | { 63 | get { return m_transformBlock; } 64 | } 65 | 66 | /// 67 | /// See 68 | /// 69 | public override ISourceBlock OutputBlock 70 | { 71 | get { return m_transformBlock; } 72 | } 73 | 74 | /// 75 | /// Link the copied data stream to another block 76 | /// 77 | private void LinkCopyTo(IDataflow other) 78 | { 79 | //first, create a new copy block 80 | Dataflow copyBuffer = new BufferBlock(m_dataflowOptions.ToGroupingBlockOption()).ToDataflow(m_dataflowOptions); 81 | 82 | RegisterChild(copyBuffer); 83 | copyBuffer.RegisterDependency(m_transformBlock); 84 | 85 | var afterAdd = ImmutableUtils.AddOptimistically(ref m_copyBuffers, copyBuffer); 86 | 87 | copyBuffer.Name = "Buffer" + afterAdd.Count; 88 | copyBuffer.LinkTo(other); 89 | } 90 | 91 | /// 92 | /// See 93 | /// 94 | public override IDataflow GoTo(IDataflow other, Predicate predicate) 95 | { 96 | if (predicate != null) 97 | { 98 | throw new ArgumentException("DataBroadcaster does not support predicate linking", "predicate"); 99 | } 100 | 101 | if (m_condBuilder.Count == 0) //not linked to any target yet 102 | { 103 | //link first output as primary output 104 | base.GoTo(other); 105 | } 106 | else 107 | { 108 | this.LinkCopyTo(other); 109 | } 110 | 111 | LogHelper.Logger.InfoFormat("{0} now links to its {1}th target ({2})", this.FullName, m_copyBuffers.Count + 1, other.Name); 112 | return other; 113 | } 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/DataDispatcher.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx 8 | { 9 | using System.Collections.Concurrent; 10 | using System.Collections.Immutable; 11 | using System.Threading.Tasks.Dataflow; 12 | 13 | /// 14 | /// Provides an abstract flow that dispatch inputs to multiple child flows by a special dispatch function, which is 15 | /// useful in situations that you want to group inputs by a certain property and let specific child flows to take 16 | /// care of different groups independently. DataDispatcher also helps creating and maintaining child flows dynamically 17 | /// in a thread-safe way. 18 | /// 19 | /// Type of input items of this dispatcher flow 20 | /// Type of the dispatch key to group input items 21 | /// 22 | /// This flow guarantees an input goes to only ONE of the child flows. Notice the difference comparing to DataBroadcaster, which 23 | /// gives the input to EVERY flow it is linked to. 24 | /// 25 | public abstract class DataDispatcher : Dataflow 26 | { 27 | protected ActionBlock m_dispatcherBlock; 28 | protected ConcurrentDictionary>> m_destinations; 29 | private Func>> m_initer; 30 | 31 | /// 32 | /// Construct an DataDispatcher instance 33 | /// 34 | /// The dispatch function 35 | public DataDispatcher(Func dispatchFunc) : this(dispatchFunc, DataflowOptions.Default) 36 | { 37 | } 38 | 39 | /// 40 | /// Construct an DataDispatcher instance 41 | /// 42 | /// The dispatch function 43 | /// Option for this dataflow 44 | public DataDispatcher(Func dispatchFunc, DataflowOptions option) 45 | : this(dispatchFunc, EqualityComparer.Default, option) 46 | { 47 | } 48 | 49 | /// 50 | /// Construct an DataDispatcher instance 51 | /// 52 | /// The dispatch function 53 | /// The key comparer for this dataflow 54 | /// Option for this dataflow 55 | public DataDispatcher(Func dispatchFunc, EqualityComparer keyComparer, DataflowOptions option) 56 | : base(option) 57 | { 58 | m_destinations = new ConcurrentDictionary>>(keyComparer); 59 | 60 | m_initer = key => new Lazy>( 61 | () => 62 | { 63 | var child = this.CreateChildFlow(key); 64 | RegisterChild(child); 65 | child.RegisterDependency(m_dispatcherBlock); 66 | return child; 67 | }); 68 | 69 | m_dispatcherBlock = new ActionBlock(async 70 | input => 71 | { 72 | var childFlow = m_destinations.GetOrAdd(dispatchFunc(input), m_initer).Value; 73 | await childFlow.SendAsync(input).ConfigureAwait(false); 74 | }, option.ToExecutionBlockOption()); 75 | 76 | RegisterChild(m_dispatcherBlock); 77 | } 78 | 79 | /// 80 | /// Create the child flow on-the-fly by the dispatch key 81 | /// 82 | /// The unique key to create and identify the child flow 83 | /// A new child dataflow which is responsible for processing items having the given dispatch key 84 | /// 85 | /// The dispatch key should have a one-one relatioship with child flow 86 | /// 87 | protected abstract Dataflow CreateChildFlow(TKey dispatchKey); 88 | 89 | /// 90 | /// See 91 | /// 92 | public override ITargetBlock InputBlock 93 | { 94 | get 95 | { 96 | return m_dispatcherBlock; 97 | } 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/Databases/DBColumnMapping.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Gridsum.DataflowEx.Databases 4 | { 5 | /// 6 | /// Represents a mapping from C# property to a DB column 7 | /// 8 | [AttributeUsage(AttributeTargets.Property, AllowMultiple = true, Inherited = true)] 9 | public class DBColumnMapping : Attribute 10 | { 11 | /// 12 | /// Constructs a mapping from the tagged property to a db column name by property name 13 | /// 14 | /// The label to help categorize all column mappings 15 | public DBColumnMapping(string destLabel) 16 | : this(destLabel, -1, null, null, ColumnMappingOption.Mandatory) 17 | { 18 | } 19 | 20 | /// 21 | /// Constructs a mapping from the tagged property to a db column by column offset in the db table 22 | /// 23 | /// The label to help categorize all column mappings 24 | /// the column offset in the db table 25 | /// default value for the property if the propety is null 26 | /// The option, or priority of this mapping 27 | public DBColumnMapping(string destLabel, int destColumnOffset, object defaultValue = null, ColumnMappingOption option = ColumnMappingOption.Mandatory) 28 | : this(destLabel, destColumnOffset, null, defaultValue, option) { } 29 | 30 | /// 31 | /// Constructs a mapping from the tagged property to a db column by column name in the db table 32 | /// 33 | /// The label to help categorize all column mappings 34 | /// the column name in the db table 35 | /// default value for the property if the propety is null 36 | /// The option, or priority of this mapping 37 | public DBColumnMapping(string destLabel, string destColumnName, object defaultValue = null, ColumnMappingOption option = ColumnMappingOption.Mandatory) 38 | : this(destLabel, -1, destColumnName, defaultValue, option) 39 | { 40 | } 41 | 42 | protected DBColumnMapping(string destLabel, int destColumnOffset, string destColumnName, object defaultValue, ColumnMappingOption option) 43 | { 44 | DestLabel = destLabel; 45 | DestColumnOffset = destColumnOffset; 46 | DestColumnName = destColumnName; 47 | DefaultValue = defaultValue; 48 | this.Option = option; 49 | } 50 | 51 | /// 52 | /// The label to help categorize all column mappings 53 | /// 54 | public string DestLabel { get; private set; } 55 | 56 | /// 57 | /// Column offset in the db table 58 | /// 59 | public int DestColumnOffset { get; set; } 60 | 61 | /// 62 | /// Column name in the db table 63 | /// 64 | public string DestColumnName { get; set; } 65 | 66 | /// 67 | /// Default value for the property if the propety is null 68 | /// 69 | public object DefaultValue { get; internal set; } 70 | 71 | /// 72 | /// The option, or priority of this mapping 73 | /// 74 | public ColumnMappingOption Option { get; set; } 75 | 76 | internal PropertyTreeNode Host { get; set; } 77 | 78 | internal bool IsDestColumnOffsetOk() 79 | { 80 | return DestColumnOffset >= 0; 81 | } 82 | 83 | internal bool IsDestColumnNameOk() 84 | { 85 | return string.IsNullOrWhiteSpace(DestColumnName) == false; 86 | } 87 | 88 | public override string ToString() 89 | { 90 | return string.Format( 91 | "[ColName:{0}, ColOffset:{1}, DefaultValue:{2}, Option: {3}]", 92 | this.DestColumnName, 93 | this.DestColumnOffset, 94 | this.DefaultValue, 95 | this.Option); 96 | } 97 | } 98 | 99 | /// 100 | /// The option of a db column mapping 101 | /// 102 | public enum ColumnMappingOption 103 | { 104 | /// 105 | /// Used typically by external mappings to overwrite Mandatory option on tagged attribute. The column must exist in the destination table. 106 | /// 107 | Overwrite = short.MinValue, 108 | 109 | /// 110 | /// The column must exist in the destination table 111 | /// 112 | Mandatory = 1, 113 | 114 | /// 115 | /// The column mapping can be ignored if the column is not found in the destination table 116 | /// This option is useful when you have one dest label for multiple different tables 117 | /// 118 | Optional = 2 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/Databases/DBColumnPath.cs: -------------------------------------------------------------------------------- 1 | namespace Gridsum.DataflowEx.Databases 2 | { 3 | using System; 4 | 5 | /// 6 | /// Represents a node in the middle of a column property path, tagged on a expandable property 7 | /// 8 | [AttributeUsage(AttributeTargets.Property, AllowMultiple = true, Inherited = true)] 9 | public class DBColumnPath : Attribute 10 | { 11 | protected readonly DBColumnPathOptions m_options; 12 | 13 | /// 14 | /// Constructs a DBColumnPath instance 15 | /// 16 | /// Options for a property path 17 | public DBColumnPath(DBColumnPathOptions options) 18 | { 19 | this.m_options = options; 20 | } 21 | 22 | internal bool HasOption(DBColumnPathOptions option) 23 | { 24 | return m_options.HasFlag(option); 25 | } 26 | } 27 | 28 | /// 29 | /// Options for a property path 30 | /// 31 | [Flags] 32 | public enum DBColumnPathOptions 33 | { 34 | /// 35 | /// Tells the type accessor engine that a property tagged with this attribute will never 36 | /// be null so that the engine will produce faster code and deliver better performance in DbBulkInserter. 37 | /// 38 | /// 39 | /// If a tagged property happens to be null. NullReferenceException will be thrown at runtime by DbBulkInserter. 40 | /// This is the cost of performance. 41 | /// 42 | DoNotGenerateNullCheck = 1, 43 | 44 | /// 45 | /// Tells the type accessor engine not to expand and look into any property tagged with this attribute. 46 | /// 47 | DoNotExpand = 2, 48 | } 49 | } -------------------------------------------------------------------------------- /Gridsum.DataflowEx/Databases/DbBulkInserter.cs: -------------------------------------------------------------------------------- 1 | using System.Data.SqlClient; 2 | using System.Threading.Tasks; 3 | 4 | namespace Gridsum.DataflowEx.Databases 5 | { 6 | using System; 7 | 8 | /// 9 | /// A bulk inserter that treats the whole lifecycle of the dataflow as a whole transaction 10 | /// (no matter how many inserted batches) 11 | /// 12 | /// Type of the strongly-typed objects to insert 13 | public class DbBulkInserter : DbBulkInserterBase where T : class 14 | { 15 | protected SqlConnection m_longConnection; 16 | protected SqlTransaction m_transaction; 17 | 18 | public DbBulkInserter(string connectionString, string destTable, DataflowOptions options, string destLabel, int bulkSize = 8192, string dbBulkInserterName = null, PostBulkInsertDelegate postBulkInsert = null) 19 | : this(new TargetTable(destLabel, connectionString, destTable), options, bulkSize, dbBulkInserterName, postBulkInsert) 20 | { 21 | } 22 | 23 | public DbBulkInserter(TargetTable targetTable, DataflowOptions options, int bulkSize = 8192, string dbBulkInserterName = null, PostBulkInsertDelegate postBulkInsert = null) 24 | : base(targetTable, options, bulkSize, dbBulkInserterName, postBulkInsert) 25 | { 26 | m_longConnection = new SqlConnection(targetTable.ConnectionString); 27 | m_longConnection.Open(); 28 | 29 | m_transaction = m_longConnection.BeginTransaction(); 30 | } 31 | 32 | protected override void CleanUp(Exception dataflowException) 33 | { 34 | if (dataflowException != null) 35 | { 36 | m_logger.ErrorFormat("{0} Rolling back all changes...", this.FullName, dataflowException); 37 | m_transaction.Rollback(); 38 | m_logger.InfoFormat("{0} Changes successfully rolled back", this.FullName); 39 | } 40 | else 41 | { 42 | m_logger.InfoFormat("{0} bulk insertions are done. Committing transaction...", this.FullName); 43 | m_transaction.Commit(); 44 | m_logger.DebugFormat("{0} Transaction successfully committed.", this.FullName); 45 | } 46 | 47 | m_longConnection.Close(); 48 | } 49 | 50 | protected override async Task DumpToDBAsync(T[] data, TargetTable targetTable) 51 | { 52 | using (var bulkReader = new BulkDataReader(m_typeAccessor, data)) 53 | { 54 | try 55 | { 56 | using (var bulkCopy = new SqlBulkCopy(m_longConnection, SqlBulkCopyOptions.TableLock, m_transaction)) 57 | { 58 | foreach (SqlBulkCopyColumnMapping map in bulkReader.ColumnMappings) 59 | { 60 | bulkCopy.ColumnMappings.Add(map); 61 | } 62 | 63 | bulkCopy.DestinationTableName = targetTable.TableName; 64 | bulkCopy.BulkCopyTimeout = (int)TimeSpan.FromMinutes(30).TotalMilliseconds; 65 | bulkCopy.BatchSize = m_bulkSize; 66 | 67 | // Write from the source to the destination. 68 | await bulkCopy.WriteToServerAsync(bulkReader).ConfigureAwait(false); 69 | } 70 | } 71 | catch (Exception e) 72 | { 73 | if (e is NullReferenceException) 74 | { 75 | m_logger.Error($"{this.FullName} NullReferenceException occurred in bulk insertion for type {typeof(T).Name}. This is probably caused by forgetting assigning value to a [NoNullCheck] attribute when constructing your object."); 76 | } 77 | //As this is an unrecoverable exception, rethrow it 78 | throw; 79 | } 80 | await this.OnPostBulkInsert(m_longConnection, targetTable, data).ConfigureAwait(false); 81 | } 82 | 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/Databases/EagerDbBulkInserter.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.Databases 8 | { 9 | using System.Data.SqlClient; 10 | 11 | /// 12 | /// A bulk inserter which treats every batch as a standalone transaction. 13 | /// 14 | /// Type of the strongly-typed objects to insert 15 | public class EagerDbBulkInserter : DbBulkInserterBase where T: class 16 | { 17 | public EagerDbBulkInserter(string connectionString, string destTable, DataflowOptions options, string destLabel, int bulkSize = 8192, string dbBulkInserterName = null, PostBulkInsertDelegate postBulkInsert = null) 18 | : base(connectionString, destTable, options, destLabel, bulkSize, dbBulkInserterName, postBulkInsert) 19 | { 20 | } 21 | 22 | public EagerDbBulkInserter(TargetTable targetTable, DataflowOptions options, int bulkSize = 8192, string dbBulkInserterName = null, PostBulkInsertDelegate postBulkInsert = null) 23 | : base(targetTable, options, bulkSize, dbBulkInserterName, postBulkInsert) 24 | { 25 | } 26 | 27 | protected async override Task DumpToDBAsync(T[] data, TargetTable targetTable) 28 | { 29 | using (var bulkReader = new BulkDataReader(m_typeAccessor, data)) 30 | { 31 | using (var conn = new SqlConnection(targetTable.ConnectionString)) 32 | { 33 | await conn.OpenAsync().ConfigureAwait(false); 34 | 35 | var transaction = conn.BeginTransaction(); 36 | try 37 | { 38 | using (var bulkCopy = new SqlBulkCopy(conn, SqlBulkCopyOptions.TableLock, transaction)) 39 | { 40 | foreach (SqlBulkCopyColumnMapping map in bulkReader.ColumnMappings) 41 | { 42 | bulkCopy.ColumnMappings.Add(map); 43 | } 44 | 45 | bulkCopy.DestinationTableName = targetTable.TableName; 46 | bulkCopy.BulkCopyTimeout = (int)TimeSpan.FromMinutes(30).TotalMilliseconds; 47 | bulkCopy.BatchSize = m_bulkSize; 48 | 49 | // Write from the source to the destination. 50 | await bulkCopy.WriteToServerAsync(bulkReader).ConfigureAwait(false); 51 | } 52 | } 53 | catch (Exception e) 54 | { 55 | if (e is NullReferenceException) 56 | { 57 | m_logger.Error($"{this.FullName} NullReferenceException occurred in bulk insertion for type {typeof(T).Name}. This is probably caused by forgetting assigning value to a [NoNullCheck] attribute when constructing your object."); 58 | } 59 | 60 | m_logger.ErrorFormat("{0} Bulk insertion failed. Rolling back all changes...", this.FullName, e); 61 | transaction.Rollback(); 62 | m_logger.InfoFormat("{0} Changes successfully rolled back", this.FullName); 63 | 64 | //As this is an unrecoverable exception, rethrow it 65 | throw; 66 | } 67 | 68 | transaction.Commit(); 69 | await this.OnPostBulkInsert(conn, targetTable, data).ConfigureAwait(false); 70 | } 71 | } 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/Databases/InvalidDBColumnMappingException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.Databases 8 | { 9 | public class InvalidDBColumnMappingException : Exception 10 | { 11 | private readonly string m_description; 12 | private readonly DBColumnMapping m_mapping; 13 | private readonly LeafPropertyNode m_node; 14 | 15 | public InvalidDBColumnMappingException(string description, DBColumnMapping mapping, LeafPropertyNode node) : base() 16 | { 17 | this.m_description = description; 18 | this.m_mapping = mapping; 19 | this.m_node = node; 20 | } 21 | 22 | public override string Message 23 | { 24 | get 25 | { 26 | return string.Format("{0} Mapping: {1} LeafNode: {2}", m_description, m_mapping, m_node); 27 | } 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/Databases/MultiDbBulkInserter.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Threading.Tasks; 4 | using System.Threading.Tasks.Dataflow; 5 | 6 | namespace Gridsum.DataflowEx.Databases 7 | { 8 | /// 9 | /// The class helps you to bulk insert parsed objects to multiple database tables (e.g. group by profileId) 10 | /// 11 | /// The db-mapped type of parsed objects (usually generated by EF/linq2sql) 12 | public class MultiDbBulkInserter : DataDispatcher where T:class 13 | { 14 | private Func m_connectionGetter; 15 | private string m_destTable; 16 | private readonly string m_destLabel; 17 | private DataflowOptions m_options; 18 | private int m_bulkSize; 19 | private readonly string m_displayName; 20 | private readonly PostBulkInsertDelegate m_postBulkInsert; 21 | 22 | /// 23 | /// Constructs a MultiDbBulkInserter instance. 24 | /// 25 | /// The option for this dataflow 26 | /// The dispatch function to decide which child flow the incoming objects will be delivered to 27 | /// Connection string generator for child flows 28 | /// The table name in database to bulk insert into for every child 29 | /// The mapping label to help choose among all column mappings 30 | /// The bulk size to insert in a batch. Default to 8192. 31 | /// A given name of this multi bulk inserter (would be nice for logging) 32 | /// A delegate that enables you to inject some customized work whenever a bulk insert is done 33 | public MultiDbBulkInserter(DataflowOptions options, 34 | Func dispatchFunc, 35 | Func connectionGetter, 36 | string destTable, 37 | string destLabel, 38 | int bulkSize = 4096 * 2, 39 | string displayName = null, 40 | PostBulkInsertDelegate postBulkInsert = null) 41 | : base(dispatchFunc, options) 42 | { 43 | m_options = options; 44 | m_connectionGetter = connectionGetter; 45 | m_destTable = destTable; 46 | m_destLabel = destLabel; 47 | m_bulkSize = bulkSize; 48 | m_displayName = displayName; 49 | this.m_postBulkInsert = postBulkInsert; 50 | } 51 | 52 | protected override Dataflow CreateChildFlow(int dispatchKey) 53 | { 54 | return new DbBulkInserter( 55 | this.m_connectionGetter(dispatchKey), 56 | this.m_destTable, 57 | this.m_options, 58 | this.m_destLabel, 59 | this.m_bulkSize, 60 | string.Format("childDbBulkInserter_{0}", dispatchKey), 61 | this.m_postBulkInsert); 62 | } 63 | 64 | /// 65 | /// See 66 | /// 67 | public override string Name 68 | { 69 | get 70 | { 71 | return m_displayName ?? base.Name; 72 | } 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/Databases/NoNullCheckAttribute.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.Databases 8 | { 9 | /// 10 | /// Tells the type accessor engine that a property tagged with this attribute will never 11 | /// be null so that the engine will produce faster code and deliver better performance in DbBulkInserter. 12 | /// 13 | /// 14 | /// If a tagged property happens to be null. NullReferenceException will be thrown at runtime by DbBulkInserter. 15 | /// This is the cost of performance. 16 | /// 17 | [Obsolete("Please use [DBColumnPath(DBColumnPathOptions.DoNotGenerateNullCheck)] instead")] 18 | [AttributeUsage(AttributeTargets.Property, AllowMultiple = true, Inherited = true)] 19 | public class NoNullCheckAttribute : Attribute 20 | { 21 | public NoNullCheckAttribute() 22 | { 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/Databases/PostBulkInsertDelegate.cs: -------------------------------------------------------------------------------- 1 | namespace Gridsum.DataflowEx.Databases 2 | { 3 | using System.Collections.Generic; 4 | using System.Data.SqlClient; 5 | using System.Threading.Tasks; 6 | 7 | /// 8 | /// The handler which allows you to take control after a bulk insertion succeeds. (e.g. you may want to 9 | /// execute a stored prodecure after every bulk insertion) 10 | /// 11 | /// The connection used by previous bulk insert (already opened) 12 | /// The destination table of the bulk insertion 13 | /// The inserted data of this round of bulk insertion 14 | /// A task represents the state of the post bulk insert job (so you can use await in the delegate) 15 | public delegate Task PostBulkInsertDelegate(SqlConnection connection, TargetTable target, ICollection insertedData); 16 | } -------------------------------------------------------------------------------- /Gridsum.DataflowEx/Databases/TargetTable.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.Databases 8 | { 9 | /// 10 | /// Represents a target for db bulk insertion and the chosen mapping category 11 | /// 12 | public class TargetTable 13 | { 14 | public readonly string ConnectionString; 15 | public readonly string TableName; 16 | public readonly string DestLabel; 17 | 18 | public TargetTable(string destLabel, string connectionString, string tableName) 19 | { 20 | this.DestLabel = destLabel; 21 | this.ConnectionString = connectionString; 22 | this.TableName = tableName; 23 | } 24 | 25 | protected bool Equals(TargetTable other) 26 | { 27 | return string.Equals(this.ConnectionString, other.ConnectionString) && string.Equals(this.TableName, other.TableName) && string.Equals(this.DestLabel, other.DestLabel); 28 | } 29 | 30 | public override bool Equals(object obj) 31 | { 32 | if (ReferenceEquals(null, obj)) 33 | { 34 | return false; 35 | } 36 | if (ReferenceEquals(this, obj)) 37 | { 38 | return true; 39 | } 40 | if (obj.GetType() != this.GetType()) 41 | { 42 | return false; 43 | } 44 | return Equals((TargetTable)obj); 45 | } 46 | 47 | public override int GetHashCode() 48 | { 49 | unchecked 50 | { 51 | int hashCode = (this.ConnectionString != null ? this.ConnectionString.GetHashCode() : 0); 52 | hashCode = (hashCode * 397) ^ (this.TableName != null ? this.TableName.GetHashCode() : 0); 53 | hashCode = (hashCode * 397) ^ (this.DestLabel != null ? this.DestLabel.GetHashCode() : 0); 54 | return hashCode; 55 | } 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/DataflowBlockExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Runtime.CompilerServices; 5 | using System.Text; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | using System.Threading.Tasks.Dataflow; 9 | using Gridsum.DataflowEx.Exceptions; 10 | 11 | namespace Gridsum.DataflowEx 12 | { 13 | using System.Collections.Immutable; 14 | using Gridsum.DataflowEx.Blocks; 15 | 16 | public static class DataflowBlockExtensions 17 | { 18 | [Obsolete] 19 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 20 | public static void SafePost(this ITargetBlock target, TIn item, int interval = 200, int retryCount = 3) 21 | { 22 | bool posted = target.Post(item); 23 | if (posted) return; 24 | 25 | if (target.Completion.IsCompleted) 26 | { 27 | string msg = string.Format( 28 | "Safe post to {0} failed as the block is {1}", 29 | Utils.GetFriendlyName(target.GetType()), 30 | target.Completion.Status); 31 | 32 | throw new PostToBlockFailedException(msg); 33 | } 34 | 35 | for (int i = 1; i <= retryCount; i++) 36 | { 37 | Thread.Sleep(interval * i); 38 | posted = target.Post(item); 39 | if (posted) return; 40 | } 41 | 42 | throw new PostToBlockFailedException( 43 | string.Format("Safe post to {0} failed after {1} retries", Utils.GetFriendlyName(target.GetType()), retryCount)); 44 | } 45 | 46 | public static Tuple GetBufferCount(this IDataflowBlock block) 47 | { 48 | dynamic b = block; 49 | 50 | var blockGenericType = block.GetType().GetGenericTypeDefinition(); 51 | if (blockGenericType == typeof(TransformBlock<,>) || blockGenericType == typeof(TransformManyBlock<,>)) 52 | { 53 | return new Tuple(b.InputCount, b.OutputCount); 54 | } 55 | 56 | if (blockGenericType == typeof(ActionBlock<>)) 57 | { 58 | return new Tuple(b.InputCount, 0); 59 | } 60 | 61 | if (blockGenericType == typeof (BufferBlock<>)) 62 | { 63 | return new Tuple(0, b.Count);; 64 | } 65 | 66 | if (blockGenericType == typeof (BatchBlock<>)) 67 | { 68 | return new Tuple(0, b.OutputCount * b.BatchSize); 69 | } 70 | 71 | if (block.GetType().Name.StartsWith("EncapsulatingPropagator")) 72 | { 73 | // we wish we could do this: (but it is not feasible due to TPL Dataflow design) 74 | // var t1 = (b.Source as IDataflowBlock).GetBufferCount(); 75 | // var t2 = (b.Target as IDataflowBlock).GetBufferCount(); 76 | // return new Tuple(t1.Item1 + t2.Item1, t1.Item2 + t2.Item2); 77 | 78 | //return 0 for encapuslated block otherwise exception will be thrown 79 | //to monitor encapsulated blocks, user should register underlying blocks as children 80 | return new Tuple(0, 0); 81 | } 82 | 83 | throw new ArgumentException("Fail to auto-detect buffer count of block: " + Utils.GetFriendlyName(block.GetType()), "block"); 84 | } 85 | 86 | 87 | public static IDisposable LinkTo(this ISourceBlock source, ITargetBlock target, DataflowLinkOptions linkOptions, Predicate predicate, Func transform) 88 | { 89 | if (source == null) 90 | throw new ArgumentNullException("source"); 91 | if (target == null) 92 | throw new ArgumentNullException("target"); 93 | if (linkOptions == null) 94 | throw new ArgumentNullException("linkOptions"); 95 | if (predicate == null) 96 | throw new ArgumentNullException("predicate"); 97 | if (transform == null) 98 | throw new ArgumentNullException("transform"); 99 | var TransformAndLinkPropagator = new TransformAndLinkPropagator(source, target, predicate, transform); 100 | return source.LinkTo((ITargetBlock)TransformAndLinkPropagator, linkOptions); 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/DataflowMerger.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using System.Threading.Tasks.Dataflow; 6 | using Gridsum.DataflowEx.Exceptions; 7 | 8 | namespace Gridsum.DataflowEx 9 | { 10 | /// 11 | /// Merges two underlying dataflows so that the combined one looks like a single dataflow from outside 12 | /// 13 | /// Block1 In Type 14 | /// Block1 Out Type & Block2 In Type 15 | /// Block2 Out Type 16 | public class DataflowMerger : Dataflow 17 | { 18 | protected readonly Dataflow m_b1; 19 | protected readonly Dataflow m_b2; 20 | 21 | public DataflowMerger(Dataflow b1, Dataflow b2) : this(b1, b2, b1.DataflowOptions) 22 | { 23 | } 24 | 25 | public DataflowMerger(Dataflow b1, Dataflow b2, DataflowOptions options) : base(options) 26 | { 27 | m_b1 = b1; 28 | m_b2 = b2; 29 | 30 | m_b1.GoTo(m_b2); 31 | 32 | RegisterChild(m_b1); 33 | RegisterChild(m_b2); 34 | } 35 | 36 | public override ISourceBlock OutputBlock 37 | { 38 | get { return m_b2.OutputBlock; } 39 | } 40 | 41 | public override ITargetBlock InputBlock 42 | { 43 | get { return m_b1.InputBlock; } 44 | } 45 | } 46 | 47 | /// 48 | /// Merges 3 underlying dataflows so that the combined one looks like a single dataflow from outside 49 | /// 50 | /// Block1 In Type 51 | /// Block1 Out Type & Block2 In Type 52 | /// Block2 Out Type & Block3 In Type 53 | /// Block3 Out Type 54 | public class DataflowMerger : Dataflow 55 | { 56 | protected readonly Dataflow m_b1; 57 | protected readonly Dataflow m_b2; 58 | protected readonly Dataflow m_b3; 59 | 60 | public DataflowMerger(Dataflow b1, Dataflow b2, Dataflow b3): this(b1, b2, b3, b1.DataflowOptions) 61 | { } 62 | 63 | public DataflowMerger(Dataflow b1, Dataflow b2, Dataflow b3, DataflowOptions options) 64 | : base(options) 65 | { 66 | m_b1 = b1; 67 | m_b2 = b2; 68 | m_b3 = b3; 69 | 70 | m_b1.GoTo(m_b2).GoTo(m_b3); 71 | 72 | RegisterChild(m_b1); 73 | RegisterChild(m_b2); 74 | RegisterChild(m_b3); 75 | } 76 | 77 | public override ISourceBlock OutputBlock 78 | { 79 | get { return m_b3.OutputBlock; } 80 | } 81 | 82 | public override ITargetBlock InputBlock 83 | { 84 | get { return m_b1.InputBlock; } 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/DataflowOptions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks.Dataflow; 3 | 4 | namespace Gridsum.DataflowEx 5 | { 6 | /// 7 | /// Provides hints and configurations to dataflows. 8 | /// 9 | /// 10 | /// This class provides hints & suggestions. The corrent adoption of the configurations depends 11 | /// on the dataflow implementations. 12 | /// 13 | public class DataflowOptions 14 | { 15 | /// 16 | /// A hint to the dataflow implementation about the in-memory buffer size of underlying blocks 17 | /// 18 | public int? RecommendedCapacity { get; set; } 19 | 20 | /// 21 | /// Whether the monitor logging is enabled for parent dataflow 22 | /// 23 | public bool FlowMonitorEnabled { get; set; } 24 | 25 | /// 26 | /// Whether the monitor logging is enabled for childrens of the dataflow 27 | /// 28 | public bool BlockMonitorEnabled { get; set; } 29 | 30 | /// 31 | /// The monitor logging mode 32 | /// 33 | public PerformanceLogMode PerformanceMonitorMode { get; set; } 34 | 35 | /// 36 | /// A hint to dataflow implementation on parallelism of underlying block if feasible 37 | /// 38 | public int? RecommendedParallelismIfMultiThreaded { get; set; } 39 | 40 | private TimeSpan m_monitorInterval; 41 | 42 | /// 43 | /// The interval of the async monitor loop 44 | /// 45 | public TimeSpan MonitorInterval 46 | { 47 | get 48 | { 49 | return m_monitorInterval == TimeSpan.Zero ? DefaultInterval : m_monitorInterval; 50 | } 51 | set 52 | { 53 | this.m_monitorInterval = value; 54 | } 55 | } 56 | 57 | private static DataflowOptions s_defaultOptions = new DataflowOptions() 58 | { 59 | BlockMonitorEnabled = false, 60 | FlowMonitorEnabled = true, 61 | PerformanceMonitorMode = PerformanceLogMode.Succinct, 62 | MonitorInterval = DefaultInterval, 63 | RecommendedCapacity = 100000, 64 | RecommendedParallelismIfMultiThreaded = Environment.ProcessorCount 65 | }; 66 | 67 | private static DataflowOptions s_verboseOptions = new DataflowOptions() 68 | { 69 | BlockMonitorEnabled = true, 70 | FlowMonitorEnabled = true, 71 | PerformanceMonitorMode = PerformanceLogMode.Verbose, 72 | MonitorInterval = DefaultInterval, 73 | RecommendedCapacity = 100000 74 | }; 75 | 76 | /// 77 | /// A predefined default setting for DataflowOptions 78 | /// 79 | public static DataflowOptions Default 80 | { 81 | get 82 | { 83 | return s_defaultOptions; 84 | } 85 | } 86 | 87 | /// 88 | /// A predefined verbose setting for DataflowOptions 89 | /// 90 | public static DataflowOptions Verbose 91 | { 92 | get 93 | { 94 | return s_verboseOptions; 95 | } 96 | } 97 | 98 | /// 99 | /// The default monitor interval, 10 seconds 100 | /// 101 | public static TimeSpan DefaultInterval 102 | { 103 | get 104 | { 105 | return TimeSpan.FromSeconds(10); 106 | } 107 | } 108 | 109 | /// 110 | /// Extract relative information from dataflow option to a block-level ExecutionDataflowBlockOptions 111 | /// 112 | /// Whether the block using return value is multi-threaded 113 | public ExecutionDataflowBlockOptions ToExecutionBlockOption(bool isBlockMultiThreaded = false) 114 | { 115 | var option = new ExecutionDataflowBlockOptions(); 116 | 117 | if (this.RecommendedCapacity != null) 118 | { 119 | option.BoundedCapacity = this.RecommendedCapacity.Value; 120 | } 121 | 122 | if (isBlockMultiThreaded) 123 | { 124 | //todo: modify the default 125 | option.MaxDegreeOfParallelism = this.RecommendedParallelismIfMultiThreaded ?? Environment.ProcessorCount; 126 | } 127 | 128 | return option; 129 | } 130 | 131 | /// 132 | /// Extract relative information from dataflow option to a block-level GroupingDataflowBlockOptions 133 | /// 134 | public GroupingDataflowBlockOptions ToGroupingBlockOption() 135 | { 136 | var option = new GroupingDataflowBlockOptions(); 137 | 138 | if (this.RecommendedCapacity != null) 139 | { 140 | option.BoundedCapacity = this.RecommendedCapacity.Value; 141 | } 142 | 143 | return option; 144 | } 145 | 146 | /// 147 | /// Mode of performance logging 148 | /// 149 | public enum PerformanceLogMode 150 | { 151 | /// 152 | /// Only dump performance statistics for dataflow/block when it has non-zero buffer count 153 | /// 154 | Succinct = 0, 155 | 156 | /// 157 | /// Always dump performance statistics for dataflow/block 158 | /// 159 | Verbose = 1 160 | } 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/ETL/ByteArrayEqualityComparer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.ETL 8 | { 9 | public sealed class ByteArrayEqualityComparer : IEqualityComparer 10 | { 11 | public bool Equals(byte[] first, byte[] second) 12 | { 13 | if (first == second) 14 | return true; 15 | if (first == null || second == null || first.Length != second.Length) 16 | return false; 17 | for (int index = 0; index < first.Length; ++index) 18 | { 19 | if ((int)first[index] != (int)second[index]) 20 | return false; 21 | } 22 | return true; 23 | } 24 | 25 | public int GetHashCode(byte[] array) 26 | { 27 | if (array == null) 28 | return 0; 29 | int num1 = 17; 30 | foreach (byte num2 in array) 31 | num1 = num1 * 31 + (int)num2; 32 | return num1; 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/ETL/IDimRow.cs: -------------------------------------------------------------------------------- 1 | namespace Gridsum.DataflowEx.ETL 2 | { 3 | using System; 4 | 5 | using C5; 6 | 7 | public interface IDimRow 8 | { 9 | long AutoIncrementKey { get; } 10 | TJoinKey JoinOn { get; } 11 | bool IsFullRow { get; } 12 | IPriorityQueueHandle> Handle { get; set; } 13 | DateTime LastHitTime { get; set; } 14 | } 15 | } -------------------------------------------------------------------------------- /Gridsum.DataflowEx/ETL/PartialDimRow.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Text; 4 | using System.Threading.Tasks; 5 | 6 | namespace Gridsum.DataflowEx.ETL 7 | { 8 | using C5; 9 | 10 | /// 11 | /// Represents a partial dim row from merge output result 12 | /// 13 | public class PartialDimRow : IDimRow 14 | { 15 | public PartialDimRow() 16 | { 17 | this.LastHitTime = DateTime.UtcNow; 18 | } 19 | 20 | public long AutoIncrementKey { get; set; } 21 | public TLookupKey JoinOn { get; set; } 22 | 23 | public bool IsFullRow 24 | { 25 | get 26 | { 27 | return false; 28 | } 29 | } 30 | 31 | public IPriorityQueueHandle> Handle { get; set; } 32 | 33 | public DateTime LastHitTime { get; set; } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/ETL/RowCache.cs: -------------------------------------------------------------------------------- 1 | namespace Gridsum.DataflowEx.ETL 2 | { 3 | using System; 4 | using System.Collections.Generic; 5 | 6 | using C5; 7 | 8 | public class RowCache 9 | { 10 | class RowComparer : Comparer> 11 | { 12 | public override int Compare(IDimRow x, IDimRow y) 13 | { 14 | TimeSpan span = x.LastHitTime - y.LastHitTime; 15 | 16 | if (span.Ticks < 0) return -1; 17 | else if (span.Ticks > 0) return 1; 18 | else 19 | { 20 | return 0; 21 | } 22 | } 23 | } 24 | 25 | private readonly int m_sizeLimit; 26 | 27 | private IntervalHeap> m_rowPriorityQueue; 28 | private Dictionary> m_dict; 29 | 30 | public RowCache(int sizeLimit, IEqualityComparer keyEqualityComparer) 31 | { 32 | this.m_sizeLimit = sizeLimit; 33 | this.m_rowPriorityQueue = new IntervalHeap>(new RowComparer()); 34 | this.m_dict = new Dictionary>(keyEqualityComparer); 35 | } 36 | 37 | public bool TryAdd(TJoinKey key, IDimRow row) 38 | { 39 | if (this.m_dict.ContainsKey(key)) 40 | { 41 | return false; 42 | } 43 | else 44 | { 45 | this.m_dict.Add(key, row); 46 | IPriorityQueueHandle> handle = null; 47 | this.m_rowPriorityQueue.Add(ref handle, row); 48 | row.Handle = handle; 49 | 50 | while (this.m_dict.Count > this.m_sizeLimit) 51 | { 52 | var discardRow = this.m_rowPriorityQueue.DeleteMin(); 53 | this.m_dict.Remove(discardRow.JoinOn); 54 | } 55 | return true; 56 | } 57 | } 58 | 59 | public bool TryGetValue(TJoinKey key, out IDimRow value) 60 | { 61 | IDimRow hit; 62 | if (this.m_dict.TryGetValue(key, out hit)) 63 | { 64 | hit.LastHitTime = DateTime.UtcNow; 65 | this.m_rowPriorityQueue.Replace(hit.Handle, hit); 66 | value = hit; 67 | return true; 68 | } 69 | else 70 | { 71 | value = null; 72 | return false; 73 | } 74 | } 75 | 76 | public int Count 77 | { 78 | get 79 | { 80 | return this.m_dict.Count; 81 | } 82 | } 83 | 84 | public int SizeLimit 85 | { 86 | get 87 | { 88 | return m_sizeLimit; 89 | } 90 | } 91 | } 92 | } -------------------------------------------------------------------------------- /Gridsum.DataflowEx/Exceptions/NoChildRegisteredException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Gridsum.DataflowEx.Exceptions 4 | { 5 | public class NoChildRegisteredException : Exception 6 | { 7 | public NoChildRegisteredException(Dataflow dataflow) : base("No child has been registered in " + dataflow.FullName) 8 | { 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /Gridsum.DataflowEx/Exceptions/PostToBlockFailedException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Gridsum.DataflowEx.Exceptions 4 | { 5 | public class PostToBlockFailedException : Exception 6 | { 7 | public PostToBlockFailedException(string message) : base(message) 8 | { 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/Exceptions/PropagatedException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace Gridsum.DataflowEx.Exceptions 5 | { 6 | public abstract class PropagatedException : Exception 7 | { 8 | protected PropagatedException(string message) : base(message) 9 | { 10 | } 11 | } 12 | 13 | public class LinkedDataflowFailedException : PropagatedException 14 | { 15 | public LinkedDataflowFailedException() 16 | : base("Some other dataflow went wrong so I am down") 17 | { 18 | } 19 | } 20 | 21 | public class SiblingUnitFailedException : PropagatedException 22 | { 23 | public SiblingUnitFailedException() 24 | : base("Some sibling went wrong so I am down") 25 | { 26 | } 27 | } 28 | 29 | public class LinkedDataflowCanceledException : PropagatedException 30 | { 31 | public LinkedDataflowCanceledException() 32 | : base("Some other dataflow was canceled so I am down") 33 | { 34 | } 35 | } 36 | 37 | public class SiblingUnitCanceledException : PropagatedException 38 | { 39 | public SiblingUnitCanceledException() 40 | : base("Some sibling was canceled so I am down") 41 | { 42 | } 43 | } 44 | 45 | public class ExceptionComparer : IComparer 46 | { 47 | public int Compare(Exception x, Exception y) 48 | { 49 | if (x.GetType() == y.GetType()) 50 | { 51 | return 0; 52 | } 53 | 54 | if (x is PropagatedException) 55 | { 56 | return -1; 57 | } 58 | 59 | if (y is PropagatedException) 60 | { 61 | return 1; 62 | } 63 | 64 | return 0; 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/Exceptions/TaskEx.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.Exceptions 8 | { 9 | using System.Collections.Immutable; 10 | 11 | public static class TaskEx 12 | { 13 | public static ExceptionComparer s_ExceptionComparer = new ExceptionComparer(); 14 | 15 | /// 16 | /// This impl is better than Task.WhenAll because it avoids meaningful exception being swallowed when awaiting on error 17 | /// 18 | public static Task AwaitableWhenAll(params Task[] tasks) 19 | { 20 | var whenall = Task.WhenAll(tasks); 21 | var tcs = new TaskCompletionSource(); 22 | 23 | whenall.ContinueWith(t => 24 | { 25 | try 26 | { 27 | if (t.IsFaulted) 28 | { 29 | var highPriorityException = UnwrapWithPriority(t.Exception); 30 | tcs.SetException(highPriorityException); 31 | } 32 | else if (t.IsCanceled) 33 | { 34 | tcs.SetCanceled(); 35 | } 36 | else 37 | { 38 | tcs.SetResult(string.Empty); 39 | } 40 | } 41 | catch (Exception e) 42 | { 43 | tcs.SetException(e); 44 | } 45 | }); 46 | 47 | return tcs.Task; 48 | } 49 | 50 | public static async Task AwaitableWhenAll(Func> listGetter, Func itemCompletion) 51 | { 52 | IReadOnlyCollection childrenSnapShot; 53 | Exception cachedException = null; 54 | do 55 | { 56 | childrenSnapShot = listGetter(); 57 | try 58 | { 59 | await AwaitableWhenAll(childrenSnapShot.Select(itemCompletion).ToArray()).ConfigureAwait(false); 60 | } 61 | catch (Exception e) 62 | { 63 | cachedException = e; 64 | } 65 | } 66 | while (!object.ReferenceEquals(listGetter(), childrenSnapShot)); //this loop deals with dynamic children registration 67 | 68 | if (cachedException != null) 69 | { 70 | //should not throw cacheException directly here, otherwise the original stack trace is lost. 71 | throw new AggregateException(cachedException); 72 | } 73 | } 74 | 75 | public static Exception UnwrapWithPriority(AggregateException ae) 76 | { 77 | if (ae == null) 78 | { 79 | throw new ArgumentNullException("ae"); 80 | }; 81 | 82 | return ae.Flatten().InnerExceptions.OrderByDescending(e => e, s_ExceptionComparer).FirstOrDefault() ?? ae; 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/Gridsum.DataflowEx.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | 2.0.0 6 | 2.0.0 7 | 2.0.0 8 | Gridsum.DataflowEx is a dataflow and etl framework redesigned on top of Microsoft TPL Dataflow library with Object-Oriented Programming in mind. 9 | 10 | DataflowEx enables you to write reusable components and easily link them together as a high-level dataflow graph while still having the power of low-level blocks from TPL Dataflow. Other cool features include: Dataflow lifecycle management, built-in dataflow health monitor, cyclic graph auto completion support, sql bulk insertion support, etc. 11 | 12 | More information is available at: https://github.com/gridsum/dataflowex 13 | true 14 | * 2.0.0 - First formal V2 release that targets .Net Standard 2.0 15 | * 2.0.0-beta - Upgrade to .Net Core 2 RTM dependencies 16 | * 2.0.0-alpha2 - Update pacakge metadata and dependent libraries 17 | * 2.0.0-alpha - Migrate to .Net Standard 2.0 18 | * 1.1.3 - Optimize some logging and exception handling. 19 | * 1.1.2 - DbBulkInserter now uses a single transaction to ensure full rollback on error. 20 | * 1.1.1 - Fix potential race condition for RegisterChild() and RegisterDependency(). 21 | * 1.1.0 - Tutorial and documentation added. Also supports manual db column mapping registration. 22 | * 1.0.9.* - Built-in flows now respect given DataflowOptions wherever applicable. 23 | * 1.0.9 - Allows optional db column mapping. 24 | * 1.0.8 - Add DbDataJoiner to help lookup and populate a dimension table. Also comes with a new ring completion mechanism. 25 | * 1.0.7 - A better DataBrancher that supports multiple outputs. Add varbinary support for bulk insertion. 26 | * 1.0.6 - Optimize property accessor in bulk inserter. Add NoNullCheck attribute. 27 | * 1.0.5 - Add LogReader abstraction. Add basic circular dependency detection logic. 28 | * 1.0.4 - Add LinkLeftToError(). Enhance StatisticsRecorder indexer. Fix DataflowEvent ToString(). 29 | * 1.0.3 - RegisterChild() becomes public and defensive. 30 | * 1.0.2 - Add Dataflow.RegisterPostDataflowTask and Dataflow.RegisterCancellationTokenSource. StatisticsRecorder now supports event aggregation. 31 | * 1.0.1 - More extensible DbBulkInserter/MultiDbBulkInserter and exact matching for multi-match. 32 | * 1.0.0 - First release after 9 beta versions. 33 | dataflow etl dataflowex block gridsum bulk insert 34 | Copyright 2017, Gridsum 35 | Gridsum 36 | karldodd 37 | http://www.gridsum.com/templets/gs/images/slogo.png 38 | https://github.com/gridsum/DataflowEx 39 | https://github.com/gridsum/DataflowEx 40 | 41 | 42 | bin\Debug\netstandard2.0\Gridsum.DataflowEx.xml 43 | 44 | 45 | bin\Release\netstandard2.0\Gridsum.DataflowEx.xml 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/IDataflow.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Threading.Tasks; 4 | using System.Threading.Tasks.Dataflow; 5 | 6 | namespace Gridsum.DataflowEx 7 | { 8 | /// 9 | /// Represents a dataflow graph 10 | /// 11 | public interface IDataflow 12 | { 13 | IEnumerable Blocks { get; } 14 | Task CompletionTask { get; } 15 | void Fault(Exception exception); 16 | string Name { get; } 17 | string FullName { get; } 18 | int BufferedCount { get; } 19 | 20 | /// 21 | /// Signals to the IDataflow that it should not accept any more messages 22 | /// 23 | void Complete(); 24 | 25 | void RegisterDependency(IDataflow dependencyDataflow); 26 | } 27 | 28 | /// 29 | /// Represents a dataflow graph that has a typed input node 30 | /// 31 | public interface IDataflow : IDataflow 32 | { 33 | /// 34 | /// The input node of the dataflow graph 35 | /// 36 | ITargetBlock InputBlock { get; } 37 | } 38 | 39 | /// 40 | /// Represents a dataflow graph that has a typed output node 41 | /// 42 | public interface IOutputDataflow : IDataflow 43 | { 44 | /// 45 | /// /// The output node of the dataflow graph 46 | /// 47 | ISourceBlock OutputBlock { get; } 48 | 49 | /// 50 | /// Links output of this dataflow to the input of another dataflow graph 51 | /// 52 | void LinkTo(IDataflow other, Predicate predicate); 53 | } 54 | 55 | /// 56 | /// Represents a dataflow graph that has a typed input node and a typed output node 57 | /// 58 | public interface IDataflow : IDataflow, IOutputDataflow 59 | { 60 | } 61 | 62 | /// 63 | /// Represents a dataflow graph that has an internal batch buffer 64 | /// 65 | public interface IBatchedDataflow : IDataflow 66 | { 67 | void TriggerBatch(); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/ImmutableUtils.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx 8 | { 9 | using System.Collections.Immutable; 10 | using System.Threading; 11 | 12 | public static class ImmutableUtils 13 | { 14 | public static ImmutableList AddOptimistically(ref ImmutableList list, T item) 15 | { 16 | ImmutableList old, added; 17 | ImmutableList beforeExchange; 18 | do 19 | { 20 | old = Volatile.Read(ref list); 21 | added = old.Add(item); 22 | beforeExchange = Interlocked.CompareExchange(ref list, added, old); 23 | 24 | //todo: reuse beforeExchange for next old value 25 | } 26 | while (beforeExchange != old); 27 | 28 | return added; 29 | } 30 | 31 | public static bool TryAddOptimistically(ref ImmutableHashSet set, T item) 32 | { 33 | ImmutableHashSet old, added; 34 | ImmutableHashSet beforeExchange; 35 | do 36 | { 37 | old = Volatile.Read(ref set); 38 | added = old.Add(item); 39 | 40 | if (object.ReferenceEquals(added, old)) 41 | { 42 | return false; 43 | } 44 | 45 | beforeExchange = Interlocked.CompareExchange(ref set, added, old); 46 | 47 | //todo: reuse beforeExchange for next old value 48 | } 49 | while (beforeExchange != old); 50 | 51 | return true; 52 | } 53 | 54 | public static bool TryAddManyOptimistically(ref ImmutableHashSet set, T[] items) 55 | { 56 | ImmutableHashSet old, added; 57 | ImmutableHashSet beforeExchange; 58 | do 59 | { 60 | old = Volatile.Read(ref set); 61 | 62 | var builder = old.ToBuilder(); 63 | foreach (var item in items) 64 | { 65 | builder.Add(item); 66 | } 67 | 68 | added = builder.ToImmutable(); 69 | 70 | if (object.ReferenceEquals(added, old)) 71 | { 72 | return false; //added nothing (all items exists) 73 | } 74 | 75 | beforeExchange = Interlocked.CompareExchange(ref set, added, old); 76 | 77 | //todo: reuse beforeExchange for next old value 78 | } 79 | while (beforeExchange != old); 80 | 81 | return true; 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/IntHolder.cs: -------------------------------------------------------------------------------- 1 | using System.Threading; 2 | 3 | namespace Gridsum.DataflowEx 4 | { 5 | public class IntHolder 6 | { 7 | private int m_count; 8 | public int Count 9 | { 10 | get { return m_count; } 11 | } 12 | 13 | public int Increment() 14 | { 15 | return Interlocked.Increment(ref m_count); 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /Gridsum.DataflowEx/LogHelper.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Reflection; 3 | using Common.Logging; 4 | 5 | namespace Gridsum.DataflowEx 6 | { 7 | internal static class LogHelper 8 | { 9 | private static readonly ILog s_logger = LogManager.GetLogger(Assembly.GetExecutingAssembly().GetName().Name); 10 | private static readonly ILog s_perfMon = LogManager.GetLogger(Assembly.GetExecutingAssembly().GetName().Name + ".PerfMon"); 11 | 12 | internal static ILog Logger 13 | { 14 | get 15 | { 16 | return s_logger; 17 | } 18 | } 19 | 20 | internal static ILog PerfMon 21 | { 22 | get 23 | { 24 | return s_perfMon; 25 | } 26 | } 27 | 28 | public static void LogByLevel(this ILog logger, LogLevel logLevel, Action formatMessageCallback) 29 | { 30 | switch (logLevel) 31 | { 32 | case LogLevel.All: 33 | case LogLevel.Trace: 34 | if (logger.IsTraceEnabled) 35 | { 36 | logger.Trace(formatMessageCallback); 37 | } 38 | break; 39 | case LogLevel.Debug: 40 | if (logger.IsDebugEnabled) 41 | { 42 | logger.Debug(formatMessageCallback); 43 | } 44 | break; 45 | case LogLevel.Info: 46 | if (logger.IsInfoEnabled) 47 | { 48 | logger.Info(formatMessageCallback); 49 | } 50 | break; 51 | case LogLevel.Warn: 52 | if (logger.IsWarnEnabled) 53 | { 54 | logger.Warn(formatMessageCallback); 55 | } 56 | break; 57 | case LogLevel.Error: 58 | if (logger.IsErrorEnabled) 59 | { 60 | logger.Error(formatMessageCallback); 61 | } 62 | break; 63 | case LogLevel.Fatal: 64 | if (logger.IsFatalEnabled) 65 | { 66 | logger.Fatal(formatMessageCallback); 67 | } 68 | break; 69 | case LogLevel.Off: 70 | default: 71 | //do nothing 72 | break; 73 | } 74 | } 75 | } 76 | } -------------------------------------------------------------------------------- /Gridsum.DataflowEx/LogReader.cs: -------------------------------------------------------------------------------- 1 | namespace Gridsum.DataflowEx 2 | { 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Collections.Specialized; 6 | using System.IO; 7 | using System.Linq; 8 | using System.Threading; 9 | using System.Threading.Tasks; 10 | 11 | /// 12 | /// Abstraction of a log reader dataflow that accepts strings as input lines. You should inherit from this class 13 | /// and construct your database graph inside the constructor. 14 | /// 15 | /// 16 | /// The class comes with a powerful ProcessAsync() implemention which will pull text from readers and process the lines. 17 | /// If something goes wrong within the dataflow, ProcessAsync() will ensure the reading part is canceled as soon as possible. 18 | /// 19 | public abstract class LogReader : Dataflow 20 | { 21 | /// 22 | /// Constructs an instance of a log reader 23 | /// 24 | public LogReader(DataflowOptions dataflowOptions) 25 | : base(dataflowOptions) 26 | { 27 | } 28 | 29 | /// 30 | /// The recorder which records and aggregates the output of the log reader 31 | /// 32 | public abstract StatisticsRecorder Recorder { get; } 33 | 34 | /// 35 | /// Asynchronously pull from text reader into the log reader 36 | /// 37 | public Task PullFromAsync(TextReader reader, CancellationToken ct) 38 | { 39 | return this.PullFromAsync(reader.ToEnumerable(), ct); 40 | } 41 | 42 | /// 43 | /// Asynchronously read from the text stream and process lines in the underlying dataflow. 44 | /// 45 | /// The text reader to read from 46 | /// 47 | /// Whether a complete signal should be sent to the log reader dataflow. 48 | /// If yes, it also ensures that the whole processing dataflow is completed before the ProcessAsync() task ends. 49 | /// Default to yes. Set the param to false if the log reader will read other text streams after this operation. 50 | /// 51 | /// A task representing the state of the async operation which returns the total count of items processed in this method 52 | public virtual async Task ProcessAsync(TextReader reader, bool completeLogReaderOnFinish = true) 53 | { 54 | return await ProcessAsync(reader.ToEnumerable(), completeLogReaderOnFinish).ConfigureAwait(false); 55 | } 56 | 57 | /// 58 | /// Asynchronously read from the text streams sequentially and process lines in the underlying dataflow. 59 | /// 60 | /// The text readers to read from 61 | /// 62 | /// Whether a complete signal should be sent to the log reader dataflow. 63 | /// If yes, it also ensures that the whole processing dataflow is completed before the ProcessMultipleAsync() task ends. 64 | /// Default to yes. Set the param to false if the log reader will read other text streams after this operation. 65 | /// 66 | /// A task representing the state of the async operation which returns the total count of items processed in this method 67 | public virtual Task ProcessMultipleAsync(IEnumerable readers, bool completeLogReaderOnFinish = true) 68 | { 69 | return ProcessMultipleAsync(readers.Select(DataflowUtils.ToEnumerable), completeLogReaderOnFinish); 70 | } 71 | 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/PatternMatch/IMatchCondition.cs: -------------------------------------------------------------------------------- 1 | namespace Gridsum.DataflowEx.PatternMatch 2 | { 3 | /// 4 | /// Represents a match condition 5 | /// 6 | /// Type of the input the condition can accept 7 | public interface IMatchCondition 8 | { 9 | bool Matches(T input); 10 | 11 | /// 12 | /// Check if the condition matches the given input 13 | /// 14 | /// The exact matched condition for the input (e.g. a child condition for MultiMatchCondition) 15 | /// For a normal condition, returns itself if input is matched 16 | IMatchCondition MatchesExact(T input); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/PatternMatch/IMatchable.cs: -------------------------------------------------------------------------------- 1 | namespace Gridsum.DataflowEx.PatternMatch 2 | { 3 | public interface IMatchable 4 | { 5 | IMatchCondition Condition { get; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/PatternMatch/MatchConditionBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.PatternMatch 8 | { 9 | /// 10 | /// IMatchCondition with a default implementation of MatchesExact 11 | /// 12 | /// 13 | public abstract class MatchConditionBase : IMatchCondition 14 | { 15 | public abstract bool Matches(T input); 16 | 17 | public IMatchCondition MatchesExact(T input) 18 | { 19 | if (this.Matches(input)) return this; 20 | else return null; 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/PatternMatch/MatchNoneCondition.cs: -------------------------------------------------------------------------------- 1 | namespace Gridsum.DataflowEx.PatternMatch 2 | { 3 | public class MatchNoneCondition : IMatchCondition 4 | { 5 | static readonly MatchNoneCondition s_matchNone = new MatchNoneCondition(); 6 | 7 | public bool Matches(T input) 8 | { 9 | return false; 10 | } 11 | 12 | public IMatchCondition MatchesExact(T input) 13 | { 14 | if (this.Matches(input)) return this; 15 | else return null; 16 | } 17 | 18 | public static MatchNoneCondition Instance 19 | { 20 | get 21 | { 22 | return s_matchNone; 23 | } 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/PatternMatch/MatchType.cs: -------------------------------------------------------------------------------- 1 | namespace Gridsum.DataflowEx.PatternMatch 2 | { 3 | /// 4 | /// MatchType enumeration for match strategies 5 | /// 6 | public enum MatchType : byte 7 | { 8 | /// 9 | /// 精确匹配。 10 | /// 11 | ExactMatch = 0, 12 | 13 | /// 14 | /// 以指定字符串开始。 15 | /// 16 | BeginsWith = 1, 17 | 18 | /// 19 | /// 以指定字符串结束。 20 | /// 21 | EndsWith = 2, 22 | 23 | /// 24 | /// 包含特定字符串。 25 | /// 26 | Contains = 3, 27 | 28 | /// 29 | /// 正则匹配 30 | /// 31 | RegexMatch = 4, 32 | } 33 | } -------------------------------------------------------------------------------- /Gridsum.DataflowEx/PatternMatch/MultiMatcher.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | 5 | namespace Gridsum.DataflowEx.PatternMatch 6 | { 7 | public class MultiMatchCondition : IMatchCondition 8 | { 9 | protected readonly IMatchCondition[] m_matchConditions; 10 | 11 | public MultiMatchCondition(IMatchCondition[] matchConditions) 12 | { 13 | m_matchConditions = matchConditions; 14 | } 15 | 16 | public bool Matches(TInput input) 17 | { 18 | return m_matchConditions.Any(matchCondition => matchCondition.Matches(input)); 19 | } 20 | 21 | public IMatchCondition MatchesExact(TInput input) 22 | { 23 | foreach (IMatchCondition child in m_matchConditions) 24 | { 25 | IMatchCondition childMatch = child.MatchesExact(input); 26 | 27 | if (childMatch != null) 28 | { 29 | return childMatch; 30 | } 31 | } 32 | 33 | return null; 34 | } 35 | 36 | } 37 | 38 | //todo: input-output caching 39 | //todo:? speed up exact string matching by using dictionary 40 | //todo:? speed up endswith/beginwith string matching by using tree 41 | public class MultiMatcher : MultiMatchCondition where TOutput : IMatchable 42 | { 43 | private readonly TOutput[] m_matchables; 44 | 45 | public MultiMatcher(TOutput[] matchables) : base(matchables.Select(o => o.Condition).ToArray()) 46 | { 47 | m_matchables = matchables; 48 | } 49 | 50 | public bool TryMatch(TInput input, out TOutput matchable) 51 | { 52 | for (int i = 0; i < m_matchables.Length; i++) 53 | { 54 | if (m_matchables[i].Condition.Matches(input)) 55 | { 56 | matchable = m_matchables[i]; 57 | return true; 58 | } 59 | } 60 | 61 | matchable = default (TOutput); 62 | return false; 63 | } 64 | 65 | 66 | public bool TryMatchExact(TInput input, out TOutput matchable, out IMatchCondition exactMatch) 67 | { 68 | for (int i = 0; i < m_matchables.Length; i++) 69 | { 70 | IMatchCondition cond = m_matchables[i].Condition.MatchesExact(input); 71 | 72 | if (cond != null) 73 | { 74 | matchable = m_matchables[i]; 75 | exactMatch = cond; 76 | return true; 77 | } 78 | } 79 | 80 | matchable = default(TOutput); 81 | exactMatch = null; 82 | return false; 83 | } 84 | 85 | public TOutput Match(TInput input, Func defaultValueFactory) 86 | { 87 | TOutput output; 88 | return TryMatch(input, out output) ? output : defaultValueFactory(); 89 | } 90 | 91 | public TOutput Match(TInput input, TOutput defaultValue) 92 | { 93 | TOutput output; 94 | return TryMatch(input, out output) ? output : defaultValue; 95 | } 96 | 97 | public IEnumerable MatchMultiple(TInput input) 98 | { 99 | for (int i = 0; i < m_matchables.Length; i++) 100 | { 101 | if (m_matchables[i].Condition.Matches(input)) 102 | { 103 | yield return m_matchables[i]; 104 | } 105 | } 106 | } 107 | 108 | public TOutput this[int index] 109 | { 110 | get 111 | { 112 | return m_matchables[index]; 113 | } 114 | } 115 | } 116 | 117 | public class FastMultiMatcher : MultiMatchCondition where TOutput : IMatchable 118 | { 119 | private readonly TOutput[] m_matchables; 120 | 121 | public FastMultiMatcher(TOutput[] matchables) : base(matchables.Select(o => o.Condition).ToArray()) 122 | { 123 | m_matchables = matchables; 124 | } 125 | 126 | 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/PatternMatch/StringMatchCondition.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Text.RegularExpressions; 3 | 4 | namespace Gridsum.DataflowEx.PatternMatch 5 | { 6 | /// 7 | /// A simple condition implementation for string. 8 | /// 9 | public class StringMatchCondition : MatchConditionBase 10 | { 11 | public StringMatchCondition(string matchPattern, MatchType matchType = MatchType.ExactMatch) 12 | { 13 | if (matchPattern == null) 14 | { 15 | throw new ArgumentNullException("matchPattern"); 16 | } 17 | 18 | this.MatchPattern = matchPattern; 19 | this.MatchType = matchType; 20 | 21 | if (matchType == MatchType.RegexMatch) 22 | { 23 | this.Regex = new Regex(matchPattern, RegexOptions.IgnoreCase | RegexOptions.Compiled); 24 | } 25 | } 26 | 27 | public string MatchPattern { get; private set; } 28 | public MatchType MatchType { get; private set; } 29 | 30 | public Regex Regex { get; set; } 31 | 32 | public override bool Matches(string input) 33 | { 34 | if (input == null) 35 | { 36 | return false; 37 | } 38 | 39 | switch (MatchType) 40 | { 41 | case MatchType.ExactMatch: 42 | // 精确匹配 43 | return MatchPattern == input; 44 | case MatchType.BeginsWith: 45 | // 处理左匹配 46 | return input.StartsWith(MatchPattern, StringComparison.Ordinal); 47 | case MatchType.EndsWith: 48 | // 处理右匹配 49 | return input.EndsWith(MatchPattern, StringComparison.Ordinal); 50 | case MatchType.Contains: 51 | // 处理包含情况 52 | return input.Contains(MatchPattern); 53 | case MatchType.RegexMatch: 54 | return Regex.IsMatch(input); 55 | default: 56 | if (LogHelper.Logger.IsWarnEnabled) 57 | { 58 | LogHelper.Logger.WarnFormat("Invalid given enum value MatchType {0}. Using 'Contains' instead.", MatchType); 59 | } 60 | return input.Contains(MatchPattern); 61 | } 62 | } 63 | } 64 | } -------------------------------------------------------------------------------- /Gridsum.DataflowEx/PatternMatch/UrlStringMatchCondition.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Gridsum.DataflowEx.PatternMatch 8 | { 9 | public class UrlStringMatchCondition : StringMatchCondition 10 | { 11 | private readonly bool m_ignoreCase; 12 | 13 | public UrlStringMatchCondition(string matchPattern, MatchType matchType, bool excludeParam, bool ignoreCase = true) : base(matchPattern, matchType) 14 | { 15 | this.m_ignoreCase = ignoreCase; 16 | ExcludeParam = excludeParam; 17 | } 18 | 19 | public override bool Matches(string input) 20 | { 21 | if (input == null) return false; 22 | 23 | if (this.ExcludeParam) 24 | { 25 | input = GetUrlWithoutParam(input); 26 | } 27 | 28 | if (m_ignoreCase) 29 | { 30 | switch (MatchType) 31 | { 32 | case MatchType.ExactMatch: 33 | return string.Equals(MatchPattern, input, StringComparison.OrdinalIgnoreCase); 34 | case MatchType.BeginsWith: 35 | return input.StartsWith(MatchPattern, StringComparison.OrdinalIgnoreCase); 36 | case MatchType.EndsWith: 37 | return input.EndsWith(MatchPattern, StringComparison.OrdinalIgnoreCase); 38 | case MatchType.Contains: 39 | return input.IndexOf(MatchPattern, StringComparison.OrdinalIgnoreCase) >= 0; 40 | case MatchType.RegexMatch: 41 | return Regex.IsMatch(input); 42 | default: 43 | if (LogHelper.Logger.IsWarnEnabled) 44 | { 45 | LogHelper.Logger.WarnFormat("Invalid given enum value MatchType {0}. Using 'Contains' instead.", MatchType); 46 | } 47 | return input.Contains(MatchPattern); 48 | } 49 | } 50 | else 51 | { 52 | return base.Matches(input); 53 | } 54 | } 55 | 56 | public bool ExcludeParam { get; private set; } 57 | 58 | private static readonly char[] s_urlParamChars = new[] { '?', '#' }; 59 | 60 | public static string GetUrlWithoutParam(string url) 61 | { 62 | url = url.Trim(); 63 | 64 | int index = url.IndexOfAny(s_urlParamChars); 65 | 66 | if (index >= 0) 67 | { 68 | url = url.Substring(0, index); 69 | } 70 | 71 | return url; 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/PropagatorDataflow.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using System.Threading.Tasks.Dataflow; 7 | 8 | namespace Gridsum.DataflowEx 9 | { 10 | using Gridsum.DataflowEx.AutoCompletion; 11 | 12 | public class PropagatorDataflow : Dataflow 13 | { 14 | private readonly IPropagatorBlock m_block; 15 | 16 | public PropagatorDataflow(IPropagatorBlock block) : this(block, DataflowOptions.Default) 17 | {} 18 | 19 | public PropagatorDataflow(IPropagatorBlock block, DataflowOptions dataflowOptions) 20 | : base(dataflowOptions) 21 | { 22 | m_block = block; 23 | RegisterChild(m_block, null); 24 | } 25 | 26 | public override ITargetBlock InputBlock 27 | { 28 | get { return m_block; } 29 | } 30 | 31 | public override ISourceBlock OutputBlock 32 | { 33 | get { return m_block; } 34 | } 35 | } 36 | 37 | public class TransformManyDataflow : Dataflow, IRingNode 38 | { 39 | private readonly Func>> m_transformMany; 40 | private TransformManyBlock m_block; 41 | 42 | public TransformManyDataflow(Func>> transformMany, DataflowOptions options) 43 | : base(options) 44 | { 45 | this.m_transformMany = transformMany; 46 | m_block = new TransformManyBlock(new Func>>(Transform), options.ToExecutionBlockOption()); 47 | RegisterChild(m_block); 48 | } 49 | 50 | private async Task> Transform(TIn @in) 51 | { 52 | IsBusy = true; 53 | var outs = await m_transformMany(@in).ConfigureAwait(false); 54 | IsBusy = false; 55 | return outs; 56 | } 57 | 58 | public override ITargetBlock InputBlock 59 | { 60 | get 61 | { 62 | return m_block; 63 | } 64 | } 65 | 66 | public override ISourceBlock OutputBlock 67 | { 68 | get 69 | { 70 | return m_block; 71 | } 72 | } 73 | 74 | public bool IsBusy { get; private set; } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/TargetDataflow.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using System.Threading.Tasks.Dataflow; 7 | 8 | namespace Gridsum.DataflowEx 9 | { 10 | public class TargetDataflow : Dataflow 11 | { 12 | private readonly ITargetBlock m_block; 13 | 14 | public TargetDataflow(ITargetBlock block) : this(block, DataflowOptions.Default) 15 | { 16 | } 17 | 18 | public TargetDataflow(ITargetBlock block, DataflowOptions dataflowOptions) 19 | : base(dataflowOptions) 20 | { 21 | m_block = block; 22 | RegisterChild(m_block); 23 | } 24 | 25 | public override ITargetBlock InputBlock 26 | { 27 | get { return m_block; } 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /Gridsum.DataflowEx/Utils.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | 4 | namespace Gridsum.DataflowEx 5 | { 6 | using System.Collections.Immutable; 7 | using System.Data; 8 | using System.Diagnostics; 9 | using System.Runtime.CompilerServices; 10 | using System.Text; 11 | using System.Threading; 12 | using System.Threading.Tasks; 13 | 14 | using Common.Logging; 15 | 16 | public static class Utils 17 | { 18 | public static string GetFriendlyName(this Type type) 19 | { 20 | if (type.IsGenericType) 21 | return string.Format("{0}<{1}>", type.Name.Split('`')[0], string.Join(", ", type.GetGenericArguments().Select(GetFriendlyName))); 22 | else 23 | return type.Name; 24 | } 25 | 26 | public static bool IsNullableType(this Type type) 27 | { 28 | Type tmp; 29 | return IsNullableType(type, out tmp); 30 | } 31 | 32 | public static bool IsNullableType(this Type type, out Type innerValueType) 33 | { 34 | innerValueType = null; 35 | if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Nullable<>)) 36 | { 37 | innerValueType = Nullable.GetUnderlyingType(type); 38 | return true; 39 | } 40 | return false; 41 | } 42 | 43 | public static string FlattenColumnNames(DataColumnCollection columns, string tableName) 44 | { 45 | var sb = new StringBuilder(); 46 | sb.Append('('); 47 | foreach (DataColumn column in columns) 48 | { 49 | if (column.AutoIncrement) continue; //skip auto increment column 50 | 51 | sb.Append(tableName); 52 | if (!string.IsNullOrEmpty(tableName)) 53 | { 54 | sb.Append('.'); 55 | } 56 | sb.Append('['); 57 | sb.Append(column.ColumnName); 58 | sb.Append(']'); 59 | sb.Append(','); 60 | } 61 | 62 | sb.Remove(sb.Length - 1, 1); 63 | sb.Append(')'); 64 | return sb.ToString(); 65 | } 66 | 67 | public static DataColumn GetAutoIncrementColumn(DataColumnCollection columns) 68 | { 69 | return columns.OfType().First(c => c.AutoIncrement); 70 | } 71 | 72 | [MethodImpl(MethodImplOptions.NoInlining)] 73 | public static ILog GetNamespaceLogger() 74 | { 75 | var frame = new StackFrame(1); 76 | var callingMethod = frame.GetMethod(); 77 | return LogManager.GetLogger(callingMethod.DeclaringType.Namespace); 78 | } 79 | } 80 | } -------------------------------------------------------------------------------- /License.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gridsum/DataflowEx/552b0b45eeb4a513c4079a10d0bc57df4950448f/License.txt -------------------------------------------------------------------------------- /global.json: -------------------------------------------------------------------------------- 1 | { 2 | "sdk": { 3 | "version": "2.0.0" 4 | } 5 | } -------------------------------------------------------------------------------- /images/MyLoggerDemo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gridsum/DataflowEx/552b0b45eeb4a513c4079a10d0bc57df4950448f/images/MyLoggerDemo.png -------------------------------------------------------------------------------- /images/dbbulkinserter_screenshot1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gridsum/DataflowEx/552b0b45eeb4a513c4079a10d0bc57df4950448f/images/dbbulkinserter_screenshot1.png -------------------------------------------------------------------------------- /images/dbbulkinserter_screenshot2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gridsum/DataflowEx/552b0b45eeb4a513c4079a10d0bc57df4950448f/images/dbbulkinserter_screenshot2.png -------------------------------------------------------------------------------- /images/lookupDemo1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gridsum/DataflowEx/552b0b45eeb4a513c4079a10d0bc57df4950448f/images/lookupDemo1.png -------------------------------------------------------------------------------- /images/lookupDemo2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gridsum/DataflowEx/552b0b45eeb4a513c4079a10d0bc57df4950448f/images/lookupDemo2.png --------------------------------------------------------------------------------