├── .gitattributes ├── .gitignore ├── LICENSE ├── README.md ├── SECURITY.md └── src ├── CRA.ClientLibrary ├── AzureProvider │ ├── AzureBlobProvider.cs │ ├── AzureDataProvider.cs │ ├── AzureEndpointInfoProvider.cs │ ├── AzureShardedVertexInfoProvider.cs │ ├── AzureVertexConnectionInfoProvider.cs │ ├── AzureVertexInfoProvider.cs │ ├── ConnectionTable.cs │ ├── EndpointTable.cs │ ├── ShardedVertexTable.cs │ └── VertexTable.cs ├── CRA.ClientLibrary.csproj ├── DataProcessing │ ├── Datasets │ │ ├── ClientSideShardedDataset.cs │ │ ├── DatasetBase.cs │ │ ├── DeployableShardedDataset.cs │ │ ├── IDataset.cs │ │ ├── IDeployable.cs │ │ ├── IShardedDataset.cs │ │ └── ShardedDatasetBase.cs │ ├── Definitions │ │ ├── Operator │ │ │ ├── OperatorInformation.cs │ │ │ ├── OperatorTransforms.cs │ │ │ ├── OperatorType.cs │ │ │ └── OperatorsToplogy.cs │ │ └── Tasks │ │ │ ├── ClientTerminalTask.cs │ │ │ ├── ProduceTask.cs │ │ │ ├── ShuffleTask.cs │ │ │ ├── SubscribeTask.cs │ │ │ └── TaskBase.cs │ ├── ShardedOperators │ │ ├── ShardedEndpoints │ │ │ ├── ShardedOperatorInputBase.cs │ │ │ ├── ShardedOperatorOutputBase.cs │ │ │ ├── ShardedProducerOutput.cs │ │ │ ├── ShardedProducerSecondaryInput.cs │ │ │ ├── ShardedProducerSecondaryOutput.cs │ │ │ ├── ShardedShuffleInput.cs │ │ │ ├── ShardedShuffleOutput.cs │ │ │ ├── ShardedShuffleSecondaryInput.cs │ │ │ ├── ShardedShuffleSecondaryOutput.cs │ │ │ ├── ShardedSubscribeClientInput.cs │ │ │ ├── ShardedSubscribeClientOutput.cs │ │ │ ├── ShardedSubscribeInput.cs │ │ │ └── ShardedSubscribeOutput.cs │ │ ├── ShardedOperatorBase.cs │ │ ├── ShardedProducerOperator.cs │ │ ├── ShardedShuffleOperator.cs │ │ ├── ShardedSubscribeClientOperator.cs │ │ └── ShardedSubscribeOperator.cs │ └── Utilities │ │ ├── DatasetUtils.cs │ │ ├── DeploymentUtils.cs │ │ ├── MoveUtils.cs │ │ ├── OperatorUtils.cs │ │ ├── ParallelismUtils.cs │ │ └── TransformUtils.cs ├── DataProvider │ ├── DataProviderExtensions.cs │ ├── EndpointInfo.cs │ ├── IBlobStorageProvider.cs │ ├── IDataProvider.cs │ ├── IEndpointInfoProvider.cs │ ├── IShardedVertexInfoProvider.cs │ ├── IVertexConnectionInfoProvider.cs │ ├── IVertexInfoProvider.cs │ ├── ShardedVertexInfo.cs │ ├── VertexConnectionInfo.cs │ └── VertexInfo.cs ├── Definitions │ ├── ConnectionInfo.cs │ ├── ConnectionInitiator.cs │ ├── CoralTaskMessageType.cs │ ├── ErrorCodes.cs │ ├── ObjectWrapper.cs │ ├── ShardingInfo.cs │ ├── StreamConnection.cs │ └── StreamConnectionPool.cs ├── Main │ ├── CRAClientLibrary.cs │ ├── CRAWorker.cs │ └── ShardedCRAClientLibrary.cs ├── Properties │ └── AssemblyInfo.cs ├── Security │ ├── DummySecureStreamConnectionDescriptor.cs │ ├── ISecureStreamConnectionDescriptor.cs │ └── SampleSecureStreamConnectionDescriptor.cs ├── Tables │ ├── ConnectionTableManager.cs │ ├── EndpointTableManager.cs │ ├── ShardedVertexTableManager.cs │ └── VertexTableManager.cs ├── Utilities │ ├── AssemblyResolver.cs │ ├── AssemblyResolverClient.cs │ ├── AssemblyUtils.cs │ ├── ClientStreamUtils.cs │ ├── ExpressionUtils.cs │ ├── NetworkUtils.cs │ ├── SerializationHelper.cs │ └── StreamCommunicator.cs ├── Vertices │ ├── DetachedVertex.cs │ ├── IAsyncVertexInputEndpoint.cs │ ├── IAsyncVertexOutputEndpoint.cs │ ├── IVertex.cs │ ├── IVertexInputEndpoint.cs │ ├── IVertexOutputEndpoint.cs │ ├── VertexBase.cs │ ├── VertexInputEndpointBase.cs │ └── VertexOutputEndpointBase.cs └── app.config ├── CRA.DataProvider.File ├── CRA.DataProvider.File.csproj ├── FileBlobProvider.cs ├── FileDataProvider.cs ├── FileEndpointProvider.cs ├── FileShardedVertexProvider.cs ├── FileUtils.cs ├── FileVertexConnectionProvider.cs └── FileVertexProvider.cs ├── CRA.Worker ├── .dockerignore ├── App.config ├── CRA.Worker.csproj ├── CRA.nuspec ├── Dockerfile-debug ├── Dockerfile-release ├── Program.cs ├── Properties │ ├── AssemblyInfo.cs │ └── launchSettings.json ├── cra-two-instances-debug.yml └── cra-two-instances-release.yml ├── CRA.fxcop ├── CRA.sln ├── CRA.snk ├── CustomDictionary.xml ├── Samples ├── BandwidthTest │ ├── App.config │ ├── BandwidthTest.csproj │ ├── BandwidthTestVertex.cs │ ├── MyAsyncInput.cs │ ├── MyAsyncOutput.cs │ ├── Program.cs │ └── Properties │ │ └── AssemblyInfo.cs ├── ConnectionPair │ ├── App.config │ ├── ConnectionPair.csproj │ ├── ConnectionPairVertex.cs │ ├── MyAsyncInput.cs │ ├── MyAsyncOutput.cs │ ├── Program.cs │ └── Properties │ │ └── AssemblyInfo.cs ├── FusableConnectionPair │ ├── App.config │ ├── FusableConnectionPair.csproj │ ├── FusableConnectionPairVertex.cs │ ├── MyAsyncFusableInput.cs │ ├── MyAsyncFusableOutput.cs │ ├── Program.cs │ └── Properties │ │ └── AssemblyInfo.cs ├── ShardedConnectionPair │ ├── App.config │ ├── MyAsyncInput.cs │ ├── MyAsyncOutput.cs │ ├── Program.cs │ ├── Properties │ │ └── AssemblyInfo.cs │ ├── ShardedConnectionPair.csproj │ └── ShardedConnectionPairVertex.cs └── ShardedDatasetTest │ ├── App.config │ ├── ConfigurationsManager.cs │ ├── IIntKeyedDatasetObserver.cs │ ├── IntKeyedDataset.cs │ ├── IntKeyedDatasetExtensions.cs │ ├── Program.cs │ ├── Properties │ └── AssemblyInfo.cs │ ├── ShardedDatasetTest.csproj │ ├── StringKeyedDataset.cs │ └── observers │ └── WriteToConsoleObserver.cs └── privatesettings.config.example /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Microsoft 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Common Runtime for Applications 2 | 3 | Common Runtime for Applications (CRA) is a software layer (library) that makes it easy to create and deploy distributed dataflow-style applications on top of resource managers such as Kubernetes, YARN, and stand-alone cluster execution. Currently, we support stand-alone execution (just deploy an .exe on every machine in your cluster) as well as execution in a Kubernetes/Docker environment. 4 | 5 | CRA has been used to build both offline and streaming analytics platforms such as [Quill](https://www.microsoft.com/en-us/research/wp-content/uploads/2016/08/quill-tr-2016.pdf) and online microservice fabrics such as [Ambrosia](https://github.com/Microsoft/AMBROSIA/). Learn more about CRA here: 6 | 7 | * A [short paper](https://badrish.net/papers/cra-icde19-short.pdf) which appeared at the ICDE 2019 conference 8 | * Our [technical report](https://badrish.net/papers/cra-tr.pdf) has more details. 9 | 10 | After you clone the source code, check out the wiki at https://github.com/Microsoft/CRA/wiki for instruction on building CRA and running your first sample distributed application - either locally or in a Kubernetes (Windows) cluster using a Docker image of CRA. We have provided detailed step-by-step instructions for accomplishing this in the wiki. We show how to do this on [Azure Container Service](https://azure.microsoft.com/en-us/services/container-service/) (ACS), but CRA should work on any other Kubernetes cluster as well. 11 | 12 | # Contributing 13 | 14 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 15 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/AzureProvider/AzureBlobProvider.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.DataProvider.Azure 2 | { 3 | using CRA.DataProvider; 4 | using Microsoft.WindowsAzure.Storage.Blob; 5 | using System; 6 | using System.IO; 7 | using System.Threading.Tasks; 8 | 9 | /// 10 | /// Definition for AzureBlobProvider 11 | /// 12 | public class AzureBlobProvider 13 | : IBlobStorageProvider 14 | { 15 | private readonly CloudBlobClient _blobClient; 16 | private readonly string _parentBlobName; 17 | 18 | public AzureBlobProvider( 19 | CloudBlobClient blobClient, 20 | string parentBlobName) 21 | { 22 | _blobClient = blobClient; 23 | _parentBlobName = parentBlobName; 24 | } 25 | 26 | public async Task Delete(string pathKey) 27 | => await (await CreateBlockBlobAsync(pathKey)).DeleteIfExistsAsync(); 28 | 29 | public async Task GetReadStream(string pathKey) 30 | => await (await CreateBlockBlobAsync(pathKey)).OpenReadAsync(); 31 | 32 | public async Task GetWriteStream(string pathKey) 33 | => await (await CreateBlockBlobAsync(pathKey)).OpenWriteAsync(); 34 | 35 | private async Task CreateBlockBlobAsync(string pathKey) 36 | { 37 | CloudBlobContainer container = _blobClient 38 | .GetContainerReference(_parentBlobName); 39 | await container.CreateIfNotExistsAsync(); 40 | return container.GetBlockBlobReference(pathKey); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/AzureProvider/AzureDataProvider.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.DataProvider.Azure 2 | { 3 | using System; 4 | using System.Configuration; 5 | using Microsoft.WindowsAzure.Storage; 6 | using Microsoft.WindowsAzure.Storage.Table; 7 | using CRA.ClientLibrary; 8 | using System.Threading.Tasks; 9 | 10 | /// 11 | /// Definition for AzureDataProvider 12 | /// 13 | public class AzureDataProvider : IDataProvider 14 | { 15 | private readonly CloudStorageAccount _storageAccount; 16 | private readonly CloudTableClient _tableClient; 17 | private readonly string _storageConnectionString; 18 | 19 | public AzureDataProvider() 20 | { 21 | _storageConnectionString = null; 22 | #if !DOTNETCORE 23 | _storageConnectionString = ConfigurationManager.AppSettings.Get("AZURE_STORAGE_CONN_STRING"); 24 | #endif 25 | if (_storageConnectionString == null) 26 | { 27 | _storageConnectionString = Environment.GetEnvironmentVariable("AZURE_STORAGE_CONN_STRING"); 28 | } 29 | if (_storageConnectionString == null) 30 | { 31 | throw new InvalidOperationException("Azure storage connection string not found. Use appSettings in your app.config to provide this using the key AZURE_STORAGE_CONN_STRING, or use the environment variable AZURE_STORAGE_CONN_STRING."); 32 | } 33 | 34 | _storageAccount = CloudStorageAccount.Parse(_storageConnectionString); 35 | _tableClient = _storageAccount.CreateCloudTableClient(); 36 | } 37 | 38 | public AzureDataProvider(string storageConnectionString) 39 | { 40 | _storageConnectionString = storageConnectionString; 41 | _storageAccount = CloudStorageAccount.Parse(_storageConnectionString); 42 | _tableClient = _storageAccount.CreateCloudTableClient(); 43 | } 44 | 45 | public IVertexInfoProvider GetVertexInfoProvider() 46 | => new AzureVertexInfoProvider(CreateTableIfNotExists("cravertextable")); 47 | 48 | public IEndpointInfoProvider GetEndpointInfoProvider() 49 | => new AzureEndpointInfoProvider(CreateTableIfNotExists("craendpointtable")); 50 | 51 | public IVertexConnectionInfoProvider GetVertexConnectionInfoProvider() 52 | => new AzureVertexConnectionInfoProvider(CreateTableIfNotExists("craconnectiontable")); 53 | 54 | public IShardedVertexInfoProvider GetShardedVertexInfoProvider() 55 | => new AzureShardedVertexInfoProvider(CreateTableIfNotExists("crashardedvertextable")); 56 | 57 | private CloudTable CreateTableIfNotExists(string tableName) 58 | { 59 | CloudTable table = _tableClient.GetTableReference(tableName); 60 | try 61 | { 62 | Task.Run(async () => await table.CreateIfNotExistsAsync()).Wait(); 63 | } 64 | catch { } 65 | 66 | return table; 67 | } 68 | 69 | public IBlobStorageProvider GetBlobStorageProvider() 70 | => new AzureBlobProvider(_storageAccount.CreateCloudBlobClient(), "cra"); 71 | 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/AzureProvider/AzureShardedVertexInfoProvider.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.DataProvider.Azure 2 | { 3 | using Microsoft.WindowsAzure.Storage.Table; 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Linq; 7 | using System.Threading.Tasks; 8 | using CRA.ClientLibrary; 9 | 10 | /// 11 | /// Definition for AzureShardedShardedVertexInfoProvider 12 | /// 13 | public class AzureShardedVertexInfoProvider 14 | : IShardedVertexInfoProvider 15 | { 16 | private readonly CloudTable cloudTable; 17 | 18 | public AzureShardedVertexInfoProvider(CloudTable cloudTable) 19 | { 20 | this.cloudTable = cloudTable; 21 | } 22 | 23 | public async Task> GetAll() 24 | => (await cloudTable.ExecuteQueryAsync(new TableQuery())) 25 | .Select(vt => (ShardedVertexInfo)vt); 26 | 27 | public async Task CountAll() 28 | => (await GetAll()).Count(); 29 | 30 | public async Task GetEntryForVertex(string vertexName, string epochId) 31 | => (await GetAll()).Where(gn => vertexName == gn.VertexName && epochId == gn.EpochId).First(); 32 | 33 | public async Task> GetEntriesForVertex(string vertexName) 34 | { 35 | var query = new TableQuery() 36 | .Where( 37 | TableQuery.GenerateFilterCondition( 38 | "PartitionKey", 39 | QueryComparisons.Equal, 40 | vertexName)); 41 | 42 | List vtEntries = new List(); 43 | foreach (var vt in (await cloudTable.ExecuteQueryAsync(query))) 44 | vtEntries.Add(new ShardedVertexInfo(vt.VertexName, vt.EpochId, vt.AllInstances, vt.AllShards, vt.AddedShards, vt.RemovedShards, vt.ShardLocator)); 45 | 46 | return vtEntries; 47 | 48 | /*return (await cloudTable.ExecuteQueryAsync(query)) 49 | .Select(vt => (ShardedVertexInfo)vt);*/ 50 | } 51 | 52 | public async Task GetLatestEntryForVertex(string vertexName) 53 | => (await GetAll()).Where(gn => vertexName == gn.VertexName) 54 | .OrderByDescending(gn => int.Parse(gn.EpochId)) 55 | .First(); 56 | 57 | public Task Delete() 58 | => cloudTable.DeleteIfExistsAsync(); 59 | 60 | public Task Insert(ShardedVertexInfo shardedVertexInfo) 61 | { 62 | TableOperation insertOperation = 63 | TableOperation.InsertOrReplace((ShardedVertexTable)shardedVertexInfo); 64 | 65 | return cloudTable.ExecuteAsync(insertOperation); 66 | } 67 | 68 | public Task Delete(ShardedVertexInfo entry) 69 | { 70 | var row = (ShardedVertexTable)entry; 71 | row.ETag = "*"; 72 | return cloudTable.ExecuteAsync(TableOperation.Delete(row)); 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/AzureProvider/AzureVertexConnectionInfoProvider.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.DataProvider.Azure 2 | { 3 | using Microsoft.WindowsAzure.Storage.Table; 4 | using System.Collections.Generic; 5 | using System.Linq; 6 | using System.Threading.Tasks; 7 | using CRA.ClientLibrary; 8 | 9 | public class AzureVertexConnectionInfoProvider 10 | : IVertexConnectionInfoProvider 11 | { 12 | private CloudTable _cloudTable; 13 | 14 | public AzureVertexConnectionInfoProvider(CloudTable cloudTable) 15 | { _cloudTable = cloudTable; } 16 | 17 | public Task Add(VertexConnectionInfo vertexConnectionInfo) 18 | => _cloudTable.ExecuteAsync( 19 | TableOperation.InsertOrReplace( 20 | (ConnectionTable)vertexConnectionInfo)); 21 | 22 | public async Task ContainsRow(VertexConnectionInfo entity) 23 | { 24 | var temp = await GetAll(); 25 | 26 | return temp 27 | .Where(gn => entity.Equals(gn)) 28 | .Count() > 0; 29 | } 30 | 31 | public async Task CountAll() 32 | { 33 | var query = new TableQuery(); 34 | return (await _cloudTable.ExecuteQueryAsync(query)).Count(); 35 | } 36 | 37 | public Task Delete(VertexConnectionInfo vci) 38 | => _cloudTable.ExecuteAsync( 39 | TableOperation.Delete((ConnectionTable)vci)); 40 | 41 | public Task DeleteStore() 42 | => _cloudTable.DeleteIfExistsAsync(); 43 | 44 | public async Task Get(string fromVertex, string fromOutput, string toConnection, string toInput) 45 | { 46 | var connectionTable = new ConnectionTable( 47 | fromVertex, 48 | fromOutput, 49 | toConnection, 50 | toInput); 51 | 52 | TableOperation retrieveOperation = TableOperation.Retrieve( 53 | connectionTable.PartitionKey, 54 | connectionTable.RowKey); 55 | 56 | TableResult retrievedResult = await _cloudTable.ExecuteAsync(retrieveOperation); 57 | if (retrievedResult.Result == null) 58 | { return null; } 59 | 60 | var result = (ConnectionTable)retrievedResult.Result; 61 | return new VertexConnectionInfo(result.FromVertex, result.FromEndpoint, result.ToVertex, result.ToEndpoint, result.ETag); 62 | 63 | //return (VertexConnectionInfo)retrievedResult.Result; 64 | } 65 | 66 | public async Task> GetAll() 67 | { 68 | var query = new TableQuery(); 69 | return (await _cloudTable.ExecuteQueryAsync(query)) 70 | .Select(_ => (VertexConnectionInfo)_); 71 | } 72 | 73 | public async Task> GetAllConnectionsFromVertex(string fromVertex) 74 | { 75 | return (await GetAll()) 76 | .Where(gn => fromVertex == gn.FromVertex); 77 | } 78 | 79 | public async Task> GetAllConnectionsToVertex(string toVertex) 80 | { 81 | return (await GetAll()) 82 | .Where(gn => toVertex == gn.ToVertex); 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/CRA.ClientLibrary.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | netcoreapp2.0;net46;netstandard2.0 4 | AnyCPU 5 | 8 6 | true 7 | CRA.ClientLibrary 8 | true 9 | false 10 | ../CRA.snk 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | $(DefineConstants);DOTNETCORE 26 | 27 | 28 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/Datasets/DatasetBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Linq.Expressions; 4 | 5 | namespace CRA.ClientLibrary.DataProcessing 6 | { 7 | public class DatasetBase : VertexBase, IDataset 8 | { 9 | 10 | public DatasetBase() { } 11 | 12 | public Expression, IDataset[]>> CreateSplitter(IMoveDescriptor descriptor) 13 | { 14 | throw new NotImplementedException(); 15 | } 16 | 17 | public Expression[], IDataset>> CreateMerger(IMoveDescriptor descriptor) 18 | { 19 | throw new NotImplementedException(); 20 | } 21 | 22 | public virtual IDataset ReKey(Expression> selector) 23 | { 24 | throw new NotImplementedException(); 25 | } 26 | 27 | public virtual void ToStream(Stream stream) 28 | { 29 | throw new NotImplementedException(); 30 | } 31 | 32 | public virtual Expression>> CreateFromStreamDeserializer() 33 | { 34 | throw new NotImplementedException(); 35 | } 36 | 37 | public virtual void Subscribe(object observer) 38 | { 39 | throw new NotImplementedException(); 40 | } 41 | 42 | public virtual IDataset ToObject() 43 | { 44 | throw new NotImplementedException(); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/Datasets/IDataset.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Linq.Expressions; 4 | 5 | namespace CRA.ClientLibrary.DataProcessing 6 | { 7 | public interface IDataset 8 | { 9 | Expression, IDataset[]>> CreateSplitter(IMoveDescriptor descriptor); 10 | 11 | Expression[], IDataset>> CreateMerger(IMoveDescriptor descriptor); 12 | 13 | IDataset ReKey(Expression> selector); 14 | 15 | void Subscribe(object observer); 16 | 17 | void ToStream(Stream stream); 18 | 19 | IDataset ToObject(); 20 | 21 | Expression>> CreateFromStreamDeserializer(); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/Datasets/IDeployable.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.ClientLibrary.DataProcessing 2 | { 3 | internal interface IDeployable 4 | { 5 | void Deploy(ref TaskBase task, ref OperatorsToplogy topology, ref OperatorTransforms operandTransforms); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/Datasets/IShardedDataset.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq.Expressions; 3 | using System.Threading.Tasks; 4 | 5 | namespace CRA.ClientLibrary.DataProcessing 6 | { 7 | public interface IShardedDataset 8 | where TDataset : IDataset 9 | { 10 | 11 | IShardedDataset Transform( 12 | Expression> transformer) 13 | where TDataset2 : IDataset; 14 | 15 | IShardedDataset Transform( 16 | IShardedDataset input2, 17 | Expression> transformer) 18 | where TDataset2 : IDataset 19 | where TDatasetO : IDataset; 20 | 21 | IShardedDataset Move( 22 | Expression> splitter, 23 | Expression> merger, IMoveDescriptor moveDescriptor) 24 | where TDataset2 : IDataset 25 | where TDatasetO : IDataset; 26 | 27 | IShardedDataset ReKey(Expression> selector) 28 | where TDataset2 : IDataset; 29 | 30 | Task> Deploy(); 31 | 32 | Task Subscribe(Expression> observer); 33 | 34 | Task MultiSubscribe(Expression> observer, int runsCount); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/Definitions/Operator/OperatorTransforms.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace CRA.ClientLibrary.DataProcessing 4 | { 5 | public class OperatorTransforms 6 | { 7 | public List Transforms { get; set; } 8 | public List TransformsOperations { get; set; } 9 | public List TransformsTypes { get; set; } 10 | public List TransformsInputs { get; set; } 11 | 12 | public OperatorTransforms() 13 | { 14 | Transforms = new List(); 15 | TransformsOperations = new List(); 16 | TransformsTypes = new List(); 17 | TransformsInputs = new List(); 18 | } 19 | 20 | public OperatorTransforms(List transforms, List transformsOperations, 21 | List transformsTypes, List tranformsInputs) 22 | { 23 | Transforms = transforms; 24 | TransformsOperations = transformsOperations; 25 | TransformsTypes = transformsTypes; 26 | TransformsInputs = tranformsInputs; 27 | } 28 | 29 | public void AddTransform(string transform, string transformOperation, string transformType, OperatorInputs transformInput) 30 | { 31 | Transforms.Add(transform); 32 | TransformsOperations.Add(transformOperation); 33 | TransformsTypes.Add(transformType); 34 | TransformsInputs.Add(transformInput); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/Definitions/Operator/OperatorType.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.ClientLibrary.DataProcessing 2 | { 3 | /// 4 | /// Operations that can be applied to an IShardedDataset 5 | /// 6 | public enum OperatorType : int 7 | { 8 | /// 9 | /// Transforms one IDataset to another IDataset type 10 | /// 11 | UnaryTransform, 12 | /// 13 | /// Transforms both IDataset1 and IDataset2 to another IDataset 14 | /// 15 | BinaryTransform, 16 | /// 17 | /// Moves data between two CRA sharded vertices 18 | /// 19 | Move, 20 | /// 21 | /// Splitting phase of the move operation between two CRA sharded vertices 22 | /// 23 | MoveSplit, 24 | /// 25 | /// Merging phase of the move operation between two CRA sharded vertices 26 | /// 27 | MoveMerge, 28 | /// 29 | /// Produces data to be used as an input by other sharded CRA vertices 30 | /// 31 | Produce, 32 | /// 33 | /// Subscribes an observer to the output data from other CRA sharded vertices to be used as an output 34 | /// 35 | Subscribe, 36 | /// 37 | /// Connects the client to the other CRA sharded vertices 38 | /// 39 | ClientTerminal, 40 | /// 41 | /// Peroforms no action on a sharded dataset 42 | /// 43 | None 44 | }; 45 | } 46 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/Definitions/Tasks/ClientTerminalTask.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.Serialization; 2 | 3 | namespace CRA.ClientLibrary.DataProcessing 4 | { 5 | [DataContract] 6 | public class ClientTerminalTask : TaskBase 7 | { 8 | public ClientTerminalTask() : base(OperatorType.ClientTerminal) { } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/Definitions/Tasks/ProduceTask.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.Serialization; 2 | 3 | namespace CRA.ClientLibrary.DataProcessing 4 | { 5 | [DataContract] 6 | public class ProduceTask : TaskBase 7 | { 8 | [DataMember] 9 | private string _producer; 10 | 11 | public string DataProducer { get { return _producer; } } 12 | 13 | public ProduceTask(string producer) : base(OperatorType.Produce) 14 | { 15 | _producer = producer; 16 | } 17 | 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/Definitions/Tasks/ShuffleTask.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.Serialization; 2 | 3 | namespace CRA.ClientLibrary.DataProcessing 4 | { 5 | [DataContract] 6 | public class ShuffleTask : TaskBase 7 | { 8 | [DataMember] 9 | internal string MapperVertexName { get; set; } 10 | 11 | [DataMember] 12 | internal string ReducerVertexName { get; set; } 13 | 14 | [DataMember] 15 | internal string[] ShuffleTransforms { get; set; } 16 | 17 | [DataMember] 18 | internal string[] ShuffleTransformsOperations { get; set; } 19 | 20 | [DataMember] 21 | internal string[] ShuffleTransformsTypes { get; set; } 22 | 23 | [DataMember] 24 | internal OperatorInputs[] ShuffleTransformsInputs { get; set; } 25 | 26 | [DataMember] 27 | internal IMoveDescriptor ShuffleDescriptor { get; set; } 28 | 29 | [DataMember] 30 | internal OperatorEndpointsDescriptor _secondaryEndpointsDescriptor; 31 | 32 | public OperatorEndpointsDescriptor SecondaryEndpointsDescriptor 33 | { 34 | set 35 | { 36 | _secondaryEndpointsDescriptor = value; 37 | } 38 | 39 | get 40 | { 41 | if (_secondaryEndpointsDescriptor == null) 42 | _secondaryEndpointsDescriptor = new OperatorEndpointsDescriptor(); 43 | 44 | return _secondaryEndpointsDescriptor; 45 | } 46 | } 47 | 48 | public ShuffleTask(IMoveDescriptor shuffleDescriptor) : base(OperatorType.Move) 49 | { 50 | ShuffleDescriptor = shuffleDescriptor; 51 | } 52 | 53 | internal void PrepareShuffleTransformations(OperatorTransforms nodeMergedOperand) 54 | { 55 | if (nodeMergedOperand.Transforms.Count > 0) 56 | { 57 | ShuffleTransforms = nodeMergedOperand.Transforms.ToArray(); 58 | ShuffleTransformsOperations = nodeMergedOperand.TransformsOperations.ToArray(); 59 | ShuffleTransformsTypes = nodeMergedOperand.TransformsTypes.ToArray(); 60 | ShuffleTransformsInputs = nodeMergedOperand.TransformsInputs.ToArray(); 61 | } 62 | } 63 | 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/Definitions/Tasks/SubscribeTask.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.Serialization; 2 | 3 | namespace CRA.ClientLibrary.DataProcessing 4 | { 5 | [DataContract] 6 | public class SubscribeTask : TaskBase 7 | { 8 | public SubscribeTask() : base(OperatorType.Subscribe){} 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/ShardedOperators/ShardedEndpoints/ShardedOperatorOutputBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.IO; 5 | using System.Linq.Expressions; 6 | using System.Text; 7 | using System.Threading; 8 | using System.Threading.Tasks; 9 | 10 | namespace CRA.ClientLibrary.DataProcessing 11 | { 12 | public abstract class ShardedOperatorOutputBase : AsyncShardedVertexOutputEndpointBase 13 | { 14 | protected int _shardId; 15 | protected string _endpointName; 16 | 17 | protected byte[] _deployMsgBuffer; 18 | protected byte[] _runMsgBuffer; 19 | protected CountdownEvent _startSendingToOtherOperatorShards; 20 | protected CountdownEvent _finishSendingToOtherOperatorShards; 21 | 22 | protected Dictionary _isEndpointEstablished; 23 | 24 | public ShardedOperatorOutputBase(int shardId, int numOtherOperatorShards, string endpointName) 25 | { 26 | _shardId = shardId; 27 | _endpointName = endpointName; 28 | 29 | _deployMsgBuffer = new byte[Encoding.ASCII.GetBytes("DEPLOY").Length]; 30 | _runMsgBuffer = new byte[Encoding.ASCII.GetBytes("RUN").Length]; 31 | 32 | _startSendingToOtherOperatorShards = new CountdownEvent(numOtherOperatorShards); 33 | _finishSendingToOtherOperatorShards = new CountdownEvent(numOtherOperatorShards); 34 | 35 | _isEndpointEstablished = new Dictionary(); 36 | } 37 | 38 | public override void Dispose() 39 | { 40 | Dispose(true); 41 | GC.SuppressFinalize(this); 42 | } 43 | 44 | protected virtual void Dispose(bool disposing) 45 | { 46 | if (disposing) 47 | { 48 | Trace.TraceInformation("Disposing " + _endpointName); 49 | } 50 | } 51 | 52 | public override async Task ToStreamAsync(Stream stream, string otherVertex, int otherShardId, string otherEndpoint, CancellationToken token) 53 | { 54 | if (!(_isEndpointEstablished.ContainsKey(otherShardId) && _isEndpointEstablished[otherShardId])) 55 | { 56 | _isEndpointEstablished[otherShardId] = true; 57 | await OperatorOutputToStreamAsync(stream, otherVertex, otherShardId, otherEndpoint, token); 58 | } 59 | } 60 | 61 | public abstract Task OperatorOutputToStreamAsync(Stream stream, string otherVertex, int otherShardId, string otherEndpoint, CancellationToken token); 62 | 63 | public void StartProducer(Object dataset, Stream stream) 64 | where TDataset : IDataset 65 | { 66 | ((TDataset)dataset).ToStream(stream); 67 | } 68 | 69 | public void SubscribeObserver(object dataset, string observerExpression) 70 | where TDataset : IDataset 71 | { 72 | Expression observer = SerializationHelper.Deserialize(observerExpression); 73 | Delegate compiledObserver = Expression.Lambda(observer).Compile(); 74 | Delegate observerConstructor = (Delegate)compiledObserver.DynamicInvoke(); 75 | object observerObject = observerConstructor.DynamicInvoke(); 76 | ((TDataset)dataset).Subscribe(observerObject); 77 | } 78 | 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/ShardedOperators/ShardedEndpoints/ShardedProducerSecondaryInput.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Text; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | 7 | namespace CRA.ClientLibrary.DataProcessing 8 | { 9 | public class ShardedProducerSecondaryInput : ShardedOperatorInputBase 10 | { 11 | private ShardedProducerOperator _vertex; 12 | 13 | public ShardedProducerSecondaryInput(IVertex vertex, int shardId, int numOtherOperatorShards, string endpointName) : base(shardId, numOtherOperatorShards, endpointName) 14 | { 15 | _vertex = (ShardedProducerOperator)vertex; 16 | } 17 | 18 | public override async Task OperatorInputFromStreamAsync(Stream stream, string otherVertex, int otherShardId, string otherEndpoint, CancellationToken token) 19 | { 20 | _startReceivingFromOtherOperatorShards.Signal(); 21 | _startReceivingFromOtherOperatorShards.Wait(); 22 | 23 | if (_shardId == otherShardId) 24 | { 25 | // Start deploying 26 | _vertex._deployProduceInput.Wait(); 27 | 28 | await stream.WriteAsync(_deployMsgBuffer, 0, _deployMsgBuffer.Length); 29 | await stream.ReadAllRequiredBytesAsync(_deployMsgBuffer, 0, _deployMsgBuffer.Length); 30 | if (Encoding.ASCII.GetString(_deployMsgBuffer).Equals("DEPLOY")) 31 | { 32 | _vertex._deployProduceOutput.Signal(); 33 | 34 | // Start running 35 | _vertex._runProduceInput.Wait(); 36 | 37 | await stream.WriteAsync(_runMsgBuffer, 0, _runMsgBuffer.Length); 38 | 39 | _vertex._startCreatingSecondaryDatasets[otherVertex].Wait(); 40 | 41 | object dataset = CreateDatasetFromStream(stream, _vertex._binaryOperatorTypes[otherVertex].SecondaryKeyType, 42 | _vertex._binaryOperatorTypes[otherVertex].SecondaryPayloadType, _vertex._binaryOperatorTypes[otherVertex].SecondaryDatasetType); 43 | 44 | if (!_vertex._cachedDatasets[_shardId].ContainsKey(otherVertex)) 45 | _vertex._cachedDatasets[_shardId].Add(otherVertex, dataset); 46 | else 47 | _vertex._cachedDatasets[_shardId][otherVertex] = dataset; 48 | 49 | _vertex._finishCreatingSecondaryDatasets[otherVertex].Signal(); 50 | 51 | _vertex._runProduceOutput.Signal(); 52 | } 53 | 54 | } 55 | 56 | _finishReceivingFromOtherOperatorShards.Signal(); 57 | _finishReceivingFromOtherOperatorShards.Wait(); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/ShardedOperators/ShardedEndpoints/ShardedProducerSecondaryOutput.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Reflection; 4 | using System.Text; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | 8 | namespace CRA.ClientLibrary.DataProcessing 9 | { 10 | public class ShardedProducerSecondaryOutput : ShardedOperatorOutputBase 11 | { 12 | private ShardedProducerOperator _vertex; 13 | 14 | public ShardedProducerSecondaryOutput(IVertex vertex, int shardId, int numOtherOperatorShards, string endpointName) : base(shardId, numOtherOperatorShards, endpointName) 15 | { 16 | _vertex = (ShardedProducerOperator)vertex; 17 | } 18 | 19 | public override async Task OperatorOutputToStreamAsync(Stream stream, string otherVertex, int otherShardId, string otherEndpoint, CancellationToken token) 20 | { 21 | _startSendingToOtherOperatorShards.Signal(); 22 | _startSendingToOtherOperatorShards.Wait(); 23 | 24 | if (_shardId == otherShardId) 25 | { 26 | // Start deploying 27 | await stream.ReadAllRequiredBytesAsync(_deployMsgBuffer, 0, _deployMsgBuffer.Length); 28 | if (Encoding.ASCII.GetString(_deployMsgBuffer).Equals("DEPLOY")) 29 | { 30 | if (_vertex._hasSecondaryInput) _vertex._deployProduceInput.Signal(); 31 | if (_vertex._hasSecondaryInput) _vertex._deployProduceOutput.Wait(); 32 | 33 | await stream.WriteAsync(_deployMsgBuffer, 0, _deployMsgBuffer.Length); 34 | 35 | // Start running 36 | await stream.ReadAllRequiredBytesAsync(_runMsgBuffer, 0, _runMsgBuffer.Length); 37 | if (Encoding.ASCII.GetString(_runMsgBuffer).Equals("RUN")) 38 | { 39 | if (_vertex._hasSecondaryInput) _vertex._runProduceInput.Signal(); 40 | 41 | if (_vertex._hasSecondaryInput) 42 | { 43 | if (!_vertex._isTransformationsApplied) 44 | { 45 | lock (_vertex._transformationLock) 46 | { 47 | if (!_vertex._isTransformationsApplied) 48 | { 49 | _vertex.CreateAndTransformDataset(_shardId); 50 | _vertex._isTransformationsApplied = true; 51 | 52 | _vertex._continueAfterTransformation.Signal(); 53 | } 54 | } 55 | } 56 | } 57 | 58 | _vertex._continueAfterTransformation.Wait(); 59 | 60 | if (_vertex._hasSecondaryInput) _vertex._runProduceOutput.Wait(); 61 | 62 | MethodInfo method = typeof(ShardedOperatorOutputBase).GetMethod("StartProducer"); 63 | MethodInfo generic = method.MakeGenericMethod( 64 | new Type[] { _vertex._outputKeyType, _vertex._outputPayloadType, _vertex._outputDatasetType }); 65 | generic.Invoke(this, new Object[] { _vertex._cachedDatasets[_shardId][_vertex._outputId], stream }); 66 | } 67 | } 68 | } 69 | 70 | _finishSendingToOtherOperatorShards.Signal(); 71 | _finishSendingToOtherOperatorShards.Wait(); 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/ShardedOperators/ShardedEndpoints/ShardedShuffleSecondaryInput.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Text; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | 7 | namespace CRA.ClientLibrary.DataProcessing 8 | { 9 | public class ShardedShuffleSecondaryInput : ShardedOperatorInputBase 10 | { 11 | private ShardedShuffleOperator _vertex; 12 | 13 | public ShardedShuffleSecondaryInput(IVertex vertex, int shardId, int numOtherOperatorShards, string endpointName) : base(shardId, numOtherOperatorShards, endpointName) 14 | { 15 | _vertex = (ShardedShuffleOperator)vertex; 16 | } 17 | 18 | public override async Task OperatorInputFromStreamAsync(Stream stream, string otherVertex, int otherShardId, string otherEndpoint, CancellationToken token) 19 | { 20 | _startReceivingFromOtherOperatorShards.Signal(); 21 | _startReceivingFromOtherOperatorShards.Wait(); 22 | 23 | if (_shardId == otherShardId) 24 | { 25 | // Start deploying 26 | _vertex._deployShuffleInput.Wait(); 27 | 28 | await stream.WriteAsync(_deployMsgBuffer, 0, _deployMsgBuffer.Length); 29 | await stream.ReadAllRequiredBytesAsync(_deployMsgBuffer, 0, _deployMsgBuffer.Length); 30 | if (Encoding.ASCII.GetString(_deployMsgBuffer).Equals("DEPLOY")) 31 | { 32 | _vertex._deployShuffleOutput.Signal(); 33 | 34 | // Start running 35 | _vertex._runShuffleInput.Wait(); 36 | 37 | await stream.WriteAsync(_runMsgBuffer, 0, _runMsgBuffer.Length); 38 | 39 | _vertex._startCreatingSecondaryDatasets[otherVertex].Wait(); 40 | 41 | object dataset = CreateDatasetFromStream(stream, _vertex._binaryOperatorTypes[otherVertex].SecondaryKeyType, 42 | _vertex._binaryOperatorTypes[otherVertex].SecondaryPayloadType, _vertex._binaryOperatorTypes[otherVertex].SecondaryDatasetType); 43 | 44 | if (!_vertex._cachedDatasets[_shardId].ContainsKey(otherVertex)) 45 | _vertex._cachedDatasets[_shardId].Add(otherVertex, dataset); 46 | else 47 | _vertex._cachedDatasets[_shardId][otherVertex] = dataset; 48 | 49 | _vertex._finishCreatingSecondaryDatasets[otherVertex].Signal(); 50 | 51 | _vertex._runShuffleOutput.Signal(); 52 | } 53 | 54 | _finishReceivingFromOtherOperatorShards.Signal(); 55 | _finishReceivingFromOtherOperatorShards.Wait(); 56 | } 57 | } 58 | } 59 | } -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/ShardedOperators/ShardedEndpoints/ShardedShuffleSecondaryOutput.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Reflection; 4 | using System.Text; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | 8 | namespace CRA.ClientLibrary.DataProcessing 9 | { 10 | public class ShardedShuffleSecondaryOutput : ShardedOperatorOutputBase 11 | { 12 | private ShardedShuffleOperator _vertex; 13 | 14 | public ShardedShuffleSecondaryOutput(IVertex vertex, int shardId, int numOtherOperatorShards, string endpointName) : base(shardId, numOtherOperatorShards, endpointName) 15 | { 16 | _vertex = (ShardedShuffleOperator)vertex; 17 | } 18 | 19 | public override async Task OperatorOutputToStreamAsync(Stream stream, string otherVertex, int otherShardId, string otherEndpoint, CancellationToken token) 20 | { 21 | _startSendingToOtherOperatorShards.Signal(); 22 | _startSendingToOtherOperatorShards.Wait(); 23 | 24 | if (_shardId == otherShardId) 25 | { 26 | // Start deploying 27 | await stream.ReadAllRequiredBytesAsync(_deployMsgBuffer, 0, _deployMsgBuffer.Length); 28 | if (Encoding.ASCII.GetString(_deployMsgBuffer).Equals("DEPLOY")) 29 | { 30 | _vertex._deployShuffleInput.Signal(); 31 | _vertex._deployShuffleOutput.Wait(); 32 | 33 | await stream.WriteAsync(_deployMsgBuffer, 0, _deployMsgBuffer.Length); 34 | 35 | // Start running 36 | await stream.ReadAllRequiredBytesAsync(_runMsgBuffer, 0, _runMsgBuffer.Length); 37 | if (Encoding.ASCII.GetString(_runMsgBuffer).Equals("RUN")) 38 | { 39 | _vertex._runShuffleInput.Signal(); 40 | _vertex._runShuffleOutput.Wait(); 41 | 42 | MethodInfo method = typeof(ShardedOperatorOutputBase).GetMethod("StartProducer"); 43 | MethodInfo generic = method.MakeGenericMethod( 44 | new Type[] { _vertex._outputKeyType, _vertex._outputPayloadType, _vertex._outputDatasetType }); 45 | generic.Invoke(this, new Object[] { _vertex._cachedDatasets[_shardId][_vertex._outputId], stream }); 46 | } 47 | } 48 | } 49 | 50 | _finishSendingToOtherOperatorShards.Signal(); 51 | _finishSendingToOtherOperatorShards.Wait(); 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/ShardedOperators/ShardedEndpoints/ShardedSubscribeClientInput.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Text; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | 7 | namespace CRA.ClientLibrary.DataProcessing 8 | { 9 | public class ShardedSubscribeClientInput : ShardedOperatorInputBase 10 | { 11 | private ShardedSubscribeClientOperator _vertex; 12 | 13 | public ShardedSubscribeClientInput(IVertex vertex, int shardId, int numOtherOperatorShards, string endpointName) : base(shardId, numOtherOperatorShards, endpointName) 14 | { 15 | _vertex = (ShardedSubscribeClientOperator)vertex; 16 | 17 | _startReceivingFromOtherOperatorShards = new CountdownEvent(_vertex._numShardsConnectingTo); 18 | _finishReceivingFromOtherOperatorShards = new CountdownEvent(_vertex._numShardsConnectingTo); 19 | } 20 | 21 | public override async Task OperatorInputFromStreamAsync(Stream stream, string otherVertex, int otherShardId, string otherEndpoint, CancellationToken token) 22 | { 23 | _startReceivingFromOtherOperatorShards.Signal(); 24 | _startReceivingFromOtherOperatorShards.Wait(); 25 | 26 | // Start deploying 27 | _vertex._deploySubscribeInput.Wait(); 28 | 29 | await stream.WriteAsync(_deployMsgBuffer, 0, _deployMsgBuffer.Length); 30 | await stream.ReadAllRequiredBytesAsync(_deployMsgBuffer, 0, _deployMsgBuffer.Length); 31 | if (Encoding.ASCII.GetString(_deployMsgBuffer).Equals("DEPLOY")) 32 | { 33 | _vertex._deploySubscribeOutput.Signal(); 34 | 35 | // Start running 36 | _vertex._runSubscribeInput.Wait(); 37 | 38 | await stream.WriteAsync(_runMsgBuffer, 0, _runMsgBuffer.Length); 39 | 40 | if (!_vertex._cachedDatasets[_shardId].ContainsKey(_vertex._task.InputIds.InputId1 + otherShardId)) 41 | { 42 | object dataset = CreateDatasetFromStream(stream, _vertex._task.OperationTypes.OutputKeyType, 43 | _vertex._task.OperationTypes.OutputPayloadType, _vertex._task.OperationTypes.OutputDatasetType); 44 | _vertex._cachedDatasets[_shardId].Add(_vertex._task.InputIds.InputId1 + otherShardId, dataset); 45 | } 46 | 47 | _vertex._runSubscribeOutput.Signal(); 48 | } 49 | 50 | _finishReceivingFromOtherOperatorShards.Signal(); 51 | _finishReceivingFromOtherOperatorShards.Wait(); 52 | } 53 | 54 | } 55 | } -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/ShardedOperators/ShardedEndpoints/ShardedSubscribeClientOutput.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | using System.IO; 4 | using System.Linq.Expressions; 5 | using System.Reflection; 6 | using System.Text; 7 | using System.Threading; 8 | using System.Threading.Tasks; 9 | 10 | namespace CRA.ClientLibrary.DataProcessing 11 | { 12 | public class ShardedSubscribeClientOutput : ShardedOperatorOutputBase 13 | { 14 | private ShardedSubscribeClientOperator _vertex; 15 | 16 | public ShardedSubscribeClientOutput(IVertex vertex, int shardId, int numOtherOperatorShards, string endpointName) : base(shardId, numOtherOperatorShards, endpointName) 17 | { 18 | _vertex = (ShardedSubscribeClientOperator)vertex; 19 | } 20 | 21 | public override async Task OperatorOutputToStreamAsync(Stream stream, string otherVertex, int otherShardId, string otherEndpoint, CancellationToken token) 22 | { 23 | MemoryStream memStream = (MemoryStream)stream; 24 | memStream.Seek(0, SeekOrigin.Begin); 25 | 26 | // Start deploying 27 | memStream.Read(_deployMsgBuffer, 0, _deployMsgBuffer.Length); 28 | if (Encoding.ASCII.GetString(_deployMsgBuffer).Equals("DEPLOY")) 29 | { 30 | _vertex._deploySubscribeInput.Signal(); 31 | _vertex._deploySubscribeOutput.Wait(); 32 | 33 | // Start running 34 | memStream.Read(_runMsgBuffer, 0, _runMsgBuffer.Length); 35 | if (Encoding.ASCII.GetString(_runMsgBuffer).Equals("RUN")) 36 | { 37 | _vertex._outputObserver = Encoding.UTF8.GetString(memStream.ReadByteArray()); 38 | 39 | _vertex._runSubscribeInput.Signal(); 40 | _vertex._runSubscribeOutput.Wait(); 41 | 42 | ApplySubscribe(); 43 | } 44 | } 45 | } 46 | 47 | private void ApplySubscribe() 48 | { 49 | MethodInfo method = typeof(ShardedOperatorOutputBase).GetMethod("SubscribeObserver"); 50 | MethodInfo generic = method.MakeGenericMethod( 51 | new Type[] {_vertex._task.OperationTypes.OutputKeyType, 52 | _vertex._task.OperationTypes.OutputPayloadType, 53 | _vertex._task.OperationTypes.OutputDatasetType}); 54 | 55 | foreach (var ind in _vertex._cachedDatasets[_shardId].Keys) 56 | { 57 | try 58 | { 59 | Task.Run(() => generic.Invoke(this, new Object[] { _vertex._cachedDatasets[_shardId][ind], _vertex._outputObserver})); 60 | } 61 | catch (Exception e) 62 | { 63 | Trace.TraceError("Error: CRA failed to apply the subscribe() in the client side. " + e.ToString()); 64 | } 65 | } 66 | } 67 | } 68 | } -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/ShardedOperators/ShardedEndpoints/ShardedSubscribeInput.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Text; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | 7 | namespace CRA.ClientLibrary.DataProcessing 8 | { 9 | public class ShardedSubscribeInput : ShardedOperatorInputBase 10 | { 11 | private ShardedSubscribeOperator _vertex; 12 | 13 | public ShardedSubscribeInput(IVertex vertex, int shardId, int numOtherOperatorShards, string endpointName) : base(shardId, numOtherOperatorShards, endpointName) 14 | { 15 | _vertex = (ShardedSubscribeOperator)vertex; 16 | } 17 | 18 | public override async Task OperatorInputFromStreamAsync(Stream stream, string otherVertex, int otherShardId, string otherEndpoint, CancellationToken token) 19 | { 20 | _startReceivingFromOtherOperatorShards.Signal(); 21 | _startReceivingFromOtherOperatorShards.Wait(); 22 | 23 | if (_shardId == otherShardId) 24 | { 25 | // Start deploying 26 | _vertex._deploySubscribeInput.Wait(); 27 | 28 | await stream.WriteAsync(_deployMsgBuffer, 0, _deployMsgBuffer.Length); 29 | await stream.ReadAllRequiredBytesAsync(_deployMsgBuffer, 0, _deployMsgBuffer.Length); 30 | if (Encoding.ASCII.GetString(_deployMsgBuffer).Equals("DEPLOY")) 31 | { 32 | _vertex._deploySubscribeOutput.Signal(); 33 | 34 | // Start running 35 | _vertex._runSubscribeInput.Wait(); 36 | 37 | await stream.WriteAsync(_runMsgBuffer, 0, _runMsgBuffer.Length); 38 | 39 | if (!_vertex._cachedDatasets[_shardId].ContainsKey(_vertex._task.InputIds.InputId1)) 40 | { 41 | object dataset = CreateDatasetFromStream(stream, _vertex._task.OperationTypes.OutputKeyType, 42 | _vertex._task.OperationTypes.OutputPayloadType, _vertex._task.OperationTypes.OutputDatasetType); 43 | _vertex._cachedDatasets[_shardId].Add(_vertex._task.InputIds.InputId1, dataset); 44 | } 45 | 46 | _vertex._runSubscribeOutput.Signal(); 47 | } 48 | } 49 | 50 | _finishReceivingFromOtherOperatorShards.Signal(); 51 | _finishReceivingFromOtherOperatorShards.Wait(); 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/ShardedOperators/ShardedEndpoints/ShardedSubscribeOutput.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Reflection; 4 | using System.Text; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | 8 | namespace CRA.ClientLibrary.DataProcessing 9 | { 10 | public class ShardedSubscribeOutput : ShardedOperatorOutputBase 11 | { 12 | private ShardedSubscribeOperator _vertex; 13 | 14 | public ShardedSubscribeOutput(IVertex vertex, int shardId, int numOtherOperatorShards, string endpointName) : base(shardId, numOtherOperatorShards, endpointName) 15 | { 16 | _vertex = (ShardedSubscribeOperator)vertex; 17 | } 18 | 19 | public override async Task OperatorOutputToStreamAsync(Stream stream, string otherVertex, int otherShardId, string otherEndpoint, CancellationToken token) 20 | { 21 | // Start deploying 22 | await stream.ReadAllRequiredBytesAsync(_deployMsgBuffer, 0, _deployMsgBuffer.Length); 23 | if (Encoding.ASCII.GetString(_deployMsgBuffer).Equals("DEPLOY")) 24 | { 25 | _vertex._deploySubscribeInput.Signal(); 26 | _vertex._deploySubscribeOutput.Wait(); 27 | 28 | await stream.WriteAsync(_deployMsgBuffer, 0, _deployMsgBuffer.Length); 29 | 30 | // Start running 31 | await stream.ReadAllRequiredBytesAsync(_runMsgBuffer, 0, _runMsgBuffer.Length); 32 | if (Encoding.ASCII.GetString(_runMsgBuffer).Equals("RUN")) 33 | { 34 | _vertex._runSubscribeInput.Signal(); 35 | _vertex._runSubscribeOutput.Wait(); 36 | 37 | MethodInfo method = typeof(ShardedOperatorOutputBase).GetMethod("StartProducer"); 38 | MethodInfo generic = method.MakeGenericMethod( 39 | new Type[] { _vertex._task.OperationTypes.OutputKeyType, _vertex._task.OperationTypes.OutputPayloadType, 40 | _vertex._task.OperationTypes.OutputDatasetType }); 41 | generic.Invoke(this, new Object[] { _vertex._cachedDatasets[_shardId][_vertex._task.InputIds.InputId1], stream }); 42 | } 43 | } 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/ShardedOperators/ShardedSubscribeClientOperator.cs: -------------------------------------------------------------------------------- 1 | using CRA.DataProvider; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Diagnostics; 5 | using System.Linq.Expressions; 6 | using System.Reflection; 7 | using System.Text; 8 | using System.Threading; 9 | using System.Threading.Tasks; 10 | 11 | namespace CRA.ClientLibrary.DataProcessing 12 | { 13 | public class ShardedSubscribeClientOperator : ShardedOperatorBase 14 | { 15 | internal CountdownEvent _deploySubscribeInput; 16 | internal CountdownEvent _deploySubscribeOutput; 17 | 18 | internal CountdownEvent _runSubscribeInput; 19 | internal CountdownEvent _runSubscribeOutput; 20 | 21 | internal string _outputObserver; 22 | 23 | internal int _numShardsConnectingTo; 24 | 25 | public ShardedSubscribeClientOperator() : base() 26 | { 27 | } 28 | 29 | internal override void InitializeOperator(int shardId, ShardingInfo shardingInfo) 30 | { 31 | _hasSplittedOutput = HasSplittedOutput(); 32 | 33 | _numShardsConnectingTo = 0; 34 | var instancesMap = _task.DeployDescriptor.InstancesMap(); 35 | foreach (var entry in instancesMap.Keys) 36 | _numShardsConnectingTo += instancesMap[entry]; 37 | 38 | _deploySubscribeInput = new CountdownEvent(1); 39 | _deploySubscribeOutput = new CountdownEvent(_numShardsConnectingTo); 40 | 41 | _runSubscribeInput = new CountdownEvent(1); 42 | _runSubscribeOutput = new CountdownEvent(_numShardsConnectingTo); 43 | 44 | string[] toEndpoints = GetEndpointNamesForVertex(VertexName.Split('$')[0], _toFromConnections); 45 | var fromTuple = _toFromConnections[new Tuple(VertexName.Split('$')[0], toEndpoints[0])]; 46 | if (!fromTuple.Item4) 47 | AddAsyncInputEndpoint(toEndpoints[0], new ShardedSubscribeClientInput(this, shardId, shardingInfo.AllShards.Length, toEndpoints[0])); 48 | else 49 | throw new NotImplementedException("Shared secondary endpoints are not supported in subscribe operators!!"); 50 | 51 | string fromEndpoint = "OutputToClient" + Guid.NewGuid().ToString(); 52 | AddAsyncOutputEndpoint(fromEndpoint, new ShardedSubscribeClientOutput(this, shardId, shardingInfo.AllShards.Length, fromEndpoint)); 53 | } 54 | 55 | internal override bool HasSplittedOutput() 56 | { 57 | return false; 58 | } 59 | } 60 | } -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/ShardedOperators/ShardedSubscribeOperator.cs: -------------------------------------------------------------------------------- 1 | using CRA.DataProvider; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Diagnostics; 5 | using System.Linq.Expressions; 6 | using System.Reflection; 7 | using System.Text; 8 | using System.Threading; 9 | using System.Threading.Tasks; 10 | 11 | namespace CRA.ClientLibrary.DataProcessing 12 | { 13 | public class ShardedSubscribeOperator : ShardedOperatorBase 14 | { 15 | internal CountdownEvent _deploySubscribeInput; 16 | internal CountdownEvent _deploySubscribeOutput; 17 | 18 | internal CountdownEvent _runSubscribeInput; 19 | internal CountdownEvent _runSubscribeOutput; 20 | 21 | public ShardedSubscribeOperator() : base() 22 | { 23 | } 24 | 25 | internal override void InitializeOperator(int shardId, ShardingInfo shardingInfo) 26 | { 27 | _hasSplittedOutput = HasSplittedOutput(); 28 | 29 | _deploySubscribeInput = new CountdownEvent(1); 30 | _deploySubscribeOutput = new CountdownEvent(1); 31 | 32 | _runSubscribeInput = new CountdownEvent(1); 33 | _runSubscribeOutput = new CountdownEvent(1); 34 | 35 | string[] toEndpoints = GetEndpointNamesForVertex(VertexName.Split('$')[0], _toFromConnections); 36 | var fromTuple = _toFromConnections[new Tuple(VertexName.Split('$')[0], toEndpoints[0])]; 37 | if (!fromTuple.Item4) 38 | AddAsyncInputEndpoint(toEndpoints[0], new ShardedSubscribeInput(this, shardId, shardingInfo.AllShards.Length, toEndpoints[0])); 39 | else 40 | throw new NotImplementedException("Shared secondary endpoints are not supported in subscribe operators!!"); 41 | 42 | string[] fromEndpoints = GetEndpointNamesForVertex(VertexName.Split('$')[0], _fromToConnections); 43 | var toTuple = _fromToConnections[new Tuple(VertexName.Split('$')[0], fromEndpoints[0])]; 44 | if (!toTuple.Item4) 45 | AddAsyncOutputEndpoint(fromEndpoints[0], new ShardedSubscribeOutput(this, shardId, shardingInfo.AllShards.Length, fromEndpoints[0])); 46 | else 47 | throw new NotImplementedException("Shared secondary endpoints are not supported in subscribe operators!!"); 48 | } 49 | 50 | internal override bool HasSplittedOutput() 51 | { 52 | return false; 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/Utilities/OperatorUtils.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.ClientLibrary.DataProcessing 2 | { 3 | public static class OperatorUtils 4 | { 5 | public static string[] PrepareOutputEndpointsIdsForOperator(string toOutputId, OperatorEndpointsDescriptor endpointsDescriptor) 6 | { 7 | if (endpointsDescriptor.ToOutputs.Count != 0) 8 | { 9 | int outputEndpointsTotal = endpointsDescriptor.ToOutputs[toOutputId]; 10 | string[] outputEndpoints = new string[outputEndpointsTotal]; 11 | for (int i = 0; i < outputEndpointsTotal; i++) 12 | outputEndpoints[i] = "OutputTo" + toOutputId + i; 13 | return outputEndpoints; 14 | } 15 | 16 | return null; 17 | } 18 | 19 | public static string PrepareOutputEndpointIdForOperator(string toOutputId, OperatorEndpointsDescriptor endpointsDescriptor) 20 | { 21 | if (endpointsDescriptor.ToOutputs.Count != 0) 22 | return "OutputTo" + toOutputId; 23 | 24 | return null; 25 | } 26 | 27 | public static string[] PrepareInputEndpointsIdsForOperator(string fromInputId, OperatorEndpointsDescriptor endpointsDescriptor) 28 | { 29 | if (endpointsDescriptor.FromInputs.Count != 0) 30 | { 31 | int inputEndpointsTotal = endpointsDescriptor.FromInputs[fromInputId]; 32 | string[] inputEndpoints = new string[inputEndpointsTotal]; 33 | for (int i = 0; i < inputEndpointsTotal; i++) 34 | inputEndpoints[i] = "InputFrom" + fromInputId + i; 35 | return inputEndpoints; 36 | } 37 | 38 | return null; 39 | } 40 | 41 | public static string PrepareInputEndpointIdForOperator(string fromInputId, OperatorEndpointsDescriptor endpointsDescriptor, bool isSecondaryInput) 42 | { 43 | if (!isSecondaryInput) 44 | { 45 | if (endpointsDescriptor.FromInputs.Count != 0) 46 | return "InputFrom" + fromInputId; 47 | } 48 | else 49 | { 50 | if (endpointsDescriptor.SecondaryFromInputs.Count != 0) 51 | return "InputFrom" + fromInputId; 52 | } 53 | 54 | return null; 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProcessing/Utilities/ParallelismUtils.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.ClientLibrary.DataProcessing 2 | { 3 | public static class ParallelismUtils 4 | { 5 | public static int GetCoresCount() 6 | { 7 | int coresCount = 0; 8 | #if !DOTNETCORE 9 | foreach (var item in new System.Management.ManagementObjectSearcher("Select * from Win32_Processor").Get()) 10 | coresCount += int.Parse(item["NumberOfCores"].ToString()); 11 | #endif 12 | return coresCount; 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProvider/DataProviderExtensions.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.DataProvider 2 | { 3 | using System; 4 | using System.Linq.Expressions; 5 | using CRA.ClientLibrary; 6 | 7 | /// 8 | /// Definition for DataProviderExtensions 9 | /// 10 | public static class DataProviderExtensions 11 | { 12 | internal static Func GetVertexCreateAction( 13 | this VertexInfo vertexInfo) 14 | { 15 | var expr = SerializationHelper.Deserialize(vertexInfo.VertexCreateAction); 16 | var actionExpr = AddBox((LambdaExpression)expr); 17 | return actionExpr.Compile(); 18 | } 19 | 20 | private static Expression> AddBox(LambdaExpression expression) 21 | { 22 | Expression converted = Expression.Convert 23 | (expression.Body, typeof(IVertex)); 24 | return Expression.Lambda> 25 | (converted, expression.Parameters); 26 | } 27 | 28 | internal static Func GetShardLocator(this ShardedVertexInfo vertexInfo) 29 | { 30 | var expr = SerializationHelper.Deserialize(vertexInfo.ShardLocator); 31 | var actionExpr = (Expression>)expr; 32 | return actionExpr.Compile(); 33 | } 34 | 35 | internal static Expression> GetShardLocatorExpr(this ShardedVertexInfo vertexInfo) 36 | { 37 | var expr = SerializationHelper.Deserialize(vertexInfo.ShardLocator); 38 | return (Expression>) expr; 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProvider/EndpointInfo.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.DataProvider 2 | { 3 | using System; 4 | using System.Collections.Generic; 5 | 6 | /// 7 | /// Definition for EndpointInfo 8 | /// 9 | public struct EndpointInfo 10 | { 11 | /// 12 | /// The time interval at which workers refresh their membership entry 13 | /// 14 | public static readonly TimeSpan HeartbeatTime = TimeSpan.FromSeconds(10); 15 | 16 | public EndpointInfo( 17 | string vertexName, 18 | string endpointName, 19 | bool isInput, 20 | bool isAsync, 21 | string versionId = null) 22 | { 23 | this.VertexName = vertexName; 24 | this.EndpointName = endpointName; 25 | this.IsInput = isInput; 26 | this.IsAsync = isAsync; 27 | this.VersionId = versionId; 28 | } 29 | 30 | /// 31 | /// Name of the group 32 | /// 33 | public string VertexName { get; } 34 | 35 | /// 36 | /// Endpoint name 37 | /// 38 | public string EndpointName { get; } 39 | 40 | /// 41 | /// Is an input (or output) 42 | /// 43 | public bool IsInput { get; } 44 | 45 | /// 46 | /// Is async (or sync) 47 | /// 48 | public bool IsAsync { get; } 49 | 50 | /// 51 | /// Version Id 52 | /// 53 | public string VersionId { get; } 54 | 55 | public override string ToString() 56 | => string.Format("Vertex '{0}', Endpoint '{1}'", VertexName, EndpointName); 57 | 58 | /// 59 | /// Equals 60 | /// 61 | /// 62 | /// 63 | public override bool Equals(object obj) 64 | { 65 | switch (obj) 66 | { 67 | case EndpointInfo ei: 68 | return this == ei; 69 | } 70 | 71 | return false; 72 | } 73 | 74 | /// 75 | /// GetHashCode 76 | /// 77 | /// 78 | public override int GetHashCode() 79 | { 80 | if (this.VertexName == null || this.EndpointName == null) 81 | { return 0; } 82 | 83 | return this.VertexName.GetHashCode() ^ this.EndpointName.GetHashCode(); 84 | } 85 | 86 | public static bool operator ==(EndpointInfo left, EndpointInfo right) 87 | { 88 | return left.VertexName == right.VertexName && left.EndpointName == right.EndpointName; 89 | } 90 | 91 | public static bool operator !=(EndpointInfo left, EndpointInfo right) 92 | { 93 | return !(left == right); 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProvider/IBlobStorageProvider.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.DataProvider 2 | { 3 | using System; 4 | using System.Collections.Generic; 5 | using System.IO; 6 | using System.Threading.Tasks; 7 | 8 | /// 9 | /// Definition for IBlobStorageProvider 10 | /// 11 | public interface IBlobStorageProvider 12 | { 13 | Task Delete(string pathKey); 14 | Task GetWriteStream(string pathKey); 15 | Task GetReadStream(string pathKey); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProvider/IDataProvider.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.DataProvider 2 | { 3 | using System; 4 | using System.Collections.Generic; 5 | 6 | /// 7 | /// Definition for IDataProvider 8 | /// 9 | public interface IDataProvider 10 | { 11 | IBlobStorageProvider GetBlobStorageProvider(); 12 | IVertexInfoProvider GetVertexInfoProvider(); 13 | IEndpointInfoProvider GetEndpointInfoProvider(); 14 | IShardedVertexInfoProvider GetShardedVertexInfoProvider(); 15 | IVertexConnectionInfoProvider GetVertexConnectionInfoProvider(); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProvider/IEndpointInfoProvider.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.DataProvider 2 | { 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Threading.Tasks; 6 | 7 | /// 8 | /// Definition for IEndpointInfoProvider 9 | /// 10 | public interface IEndpointInfoProvider 11 | { 12 | Task> GetAll(); 13 | 14 | Task DeleteStore(); 15 | Task ExistsEndpoint(string vertexName, string endPoint); 16 | Task AddEndpoint(EndpointInfo endpointInfo); 17 | Task DeleteEndpoint(string vertexName, string endpointName, string versionId = "*"); 18 | Task DeleteEndpoint(EndpointInfo endpointInfo); 19 | Task GetEndpoint(string vertexName, string endpointName); 20 | Task> GetShardedEndpoints(string vertexName, string endpointName); 21 | Task> GetEndpoints(string vertexName); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProvider/IShardedVertexInfoProvider.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.DataProvider 2 | { 3 | using System.Collections.Generic; 4 | using System.Threading.Tasks; 5 | 6 | /// 7 | /// Definition for IShardedShardedVertexInfoProvider 8 | /// 9 | public interface IShardedVertexInfoProvider 10 | { 11 | Task> GetAll(); 12 | Task CountAll(); 13 | Task GetEntryForVertex(string vertexName, string epochId); 14 | Task> GetEntriesForVertex(string vertexName); 15 | Task GetLatestEntryForVertex(string vertexName); 16 | Task Delete(); 17 | Task Delete(ShardedVertexInfo entry); 18 | Task Insert(ShardedVertexInfo shardedVertexInfo); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProvider/IVertexConnectionInfoProvider.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.DataProvider 2 | { 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Threading.Tasks; 6 | 7 | /// 8 | /// Definition for IVertexConnectionInfoProvider 9 | /// 10 | public interface IVertexConnectionInfoProvider 11 | { 12 | Task> GetAll(); 13 | 14 | Task CountAll(); 15 | 16 | Task> GetAllConnectionsFromVertex(string fromVertex); 17 | Task DeleteStore(); 18 | Task> GetAllConnectionsToVertex(string toVertex); 19 | Task Add(VertexConnectionInfo vertexConnectionInfo); 20 | Task ContainsRow(VertexConnectionInfo entity); 21 | Task Get(string fromVertex, string fromOutput, string toConnection, string toInput); 22 | Task Delete(VertexConnectionInfo vci); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProvider/IVertexInfoProvider.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.DataProvider 2 | { 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Threading.Tasks; 6 | 7 | /// 8 | /// Definition for IVertexInfoProvider 9 | /// 10 | public interface IVertexInfoProvider 11 | { 12 | Task> GetAll(); 13 | Task CountAll(); 14 | Task GetInstanceFromAddress(string address, int port); 15 | Task GetRowForInstance(string instanceName); 16 | Task> GetAllRowsForInstance(string instanceName); 17 | Task GetRowForInstanceVertex(string instanceName, string vertexName); 18 | Task DeleteStore(); 19 | Task GetRowForVertexDefinition(string vertexDefinition); 20 | Task GetRowForActiveVertex(string vertexName); 21 | Task> GetVertices(string instanceName); 22 | Task> GetRowsForShardedInstanceVertex( 23 | string instanceName, 24 | string vertexName); 25 | 26 | Task> GetRowsForShardedVertex(string vertexName); 27 | Task> GetRowsForVertex(string vertexName); 28 | Task ContainsRow(VertexInfo entity); 29 | Task ContainsInstance(string instanceName); 30 | Task> GetVertexNames(); 31 | Task> GetVertexDefinitions(); 32 | Task> GetInstanceNames(); 33 | Task DeleteVertexInfo(string instanceName, string vertexName); 34 | Task DeleteVertexInfo(VertexInfo vertexInfo); 35 | Task InsertOrReplace(VertexInfo newInfo); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProvider/ShardedVertexInfo.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.DataProvider 2 | { 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Linq.Expressions; 6 | using CRA.ClientLibrary; 7 | 8 | /// 9 | /// Definition for ShardedVertexInfo 10 | /// 11 | public struct ShardedVertexInfo 12 | { 13 | public ShardedVertexInfo( 14 | string vertexName, 15 | string epochId, 16 | string allInstances, 17 | string allShards, 18 | string addedShards, 19 | string removedShards, 20 | string shardLocator, 21 | string versionId = null) 22 | { 23 | this.VertexName = vertexName; 24 | this.EpochId = epochId; 25 | this.AllInstances = allInstances; 26 | this.AllShards = allShards; 27 | this.AddedShards = addedShards; 28 | this.RemovedShards = removedShards; 29 | this.ShardLocator = shardLocator; 30 | this.VersionId = versionId; 31 | } 32 | 33 | public static ShardedVertexInfo Create( 34 | string vertexName, 35 | string epochId, 36 | List allInstances, 37 | List allShards, 38 | List addedShards, 39 | List removedShards, 40 | Expression> shardLocator) 41 | { 42 | var strAllInstances = string.Join(";", allInstances); 43 | var strAllShards = string.Join(";", allShards); 44 | var strAddedShards = string.Join(";", addedShards); 45 | var strRemovedShards = string.Join(";", removedShards); 46 | var strShardLocator = ""; 47 | 48 | if (shardLocator != null) 49 | { 50 | var closureEliminator = new ClosureEliminator(); 51 | Expression vertexedUserLambdaExpression = closureEliminator.Visit(shardLocator); 52 | strShardLocator = SerializationHelper.Serialize(vertexedUserLambdaExpression); 53 | } 54 | 55 | return new ShardedVertexInfo( 56 | vertexName: vertexName, 57 | epochId: epochId, 58 | allInstances: strAllInstances, 59 | allShards: strAllShards, 60 | addedShards: strAddedShards, 61 | removedShards: strRemovedShards, 62 | shardLocator: strShardLocator); 63 | } 64 | 65 | 66 | public string VertexName { get; } 67 | public string EpochId { get; } 68 | public string AllInstances { get; } 69 | public string AllShards { get; } 70 | public string AddedShards { get; } 71 | public string RemovedShards { get; } 72 | public string ShardLocator { get; } 73 | public string VersionId { get; } 74 | 75 | public override string ToString() 76 | => string.Format("Vertex '{0}', EpochId '{1}'", this.VertexName, this.EpochId); 77 | 78 | public override bool Equals(object obj) 79 | { 80 | switch (obj) 81 | { 82 | case ShardedVertexInfo other: 83 | return other.VertexName == this.VertexName && other.EpochId == this.EpochId; 84 | } 85 | 86 | return false; 87 | } 88 | 89 | public override int GetHashCode() 90 | => this.VertexName.GetHashCode() ^ this.EpochId.GetHashCode(); 91 | 92 | public static bool operator ==(ShardedVertexInfo left, ShardedVertexInfo right) 93 | => left.VertexName == right.VertexName && left.EpochId == right.EpochId; 94 | 95 | public static bool operator !=(ShardedVertexInfo left, ShardedVertexInfo right) 96 | => !(left == right); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/DataProvider/VertexConnectionInfo.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.DataProvider 2 | { 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Globalization; 6 | 7 | /// 8 | /// Definition for VertexConnectionInfo 9 | /// 10 | public struct VertexConnectionInfo 11 | { 12 | public VertexConnectionInfo( 13 | string fromVertex, 14 | string fromEndpoint, 15 | string toVertex, 16 | string toEndpoint, 17 | string versionId = null) 18 | { 19 | FromVertex = fromVertex; 20 | FromEndpoint = fromEndpoint; 21 | ToVertex = toVertex; 22 | ToEndpoint = toEndpoint; 23 | VersionId = versionId; 24 | } 25 | 26 | public string FromVertex { get; } 27 | 28 | public string FromEndpoint { get; } 29 | 30 | public string ToVertex { get; } 31 | 32 | public string ToEndpoint { get; } 33 | 34 | public string VersionId { get; } 35 | 36 | public override string ToString() 37 | { 38 | return string.Format( 39 | CultureInfo.CurrentCulture, 40 | "FromVertex '{0}', FromEndpoint '{1}', ToVertex '{2}', ToEndpoint '{3}'", 41 | FromVertex, 42 | FromEndpoint, 43 | ToVertex, 44 | ToEndpoint); 45 | } 46 | 47 | public override bool Equals(object obj) 48 | { 49 | switch (obj) 50 | { 51 | case VertexConnectionInfo other: 52 | return this == other; 53 | } 54 | 55 | return false; 56 | } 57 | 58 | public override int GetHashCode() 59 | { 60 | return 61 | this.FromVertex.GetHashCode() 62 | ^ (this.FromEndpoint.GetHashCode() << 1) 63 | ^ (this.ToVertex.GetHashCode()) << 2 64 | ^ (this.ToEndpoint.GetHashCode() << 3); 65 | } 66 | 67 | public static bool operator ==( 68 | VertexConnectionInfo left, 69 | VertexConnectionInfo right) 70 | { 71 | return left.FromEndpoint == right.FromEndpoint 72 | && left.FromVertex == right.FromVertex 73 | && left.ToEndpoint == right.ToEndpoint 74 | && left.ToVertex == right.ToVertex; 75 | } 76 | 77 | public static bool operator !=( 78 | VertexConnectionInfo left, 79 | VertexConnectionInfo right) 80 | { 81 | return !(left == right); 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Definitions/ConnectionInfo.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Runtime.Serialization; 3 | 4 | namespace CRA.ClientLibrary 5 | { 6 | /// 7 | /// Describes a connection between two vertex/endpoint pairs 8 | /// 9 | [Serializable, DataContract] 10 | public class ConnectionInfo 11 | { 12 | /// 13 | /// Connection is from this vertex 14 | /// 15 | [DataMember] 16 | public string FromVertex { get; set; } 17 | /// 18 | /// Connection is from this output endpoint 19 | /// 20 | 21 | [DataMember] 22 | 23 | public string FromEndpoint { get; set; } 24 | /// 25 | /// Connection is to this vertex 26 | /// 27 | 28 | [DataMember] 29 | public string ToVertex { get; set; } 30 | 31 | /// 32 | /// Connection is to this input endpoint 33 | /// 34 | [DataMember] 35 | public string ToEndpoint { get; set; } 36 | 37 | 38 | /// 39 | /// Constructor 40 | /// 41 | /// 42 | /// 43 | /// 44 | /// 45 | public ConnectionInfo(string fromVertex, string fromEndpoint, string toVertex, string toEndpoint) 46 | { 47 | this.FromVertex = fromVertex; 48 | this.FromEndpoint = fromEndpoint; 49 | this.ToVertex = toVertex; 50 | this.ToEndpoint = toEndpoint; 51 | } 52 | 53 | /// 54 | /// String representation of a CRA conection 55 | /// 56 | /// 57 | public override string ToString() 58 | { 59 | return new { FromVertex = FromVertex, FromEndpoint = FromEndpoint, ToVertex = ToVertex, ToEndpoint = ToEndpoint }.ToString(); 60 | } 61 | 62 | /// 63 | /// Check if two instances are equal 64 | /// 65 | /// 66 | /// 67 | public override bool Equals(object obj) 68 | { 69 | var otherConnectionInfo = (ConnectionInfo)obj; 70 | if (otherConnectionInfo == null) return false; 71 | 72 | return 73 | (FromVertex == otherConnectionInfo.FromVertex) && 74 | (ToVertex == otherConnectionInfo.ToVertex) && 75 | (FromEndpoint == otherConnectionInfo.FromEndpoint) && 76 | (ToEndpoint == otherConnectionInfo.ToEndpoint); 77 | } 78 | 79 | public override int GetHashCode() 80 | { 81 | return FromVertex.GetHashCode() ^ FromEndpoint.GetHashCode() ^ ToVertex.GetHashCode() ^ ToEndpoint.GetHashCode(); 82 | } 83 | } 84 | 85 | [Serializable, DataContract] 86 | public class ConnectionInfoWithLocality : ConnectionInfo 87 | { 88 | [DataMember] 89 | public bool IsOnSameCRAInstance { get; set; } 90 | 91 | [DataMember] 92 | public bool IsSecondary { get; set; } 93 | 94 | public ConnectionInfoWithLocality(string fromVertex, string fromEndpoint, string toVertex, string toEndpoint, bool isOnSameCRAInstance, bool isSecondary = false) : base(fromVertex, fromEndpoint, toVertex, toEndpoint) 95 | { 96 | IsOnSameCRAInstance = isOnSameCRAInstance; 97 | IsSecondary = isSecondary; 98 | } 99 | } 100 | 101 | } 102 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Definitions/ConnectionInitiator.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.ClientLibrary 2 | { 3 | /// 4 | /// Direction of data flow 5 | /// 6 | public enum ConnectionInitiator 7 | { 8 | /// 9 | /// Initiate connection from "from" vertex 10 | /// 11 | FromSide, 12 | 13 | /// 14 | /// Initiate connection from "to" vertex 15 | /// 16 | ToSide 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Definitions/CoralTaskMessageType.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.ClientLibrary 2 | { 3 | internal enum CRATaskMessageType 4 | { 5 | LOAD_VERTEX, 6 | CONNECT_VERTEX_INITIATOR, 7 | CONNECT_VERTEX_RECEIVER, 8 | CONNECT_VERTEX_INITIATOR_REVERSE, 9 | CONNECT_VERTEX_RECEIVER_REVERSE, 10 | PING, 11 | READY, 12 | RELEASE 13 | }; 14 | 15 | } 16 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Definitions/ErrorCodes.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.ClientLibrary 2 | { 3 | /// 4 | /// Error codes for CRA method calls 5 | /// 6 | public enum CRAErrorCode : int 7 | { 8 | /// 9 | /// Success 10 | /// 11 | Success, 12 | /// 13 | /// Vertex not found 14 | /// 15 | VertexNotFound, 16 | /// 17 | /// ActiveVertex not found 18 | /// 19 | ActiveVertexNotFound, 20 | /// 21 | /// Vertex output endpoint not found 22 | /// 23 | VertexOutputNotFound, 24 | /// 25 | /// Vertex input endpoint not found 26 | /// 27 | VertexInputNotFound, 28 | /// 29 | /// Vertex already exists 30 | /// 31 | VertexAlreadyExists, 32 | /// 33 | /// Recovering 34 | /// 35 | ServerRecovering, 36 | /// 37 | /// Race condition adding connection 38 | /// 39 | ConnectionAdditionRace, 40 | /// 41 | /// Vertex endpoint (input or output) not found 42 | /// 43 | VertexEndpointNotFound, 44 | /// 45 | /// Failed to establish a connection 46 | /// 47 | ConnectionEstablishFailed, 48 | /// 49 | /// Failed to match endpoints between fromVertices and toVertices 50 | /// 51 | VerticesEndpointsNotMatched 52 | }; 53 | } 54 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Definitions/ObjectWrapper.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.ClientLibrary 2 | { 3 | internal struct ObjectWrapper 4 | { 5 | public string type; 6 | public string data; 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Definitions/ShardingInfo.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq.Expressions; 4 | using System.Runtime.Serialization; 5 | 6 | namespace CRA.ClientLibrary 7 | { 8 | /// 9 | /// Describes the sharding configuration 10 | /// 11 | public class ShardingInfo 12 | { 13 | /// 14 | /// List of all shards 15 | /// 16 | public int[] AllShards { get; set; } 17 | 18 | /// 19 | /// Shards added most recently 20 | /// 21 | public int[] AddedShards { get; set; } 22 | 23 | /// 24 | /// Shards removed most recently 25 | /// 26 | public int[] RemovedShards { get; set; } 27 | 28 | /// 29 | /// Map of key to offset in AllShards array 30 | /// 31 | public Expression> ShardLocator { get; set; } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Definitions/StreamConnectionPool.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Concurrent; 2 | using System.Threading; 3 | 4 | namespace CRA.ClientLibrary 5 | { 6 | /// 7 | /// A class represents a pool of stream connections in a CRA client 8 | /// 9 | internal class StreamConnectionPool 10 | { 11 | const int DEFAULT_CONNECTION_POOL_SIZE = 1000; 12 | 13 | private ConcurrentQueue _queue; 14 | private long _createdObjects; 15 | private int _size; 16 | private bool _isLimitedPool; 17 | 18 | /// 19 | /// Constructor 20 | /// 21 | /// 22 | public StreamConnectionPool(bool isLimitedPool = false) : this(0, isLimitedPool) 23 | { 24 | } 25 | 26 | /// 27 | /// Constructor 28 | /// 29 | /// 30 | /// 31 | public StreamConnectionPool(int inputSize, bool isLimitedPool = false) 32 | { 33 | _queue = new ConcurrentQueue(); 34 | 35 | if (inputSize <= 0) 36 | { 37 | _size = DEFAULT_CONNECTION_POOL_SIZE; 38 | } 39 | else 40 | { 41 | _size = inputSize; 42 | } 43 | 44 | _isLimitedPool = isLimitedPool; 45 | Interlocked.Exchange(ref _createdObjects, 0); 46 | } 47 | 48 | /// 49 | /// Adds/Returns a stream connection to the pool of connections 50 | /// 51 | /// 52 | /// 53 | public bool Return(StreamConnection item) 54 | { 55 | if (_isLimitedPool && (_createdObjects > _size)) 56 | { 57 | return false; 58 | } 59 | 60 | _queue.Enqueue(item); 61 | Interlocked.Increment(ref _createdObjects); 62 | 63 | return true; 64 | } 65 | 66 | /// 67 | /// Retrieves a stream connection from the pool of connections 68 | /// 69 | /// 70 | /// 71 | public bool Get(out StreamConnection result) 72 | { 73 | if (!_queue.TryDequeue(out result)) 74 | { 75 | return false; 76 | } 77 | 78 | Interlocked.Decrement(ref _createdObjects); 79 | return true; 80 | } 81 | 82 | /// 83 | /// Frees the pool of stream connections 84 | /// 85 | /// 86 | public void Free(bool reset = false) 87 | { 88 | StreamConnection result; 89 | while (_queue.TryDequeue(out result)) 90 | { 91 | result.Dispose(); 92 | Interlocked.Decrement(ref _createdObjects); 93 | } 94 | if (reset) 95 | { 96 | Interlocked.Exchange(ref _createdObjects, 0); 97 | } 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics.CodeAnalysis; 2 | using System.Reflection; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | //[assembly: AssemblyTitle("CRA.ClientLibrary")] 9 | [assembly: AssemblyDescription("")] 10 | //[assembly: AssemblyConfiguration("")] 11 | //[assembly: AssemblyCompany("")] 12 | //[assembly: AssemblyProduct("CRA.ClientLibrary")] 13 | [assembly: AssemblyCopyright("Copyright © 2017")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("ef23eb6a-e329-496d-9b7a-8cad66ea4e3a")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | //[assembly: AssemblyVersion("1.0.0.0")] 36 | //[assembly: AssemblyFileVersion("1.0.0.0")] 37 | 38 | [module: SuppressMessage("Microsoft.Design", "CA1006:DoNotNestGenericTypesInMemberSignatures", Scope = "member", Target = "CRA.ClientLibrary.VertexTable.#.ctor(System.String,System.String,System.String,System.String,System.Int32,System.Linq.Expressions.Expression`1>,System.Object)")] 39 | [module: SuppressMessage("Microsoft.Design", "CA1006:DoNotNestGenericTypesInMemberSignatures", Scope = "member", Target = "CRA.ClientLibrary.CRAClientLibrary.#DefineVertex(System.String,System.Linq.Expressions.Expression`1>)")] 40 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Security/DummySecureStreamConnectionDescriptor.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | 4 | namespace CRA.ClientLibrary 5 | { 6 | /// 7 | /// Interface to define callbacks for securing TCP connections 8 | /// 9 | public class DummySecureStreamConnectionDescriptor : ISecureStreamConnectionDescriptor 10 | { 11 | public DummySecureStreamConnectionDescriptor() 12 | { 13 | 14 | } 15 | 16 | public Stream CreateSecureClient(Stream stream, string serverName) 17 | { 18 | return stream; 19 | } 20 | 21 | public Stream CreateSecureServer(Stream stream) 22 | { 23 | return stream; 24 | } 25 | 26 | public void TeardownSecureClient(Stream stream) 27 | { 28 | } 29 | 30 | public void TeardownSecureServer(Stream stream) 31 | { 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Security/ISecureStreamConnectionDescriptor.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | 4 | namespace CRA.ClientLibrary 5 | { 6 | /// 7 | /// Interface to define callbacks for securing TCP connections 8 | /// 9 | public interface ISecureStreamConnectionDescriptor 10 | { 11 | Stream CreateSecureClient(Stream stream, string serverName); 12 | Stream CreateSecureServer(Stream stream); 13 | void TeardownSecureClient(Stream stream); 14 | void TeardownSecureServer(Stream stream); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Security/SampleSecureStreamConnectionDescriptor.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | using System.IO; 4 | using System.Net.Security; 5 | using System.Security.Authentication; 6 | using System.Security.Cryptography.X509Certificates; 7 | 8 | namespace CRA.ClientLibrary 9 | { 10 | /// 11 | /// Interface to define callbacks for securing TCP connections 12 | /// 13 | public class SampleSecureStreamConnectionDescriptor : ISecureStreamConnectionDescriptor 14 | { 15 | public SampleSecureStreamConnectionDescriptor() 16 | { 17 | } 18 | 19 | public static bool ValidateCertificate( 20 | object sender, 21 | X509Certificate certificate, 22 | X509Chain chain, 23 | SslPolicyErrors sslPolicyErrors) 24 | { 25 | if (sslPolicyErrors == SslPolicyErrors.None) 26 | return true; 27 | 28 | Trace.TraceError("Certificate error: {0}", sslPolicyErrors); 29 | 30 | // Do not allow this client to communicate with unauthenticated servers. 31 | return false; 32 | } 33 | 34 | public Stream CreateSecureClient(Stream stream, string serverName) 35 | { 36 | SslStream sslStream = new SslStream(stream, true, 37 | new RemoteCertificateValidationCallback(ValidateCertificate), null); 38 | 39 | try 40 | { 41 | sslStream.AuthenticateAsClient(serverName); 42 | } 43 | catch (AuthenticationException e) 44 | { 45 | Trace.TraceError("Exception: {0}", e.Message); 46 | if (e.InnerException != null) 47 | { 48 | Trace.TraceError("Inner exception: {0}", e.InnerException.Message); 49 | } 50 | Trace.TraceError("Authentication failed - closing the connection."); 51 | sslStream.Close(); 52 | return null; 53 | } 54 | return sslStream; 55 | } 56 | 57 | public Stream CreateSecureServer(Stream stream) 58 | { 59 | var _certificate = new X509Certificate2(); 60 | 61 | SslStream sslStream = new SslStream(stream, true, 62 | new RemoteCertificateValidationCallback(ValidateCertificate), null); 63 | 64 | try 65 | { 66 | sslStream.AuthenticateAsServer(_certificate, false, SslProtocols.Tls, true); 67 | } 68 | catch (AuthenticationException e) 69 | { 70 | Trace.TraceError("Exception: {0}", e.Message); 71 | if (e.InnerException != null) 72 | { 73 | Trace.TraceError("Inner exception: {0}", e.InnerException.Message); 74 | } 75 | Trace.TraceError("Authentication failed - closing the connection."); 76 | sslStream.Close(); 77 | return null; 78 | } 79 | return sslStream; 80 | } 81 | 82 | public void TeardownSecureClient(Stream stream) 83 | { 84 | stream.Close(); 85 | } 86 | 87 | public void TeardownSecureServer(Stream stream) 88 | { 89 | stream.Close(); 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Tables/ConnectionTableManager.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using CRA.DataProvider; 3 | using System.Threading.Tasks; 4 | 5 | namespace CRA.ClientLibrary 6 | { 7 | /// 8 | /// An assignment of one machine to a group 9 | /// 10 | public class ConnectionTableManager 11 | { 12 | private IVertexConnectionInfoProvider _connectionProvider; 13 | 14 | internal ConnectionTableManager(IDataProvider dataProvider) 15 | { 16 | _connectionProvider = dataProvider.GetVertexConnectionInfoProvider(); 17 | } 18 | 19 | internal Task DeleteTable() 20 | => _connectionProvider.DeleteStore(); 21 | 22 | 23 | internal Task AddConnection(string fromVertex, string fromOutput, string toConnection, string toInput) 24 | => _connectionProvider.Add( 25 | new VertexConnectionInfo( 26 | fromVertex: fromVertex, 27 | fromEndpoint: fromOutput, 28 | toVertex: toConnection, 29 | toEndpoint: toInput)); 30 | 31 | internal async Task DeleteConnection(string fromVertex, string fromOutput, string toConnection, string toInput) 32 | { 33 | var vci = await _connectionProvider.Get(fromVertex, fromOutput, toConnection, toInput); 34 | 35 | if (vci != null) 36 | { await _connectionProvider.Delete(vci.Value); } 37 | } 38 | 39 | internal Task> GetConnectionsFromVertex(string vertexName) 40 | => _connectionProvider.GetAllConnectionsFromVertex(vertexName); 41 | 42 | internal Task> GetConnectionsToVertex(string vertexName) 43 | => _connectionProvider.GetAllConnectionsToVertex(vertexName); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Tables/EndpointTableManager.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using CRA.DataProvider; 5 | using System.Threading.Tasks; 6 | using System.Diagnostics; 7 | 8 | namespace CRA.ClientLibrary 9 | { 10 | /// 11 | /// An assignment of one machine to a group 12 | /// 13 | public class EndpointTableManager 14 | { 15 | private IEndpointInfoProvider _endpointDataProvider; 16 | 17 | internal EndpointTableManager(IDataProvider dataProvider) 18 | { 19 | _endpointDataProvider = dataProvider.GetEndpointInfoProvider(); 20 | } 21 | 22 | internal Task DeleteTable() 23 | => _endpointDataProvider.DeleteStore(); 24 | 25 | internal Task ExistsEndpoint(string vertexName, string endPoint) 26 | => _endpointDataProvider.ExistsEndpoint(vertexName, endPoint); 27 | 28 | internal Task AddEndpoint(string vertexName, string endpointName, bool isInput, bool isAsync) 29 | => _endpointDataProvider.AddEndpoint( 30 | new EndpointInfo( 31 | vertexName: vertexName, 32 | endpointName: endpointName, 33 | isInput: isInput, 34 | isAsync: isAsync)); 35 | 36 | internal Task DeleteEndpoint(string vertexName, string endpointName) 37 | => _endpointDataProvider.DeleteEndpoint(vertexName, endpointName); 38 | 39 | internal async Task RemoveEndpoint(string vertexName, string endpointName) 40 | { 41 | var endpointInfo = await _endpointDataProvider.GetEndpoint(vertexName, endpointName); 42 | 43 | if (endpointInfo != null) 44 | { await _endpointDataProvider.DeleteEndpoint(vertexName, endpointName, endpointInfo.Value.VersionId); } 45 | else 46 | { Trace.TraceError("RemoveEndpoint: Could not retrieve the entity."); } 47 | } 48 | 49 | internal async Task RemoveShardedEndpoints(string vertexName, string endpointName) 50 | { 51 | var endpointInfos = await _endpointDataProvider.GetShardedEndpoints(vertexName, endpointName); 52 | 53 | var tasks = new List(); 54 | foreach(var endpoint in endpointInfos) 55 | { tasks.Add(_endpointDataProvider.DeleteEndpoint(endpoint)); } 56 | 57 | await Task.WhenAll(tasks); 58 | } 59 | 60 | internal async Task> GetInputEndpoints(string vertexName) 61 | => (await _endpointDataProvider.GetEndpoints(vertexName)) 62 | .Where(e => e.IsInput) 63 | .Select(e => e.EndpointName) 64 | .ToList(); 65 | 66 | internal async Task> GetOutputEndpoints(string vertexName) 67 | => (await _endpointDataProvider.GetEndpoints(vertexName)) 68 | .Where(e => !e.IsInput) 69 | .Select(e => e.EndpointName) 70 | .ToList(); 71 | 72 | internal async Task DeleteContents() 73 | { 74 | var pendingTasks = new List(); 75 | foreach(var ei in (await _endpointDataProvider.GetAll())) 76 | { 77 | pendingTasks.Add( 78 | _endpointDataProvider.DeleteEndpoint( 79 | ei.VertexName, 80 | ei.EndpointName, 81 | ei.VersionId)); 82 | } 83 | 84 | await Task.WhenAll(pendingTasks); 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Utilities/AssemblyResolver.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading; 3 | using System.Reflection; 4 | using System.Collections.Concurrent; 5 | using System.Collections.Generic; 6 | using System.Diagnostics.Contracts; 7 | 8 | #pragma warning disable 420 9 | 10 | namespace CRA.ClientLibrary 11 | { 12 | internal static class AssemblyResolver 13 | { 14 | private static readonly ConcurrentDictionary assemblies = 15 | new ConcurrentDictionary(); 16 | 17 | private static volatile int handlerRegistered; 18 | 19 | public static IEnumerable RegisteredAssemblies 20 | { 21 | get 22 | { 23 | return assemblies.Keys; 24 | } 25 | } 26 | 27 | public static bool ContainsAssembly(string name) 28 | { 29 | Contract.Requires(name != null); 30 | return assemblies.ContainsKey(name); 31 | } 32 | 33 | public static byte[] GetAssemblyBytes(string name) 34 | { 35 | Contract.Requires(name != null); 36 | return assemblies[name]; 37 | } 38 | 39 | public static void Register(string name, byte[] assembly) 40 | { 41 | Contract.Requires(name != null); 42 | Contract.Requires(assembly != null); 43 | assemblies.TryAdd(name, assembly); 44 | if (handlerRegistered == 0 && 45 | Interlocked.CompareExchange(ref handlerRegistered, 1, 0) == 0) 46 | { 47 | AppDomain.CurrentDomain.AssemblyResolve += Resolver; 48 | } 49 | } 50 | 51 | private static Assembly Resolver(object sender, ResolveEventArgs arguments) 52 | { 53 | Assembly assembly = null; 54 | byte[] assemblyBytes; 55 | if (assemblies.TryGetValue(arguments.Name, out assemblyBytes)) 56 | { 57 | if (IsAssemblyLoaded(arguments.Name, out assembly)) 58 | return assembly; 59 | else 60 | return Assembly.Load(assemblyBytes); 61 | } 62 | return assembly; 63 | } 64 | 65 | private static bool IsAssemblyLoaded(string fullName, out Assembly lAssembly) 66 | { 67 | Assembly[] assemblies = AppDomain.CurrentDomain.GetAssemblies(); 68 | 69 | foreach (Assembly assembly in assemblies) 70 | { 71 | if (assembly.FullName == fullName) 72 | { 73 | lAssembly = assembly; 74 | return true; 75 | } 76 | } 77 | 78 | lAssembly = null; 79 | return false; 80 | } 81 | 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Utilities/AssemblyResolverClient.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Collections.Generic; 3 | using System.Diagnostics.Contracts; 4 | 5 | namespace CRA.ClientLibrary 6 | { 7 | internal static class AssemblyResolverClient 8 | { 9 | internal struct ApplicationAssembly 10 | { 11 | public string Name; 12 | 13 | public string FileName; 14 | } 15 | 16 | private static void GetApplicationAssemblies( 17 | AssemblyName current, List assemblies, HashSet exclude) 18 | { 19 | Contract.Requires(current != null); 20 | Contract.Requires(assemblies != null); 21 | Contract.Requires(exclude != null); 22 | 23 | // If this assembly is in the "exclude" set, then done. 24 | var name = current.FullName; 25 | if (exclude.Contains(name)) 26 | { 27 | return; 28 | } 29 | 30 | // Add to "exclude" set so this assembly isn't re-added if it is referenced additional times. 31 | exclude.Add(name); 32 | 33 | // Recursively add any assemblies that "current" references. 34 | var assembly = Assembly.Load(current); 35 | foreach (var referenced in assembly.GetReferencedAssemblies()) 36 | { 37 | GetApplicationAssemblies(referenced, assemblies, exclude); 38 | } 39 | 40 | // Lastly, now that all dependant (referenced) assemblies are added, add the "current" assembly. 41 | assemblies.Add(new ApplicationAssembly { Name = name, FileName = assembly.Location }); 42 | } 43 | 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Utilities/ExpressionUtils.cs: -------------------------------------------------------------------------------- 1 | using Aqua.TypeSystem; 2 | using System.Linq.Expressions; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System; 6 | using RLinq = Remote.Linq.Expressions; 7 | 8 | namespace CRA.ClientLibrary 9 | { 10 | internal class ClosureEliminator : ExpressionVisitor 11 | { 12 | protected override Expression VisitMember(MemberExpression node) 13 | { 14 | if ((node.Expression != null) && (node.Expression.NodeType == ExpressionType.Constant)) 15 | { 16 | object target = ((ConstantExpression)node.Expression).Value, value; 17 | switch (node.Member.MemberType) 18 | { 19 | case System.Reflection.MemberTypes.Property: 20 | value = ((System.Reflection.PropertyInfo)node.Member).GetValue(target, null); 21 | break; 22 | case System.Reflection.MemberTypes.Field: 23 | value = ((System.Reflection.FieldInfo)node.Member).GetValue(target); 24 | break; 25 | default: 26 | value = target = null; 27 | break; 28 | } 29 | if (target != null) 30 | { 31 | if (value.GetType().IsSubclassOf(typeof(Expression))) 32 | { 33 | return this.Visit(value as Expression); 34 | } 35 | else 36 | { 37 | return Expression.Constant(value, node.Type); 38 | } 39 | } 40 | } 41 | 42 | return base.VisitMember(node); 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Utilities/NetworkUtils.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Net.NetworkInformation; 6 | using System.Net; 7 | using System.Threading.Tasks; 8 | using System.Net.Sockets; 9 | 10 | namespace CRA.ClientLibrary 11 | { 12 | public static class NetworkUtils 13 | { 14 | internal static async Task ConnectAsync(this TcpClient tcpClient, string host, int port, int timeoutMs) 15 | { 16 | var cancelTask = Task.Delay(timeoutMs); 17 | var connectTask = tcpClient.ConnectAsync(host, port); 18 | await await Task.WhenAny(connectTask, cancelTask); 19 | if (cancelTask.IsCompleted) 20 | { 21 | throw new Exception("Timed out"); 22 | } 23 | } 24 | 25 | /* Copyright 2018 mniak (https://gist.github.com/jrusbatch/4211535)*/ 26 | public static int GetAvailablePort(int startingPort = 5000) 27 | { 28 | var properties = IPGlobalProperties.GetIPGlobalProperties(); 29 | 30 | //getting active connections 31 | var tcpConnectionPorts = properties.GetActiveTcpConnections() 32 | .Where(n => n.LocalEndPoint.Port >= startingPort) 33 | .Select(n => n.LocalEndPoint.Port); 34 | 35 | //getting active tcp listners - WCF service listening in tcp 36 | var tcpListenerPorts = properties.GetActiveTcpListeners() 37 | .Where(n => n.Port >= startingPort) 38 | .Select(n => n.Port); 39 | 40 | //getting active udp listeners 41 | var udpListenerPorts = properties.GetActiveUdpListeners() 42 | .Where(n => n.Port >= startingPort) 43 | .Select(n => n.Port); 44 | 45 | var port = Enumerable.Range(startingPort, ushort.MaxValue) 46 | .Where(i => !tcpConnectionPorts.Contains(i)) 47 | .Where(i => !tcpListenerPorts.Contains(i)) 48 | .Where(i => !udpListenerPorts.Contains(i)) 49 | .FirstOrDefault(); 50 | 51 | return port; 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Utilities/SerializationHelper.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Reflection; 3 | using System.Linq.Expressions; 4 | using Newtonsoft.Json; 5 | using Remote.Linq; 6 | using Remote.Linq.ExpressionVisitors; 7 | 8 | namespace CRA.ClientLibrary 9 | { 10 | internal class SerializationHelper 11 | { 12 | private SerializationHelper() { } 13 | 14 | private static readonly JsonSerializerSettings _serializerSettings = 15 | new JsonSerializerSettings().ConfigureRemoteLinq(); 16 | 17 | /// 18 | /// Serializes a LINQ expression. 19 | /// 20 | /// The expression. 21 | /// The serialized expression. 22 | internal static string Serialize(Expression expression) 23 | { 24 | var toSerialize = expression.ToRemoteLinqExpression() 25 | .ReplaceGenericQueryArgumentsByNonGenericArguments(); 26 | return JsonConvert.SerializeObject(toSerialize, _serializerSettings); 27 | } 28 | 29 | /// 30 | /// Deserializes a LINQ expression. 31 | /// 32 | /// The serialized expression. 33 | /// The expression. 34 | internal static Expression Deserialize(string expression) 35 | { 36 | var deserialized = JsonConvert.DeserializeObject( 37 | expression, _serializerSettings); 38 | var ret = deserialized.ReplaceNonGenericQueryArgumentsByGenericArguments().ToLinqExpression(); 39 | return ret; 40 | } 41 | 42 | internal static string SerializeObject(object obj) 43 | { 44 | if (obj == null) 45 | { 46 | return JsonConvert.SerializeObject(obj, _serializerSettings); 47 | } 48 | var tmp = new ObjectWrapper 49 | { 50 | type = obj.GetType().AssemblyQualifiedName, 51 | data = JsonConvert.SerializeObject(obj, _serializerSettings) 52 | }; 53 | 54 | return JsonConvert.SerializeObject(tmp, typeof(ObjectWrapper), _serializerSettings); 55 | } 56 | 57 | internal static object DeserializeObject(string obj) 58 | { 59 | if (obj == "null") return null; 60 | var ow = JsonConvert.DeserializeObject(obj, _serializerSettings); 61 | return JsonConvert.DeserializeObject(ow.data, Type.GetType(ow.type), _serializerSettings); 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Vertices/IAsyncVertexInputEndpoint.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Threading; 4 | using System.Threading.Tasks; 5 | 6 | namespace CRA.ClientLibrary 7 | { 8 | /// 9 | /// Interface for async input endpoints in CRA 10 | /// 11 | public interface IAsyncVertexInputEndpoint : IDisposable 12 | { 13 | /// 14 | /// Async version of FromStream 15 | /// 16 | /// 17 | /// 18 | /// 19 | /// 20 | /// 21 | Task FromStreamAsync(Stream stream, string otherVertex, string otherEndpoint, CancellationToken token); 22 | } 23 | 24 | /// 25 | /// Interface for async sharded input endpoints in CRA 26 | /// 27 | public interface IAsyncShardedVertexInputEndpoint : IAsyncVertexInputEndpoint 28 | { 29 | /// 30 | /// Async version of FromStream 31 | /// 32 | /// 33 | /// 34 | /// 35 | /// 36 | /// 37 | Task FromStreamAsync(Stream stream, string otherVertex, int otherShardId, string otherEndpoint, CancellationToken token); 38 | 39 | /// 40 | /// Provide info on sharding for the "other" sharded vertex that 41 | /// this endpoint reads from. 42 | /// 43 | /// 44 | /// 45 | void UpdateShardingInfo(string otherVertex, ShardingInfo shardingInfo); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Vertices/IAsyncVertexOutputEndpoint.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Threading; 4 | using System.Threading.Tasks; 5 | 6 | namespace CRA.ClientLibrary 7 | { 8 | /// 9 | /// Interface for async output endpoints in CRA 10 | /// 11 | public interface IAsyncVertexOutputEndpoint : IDisposable 12 | { 13 | /// 14 | /// Async version of ToStream 15 | /// 16 | /// 17 | /// 18 | /// 19 | /// 20 | /// 21 | Task ToStreamAsync(Stream stream, string otherVertex, string otherEndpoint, CancellationToken token); 22 | } 23 | 24 | /// 25 | /// Interface for async output endpoints in CRA 26 | /// 27 | public interface IAsyncShardedVertexOutputEndpoint : IAsyncVertexOutputEndpoint 28 | { 29 | /// 30 | /// Async version of ToStream 31 | /// 32 | /// 33 | /// 34 | /// 35 | /// 36 | /// 37 | Task ToStreamAsync(Stream stream, string otherVertex, int otherShardId, string otherEndpoint, CancellationToken token); 38 | 39 | /// 40 | /// Provide info on sharding for the "other" sharded vertex that 41 | /// this endpoint writes to. 42 | /// 43 | /// 44 | /// 45 | void UpdateShardingInfo(string otherVertex, ShardingInfo shardingInfo); 46 | } 47 | 48 | /// 49 | /// Interface for async output endpoints in CRA with fusable output 50 | /// 51 | public interface IAsyncFusableVertexOutputEndpoint : IAsyncVertexOutputEndpoint 52 | { 53 | /// 54 | /// Can this output endpoint fuse with the specified input endpoint? 55 | /// 56 | /// 57 | /// 58 | /// 59 | /// 60 | bool CanFuseWith(IAsyncVertexInputEndpoint endpoint, string otherVertex, string otherEndpoint); 61 | 62 | /// 63 | /// Async version of ToInput 64 | /// 65 | /// 66 | /// 67 | /// 68 | Task ToInputAsync(IAsyncVertexInputEndpoint endpoint, string otherVertex, string otherEndpoint, CancellationToken token); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Vertices/IVertex.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Threading.Tasks; 4 | 5 | namespace CRA.ClientLibrary 6 | { 7 | 8 | /// 9 | /// User provided notion of a running vertex 10 | /// 11 | public interface IVertex : IDisposable 12 | { 13 | /// 14 | /// Ingress points for a vertex; these are observers 15 | /// 16 | ConcurrentDictionary InputEndpoints { get; } 17 | 18 | /// 19 | /// Egress points for a vertex; these are observables 20 | /// 21 | ConcurrentDictionary OutputEndpoints { get; } 22 | 23 | /// 24 | /// Ingress points for a vertex; these are observers 25 | /// 26 | ConcurrentDictionary AsyncInputEndpoints { get; } 27 | 28 | /// 29 | /// Egress points for a vertex; these are observables 30 | /// 31 | ConcurrentDictionary AsyncOutputEndpoints { get; } 32 | 33 | 34 | /// 35 | /// Callback that vertex will invoke when a new input is added 36 | /// 37 | /// 38 | void OnAddInputEndpoint(Action addInputCallback); 39 | 40 | /// 41 | /// Callback that vertex will invoke when a new output is added 42 | /// 43 | /// 44 | void OnAddAsyncOutputEndpoint(Action addOutputCallback); 45 | 46 | /// 47 | /// Callback that vertex will invoke when a new input is added 48 | /// 49 | /// 50 | void OnAddAsyncInputEndpoint(Action addInputCallback); 51 | 52 | /// 53 | /// Callback that vertex will invoke when a new output is added 54 | /// 55 | /// 56 | void OnAddOutputEndpoint(Action addOutputCallback); 57 | 58 | /// 59 | /// Callback when vertex is disposed 60 | /// 61 | void OnDispose(Action disposeCallback); 62 | 63 | /// 64 | /// Gets an instance of the CRA Client Library that the vertex 65 | /// can use to communicate with the CRA runtime. 66 | /// 67 | /// Instance of CRA Client Library 68 | CRAClientLibrary ClientLibrary { get; } 69 | 70 | /// 71 | /// Initialize vertex asynchronously with specified params 72 | /// 73 | /// 74 | /// 75 | Task InitializeAsync(object vertexParameter); 76 | } 77 | 78 | /// 79 | /// User provided notion of a running sharded vertex 80 | /// 81 | public interface IShardedVertex : IVertex 82 | { 83 | /// 84 | /// Initialize sharded vertex with shard-id and specified params 85 | /// 86 | /// 87 | /// 88 | Task InitializeAsync(int shardId, ShardingInfo shardingInfo, object vertexParameter); 89 | 90 | /// 91 | /// Update sharding info 92 | /// 93 | /// 94 | void UpdateShardingInfo(ShardingInfo shardingInfo); 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Vertices/IVertexInputEndpoint.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Threading; 4 | 5 | namespace CRA.ClientLibrary 6 | { 7 | /// 8 | /// Interface for input endpoints in CRA 9 | /// 10 | public interface IVertexInputEndpoint : IDisposable 11 | { 12 | /// 13 | /// Call to provide a stream for input to read from 14 | /// 15 | /// 16 | /// 17 | /// 18 | /// 19 | void FromStream(Stream stream, string otherVertex, string otherEndpoint, CancellationToken token); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Vertices/IVertexOutputEndpoint.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Threading; 4 | 5 | namespace CRA.ClientLibrary 6 | { 7 | /// 8 | /// Interface for output endpoints in CRA 9 | /// 10 | public interface IVertexOutputEndpoint : IDisposable 11 | { 12 | /// 13 | /// Call to provide a stream for output to write to 14 | /// 15 | /// 16 | /// 17 | /// 18 | /// 19 | void ToStream(Stream stream, string otherVertex, string otherEndpoint, CancellationToken token); 20 | } 21 | 22 | /// 23 | /// Interface for output endpoints in CRA with fusable output 24 | /// 25 | public interface IFusableVertexOutputEndpoint : IVertexOutputEndpoint 26 | { 27 | /// 28 | /// Can this output endpoint fuse with the specified input endpoint? 29 | /// 30 | /// 31 | /// 32 | /// 33 | /// 34 | bool CanFuseWith(IVertexInputEndpoint endpoint, string otherVertex, string otherEndpoint); 35 | 36 | /// 37 | /// Call to provide an input endpoint for output to write to 38 | /// 39 | /// 40 | /// 41 | void ToInput(IVertexInputEndpoint endpoint, string otherVertex, string otherEndpoint, CancellationToken token); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Vertices/VertexInputEndpointBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.IO; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | 7 | namespace CRA.ClientLibrary 8 | { 9 | /// 10 | /// Base class for Vertex abstraction 11 | /// 12 | public abstract class AsyncShardedVertexInputEndpointBase : IAsyncShardedVertexInputEndpoint 13 | { 14 | public abstract void Dispose(); 15 | public virtual Task FromStreamAsync(Stream stream, string otherVertex, int otherShardId, string otherEndpoint, CancellationToken token) 16 | { 17 | return FromStreamAsync(stream, otherVertex + "$" + otherShardId, otherEndpoint, token); 18 | } 19 | 20 | public virtual Task FromStreamAsync(Stream stream, string otherVertex, string otherEndpoint, CancellationToken token) 21 | { 22 | return Task.CompletedTask; 23 | } 24 | 25 | public virtual void UpdateShardingInfo(string otherVertex, ShardingInfo shardingInfo) 26 | { 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/Vertices/VertexOutputEndpointBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.IO; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | 7 | namespace CRA.ClientLibrary 8 | { 9 | /// 10 | /// Base class for Vertex abstraction 11 | /// 12 | public abstract class AsyncShardedVertexOutputEndpointBase : IAsyncShardedVertexOutputEndpoint 13 | { 14 | public abstract void Dispose(); 15 | public virtual Task ToStreamAsync(Stream stream, string otherVertex, int otherShardId, string otherEndpoint, CancellationToken token) 16 | { 17 | return ToStreamAsync(stream, otherVertex + "$" + otherShardId, otherEndpoint, token); 18 | } 19 | public virtual Task ToStreamAsync(Stream stream, string otherVertex, string otherEndpoint, CancellationToken token) 20 | { 21 | return Task.CompletedTask; 22 | } 23 | 24 | public virtual void UpdateShardingInfo(string otherVertex, ShardingInfo shardingInfo) 25 | { 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/CRA.ClientLibrary/app.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /src/CRA.DataProvider.File/CRA.DataProvider.File.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp2.0;net46;netstandard2.0 5 | true 6 | false 7 | ../CRA.snk 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /src/CRA.DataProvider.File/FileBlobProvider.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.DataProvider.File 2 | { 3 | using System.IO; 4 | using System.Threading.Tasks; 5 | using CRA.DataProvider; 6 | 7 | /// 8 | /// Definition for FileBlobProvider 9 | /// 10 | public class FileBlobProvider 11 | : IBlobStorageProvider 12 | { 13 | private readonly string _blobDirectory; 14 | 15 | public FileBlobProvider(string blobDirectory) 16 | { _blobDirectory = FileUtils.GetDirectory(blobDirectory); } 17 | 18 | public Task Delete(string pathKey) 19 | { 20 | File.Delete(Path.Combine(_blobDirectory, pathKey)); 21 | return Task.FromResult(true); 22 | } 23 | 24 | public Task GetReadStream(string pathKey) 25 | => Task.FromResult( 26 | FileUtils.GetReadStream( 27 | Path.Combine( 28 | _blobDirectory, pathKey))); 29 | 30 | public Task GetWriteStream(string pathKey) 31 | => Task.FromResult( 32 | FileUtils.GetReadWriteStream( 33 | Path.Combine(_blobDirectory, pathKey))); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/CRA.DataProvider.File/FileDataProvider.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.DataProvider.File 2 | { 3 | using System; 4 | using System.IO; 5 | using CRA.DataProvider; 6 | 7 | /// 8 | /// Definition for FileProviderImpl 9 | /// 10 | public class FileDataProvider : IDataProvider 11 | { 12 | private readonly string _directoryPath; 13 | 14 | public FileDataProvider() 15 | { _directoryPath = FileUtils.GetDirectory(GetDefaultDirectory()); } 16 | 17 | public FileDataProvider(string directoryPath) 18 | { _directoryPath = FileUtils.GetDirectory(directoryPath); } 19 | 20 | public IBlobStorageProvider GetBlobStorageProvider() 21 | => new FileBlobProvider( 22 | GetDirectory("Blobs")); 23 | 24 | public IEndpointInfoProvider GetEndpointInfoProvider() 25 | => new FileEndpointProvider( 26 | Path.Combine(GetDirectory("Data"), "endpoints.json")); 27 | 28 | public IShardedVertexInfoProvider GetShardedVertexInfoProvider() 29 | => new FileShardedVertexProvider( 30 | Path.Combine(GetDirectory("Data"), "sharded_vertexes.json")); 31 | 32 | public IVertexConnectionInfoProvider GetVertexConnectionInfoProvider() 33 | => new FileVertexConnectionProvider( 34 | Path.Combine(GetDirectory("Data"), "vertex_connections.json")); 35 | 36 | public IVertexInfoProvider GetVertexInfoProvider() 37 | => new FileVertexProvider( 38 | Path.Combine(GetDirectory("Data"), "vertex.json")); 39 | 40 | public static string GetDefaultDirectory() 41 | => Path.Combine( 42 | Environment.GetEnvironmentVariable("TEMP"), 43 | "CRA"); 44 | 45 | private string GetDirectory(string subPath) 46 | { 47 | string subDirectoryPath = Path.Combine(_directoryPath, subPath); 48 | return FileUtils.GetDirectory(subPath); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/CRA.DataProvider.File/FileShardedVertexProvider.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.DataProvider.File 2 | { 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Linq; 6 | using System.Threading.Tasks; 7 | using CRA.DataProvider; 8 | 9 | /// 10 | /// Definition for FileShardedVertexProvider 11 | /// 12 | public class FileShardedVertexProvider 13 | : IShardedVertexInfoProvider 14 | { 15 | private readonly string _fileName; 16 | 17 | public FileShardedVertexProvider(string fileName) 18 | { _fileName = fileName; } 19 | 20 | public Task CountAll() 21 | => FileUtils.CountAll(_fileName); 22 | 23 | public Task Delete() 24 | { 25 | System.IO.File.Delete(_fileName); 26 | return Task.FromResult(true); 27 | } 28 | 29 | public Task Delete(ShardedVertexInfo entry) 30 | => FileUtils.DeleteItem( 31 | _fileName, 32 | entry, 33 | MatchVersion); 34 | 35 | public async Task> GetAll() 36 | => await FileUtils.GetAll( 37 | _fileName, 38 | (e) => true); 39 | 40 | public async Task> GetEntriesForVertex(string vertexName) 41 | => await FileUtils.GetAll( 42 | _fileName, 43 | (e) => e.VertexName == vertexName); 44 | 45 | public async Task GetEntryForVertex(string vertexName, string epochId) 46 | => (await FileUtils.Get( 47 | _fileName, 48 | (e) => e.VertexName == vertexName && e.EpochId == epochId)).Value; 49 | 50 | public async Task GetLatestEntryForVertex(string vertexName) 51 | => (await FileUtils.GetAll( 52 | _fileName, 53 | (e) => e.VertexName == vertexName)) 54 | .OrderByDescending(e => e.EpochId) 55 | .First(); 56 | 57 | public Task Insert(ShardedVertexInfo shardedVertexInfo) 58 | => FileUtils.InsertOrUpdate( 59 | _fileName, 60 | shardedVertexInfo, 61 | MatchVersion, 62 | UpdateVerion); 63 | 64 | private ShardedVertexInfo UpdateVerion(ShardedVertexInfo svInfo) 65 | => new ShardedVertexInfo( 66 | vertexName: svInfo.VertexName, 67 | epochId: svInfo.EpochId, 68 | addedShards: svInfo.AddedShards, 69 | allShards: svInfo.AllShards, 70 | allInstances: svInfo.AllInstances, 71 | removedShards: svInfo.RemovedShards, 72 | shardLocator: svInfo.ShardLocator, 73 | versionId: FileUtils.GetUpdateVersionId(svInfo.VersionId)); 74 | 75 | private (bool matched, bool versionMatched) MatchVersion(ShardedVertexInfo dbItem, ShardedVertexInfo newItem) 76 | { 77 | if (dbItem.VertexName == newItem.VertexName 78 | && dbItem.EpochId == newItem.EpochId) 79 | { 80 | if (newItem.VersionId == null 81 | || newItem.VersionId == "*" 82 | || newItem.VersionId == dbItem.VersionId) 83 | { 84 | return (true, true); 85 | } 86 | 87 | return (true, false); 88 | } 89 | 90 | return (false, false); 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/CRA.DataProvider.File/FileVertexConnectionProvider.cs: -------------------------------------------------------------------------------- 1 | namespace CRA.DataProvider.File 2 | { 3 | using System.Collections.Generic; 4 | using System.Threading.Tasks; 5 | using CRA.DataProvider; 6 | 7 | /// 8 | /// Definition for FileVertexConnectionProvider 9 | /// 10 | public class FileVertexConnectionProvider : IVertexConnectionInfoProvider 11 | { 12 | private readonly string _fileName; 13 | 14 | public FileVertexConnectionProvider(string fileName) 15 | { _fileName = fileName; } 16 | 17 | public Task Add(VertexConnectionInfo vertexConnectionInfo) 18 | => FileUtils.InsertOrUpdate( 19 | _fileName, 20 | vertexConnectionInfo, 21 | MatchVersion, 22 | UpdateVerion); 23 | 24 | public Task ContainsRow(VertexConnectionInfo entity) 25 | => FileUtils.Exists( 26 | _fileName, 27 | entity, 28 | MatchVersion); 29 | 30 | public Task CountAll() 31 | => FileUtils.CountAll( 32 | _fileName); 33 | 34 | public Task Delete(VertexConnectionInfo vci) 35 | => FileUtils.DeleteItem( 36 | _fileName, 37 | vci, 38 | MatchVersion); 39 | 40 | public Task DeleteStore() 41 | { 42 | System.IO.File.Delete(_fileName); 43 | return Task.FromResult(true); 44 | } 45 | 46 | public Task Get(string fromVertex, string fromOutput, string toConnection, string toInput) 47 | => FileUtils.Get( 48 | _fileName, 49 | (elem) => elem.FromVertex == fromVertex 50 | && elem.FromEndpoint == fromOutput 51 | && elem.ToVertex == toConnection 52 | && elem.ToEndpoint == toInput); 53 | 54 | public async Task> GetAll() 55 | => await FileUtils.GetAll( 56 | _fileName, 57 | (e) => true); 58 | 59 | public async Task> GetAllConnectionsFromVertex(string fromVertex) 60 | => await FileUtils.GetAll( 61 | _fileName, 62 | (e) => e.FromVertex == fromVertex); 63 | 64 | public async Task> GetAllConnectionsToVertex(string toVertex) 65 | => await FileUtils.GetAll( 66 | _fileName, 67 | (e) => e.ToVertex == toVertex); 68 | 69 | private VertexConnectionInfo UpdateVerion(VertexConnectionInfo vcInfo) 70 | => new VertexConnectionInfo( 71 | fromVertex: vcInfo.FromVertex, 72 | fromEndpoint: vcInfo.FromEndpoint, 73 | toVertex: vcInfo.ToVertex, 74 | toEndpoint: vcInfo.ToEndpoint, 75 | versionId: FileUtils.GetUpdateVersionId(vcInfo.VersionId)); 76 | 77 | private (bool matched, bool versionMatched) MatchVersion(VertexConnectionInfo dbItem, VertexConnectionInfo newItem) 78 | { 79 | if (dbItem == newItem) 80 | { 81 | if (newItem.VersionId == null 82 | || newItem.VersionId == "*" 83 | || newItem.VersionId == dbItem.VersionId) 84 | { 85 | return (true, true); 86 | } 87 | 88 | return (true, false); 89 | } 90 | 91 | return (false, false); 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/CRA.Worker/.dockerignore: -------------------------------------------------------------------------------- 1 | privatesettings.config 2 | -------------------------------------------------------------------------------- /src/CRA.Worker/App.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /src/CRA.Worker/CRA.Worker.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | Exe 4 | netcoreapp2.0;net46 5 | AnyCPU 6 | true 7 | CRA.Worker 8 | true 9 | true 10 | false 11 | ../CRA.snk 12 | 13 | 14 | 15 | $(DefineConstants);DOTNETCORE 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | true 29 | true 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /src/CRA.Worker/CRA.nuspec: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Microsoft.CRA 5 | $version$ 6 | Common Runtime for Applications (CRA) 7 | Microsoft 8 | Microsoft 9 | https://github.com/microsoft/CRA 10 | MIT 11 | true 12 | 13 | Common Runtime for Applications (CRA) is a software layer (library) that makes it 14 | easy to create and deploy distributed dataflow-style applications on top of resource 15 | managers such as Kubernetes, YARN, and stand-alone cluster execution. Currently, we 16 | support stand-alone execution (just deploy an .exe on every machine in your cluster) 17 | as well as execution in a Kubernetes/Docker environment. 18 | 19 | See the project website at https://github.com/microsoft/CRA for more details 20 | © Microsoft Corporation. All rights reserved. 21 | common runtime applications 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /src/CRA.Worker/Dockerfile-debug: -------------------------------------------------------------------------------- 1 | FROM microsoft/windowsservercore 2 | 3 | # deploy the Hello World app 4 | COPY ./bin/Debug /root/CRA.Worker/bin/Debug 5 | 6 | # entrypoint for CRA worker 7 | ENTRYPOINT ["/root/CRA.Worker/bin/Debug/CRA.Worker.exe"] 8 | -------------------------------------------------------------------------------- /src/CRA.Worker/Dockerfile-release: -------------------------------------------------------------------------------- 1 | FROM microsoft/windowsservercore 2 | 3 | # deploy the Hello World app 4 | COPY ./bin/Release /root/CRA.Worker/bin/Release 5 | 6 | # entrypoint for CRA worker 7 | ENTRYPOINT ["/root/CRA.Worker/bin/Release/CRA.Worker.exe"] 8 | -------------------------------------------------------------------------------- /src/CRA.Worker/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | //[assembly: AssemblyTitle("CRA.Worker")] 9 | [assembly: AssemblyDescription("")] 10 | //[assembly: AssemblyConfiguration("")] 11 | //[assembly: AssemblyCompany("")] 12 | //[assembly: AssemblyProduct("CRA.Worker")] 13 | [assembly: AssemblyCopyright("Copyright © 2017")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("85331f0d-b724-45d7-9bb2-ac703e73de3b")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | //[assembly: AssemblyVersion("1.0.0.0")] 36 | //[assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /src/CRA.Worker/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "profiles": { 3 | "CRA.Worker": { 4 | "commandName": "Project", 5 | "commandLineArgs": "crainst02 12000 127.0.0.1" 6 | } 7 | } 8 | } -------------------------------------------------------------------------------- /src/CRA.Worker/cra-two-instances-debug.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Kubernetes Deployment File for Common Runtime for Applications (CRA) workers 3 | # 4 | # Before deployment, you need to specify your storage connection string to Kubernetes as follows: 5 | # kubectl create configmap cra-config --from-literal=azure_storage_conn_string=your_key 6 | # 7 | # where your_key is your Azure storage connection string. 8 | # 9 | 10 | kind : Deployment 11 | apiVersion : extensions/v1beta1 12 | metadata : 13 | name : crainst01 14 | namespace : default 15 | labels : 16 | app : crainst01 17 | spec : 18 | replicas : 1 19 | template : 20 | metadata : 21 | name : crainst01 22 | labels: 23 | app: crainst01 24 | spec: 25 | containers: 26 | - name : crainst01 27 | image : mydockercontainer/cra-0.1-debug:latest 28 | ports: 29 | - containerPort: 1500 30 | env: 31 | - name: AZURE_STORAGE_CONN_STRING 32 | valueFrom: 33 | configMapKeyRef: 34 | name: cra-config # Name of the ConfigMap containing the storage connection string 35 | key: azure_storage_conn_string # Key in the configmap associated with the storage connection string 36 | args : ["crainst01", "1500"] # CRA instance name: crainst01, exposed on port 1500 37 | --- 38 | kind : Deployment 39 | apiVersion : extensions/v1beta1 40 | metadata : 41 | name : crainst02 42 | namespace : default 43 | labels : 44 | app : crainst02 45 | spec : 46 | replicas : 1 47 | template : 48 | metadata : 49 | name : crainst02 50 | labels: 51 | app: crainst02 52 | spec: 53 | containers: 54 | - name : crainst02 55 | image : mydockercontainer/cra-0.1-debug:latest 56 | ports: 57 | - containerPort: 1501 58 | env: 59 | - name: AZURE_STORAGE_CONN_STRING 60 | valueFrom: 61 | configMapKeyRef: 62 | name: cra-config # Name of the ConfigMap containing the storage connection string 63 | key: azure_storage_conn_string # Key in the configmap associated with the storage connection string 64 | args : ["crainst02", "1501"] # CRA instance name: crainst02, exposed on port 1501 65 | -------------------------------------------------------------------------------- /src/CRA.Worker/cra-two-instances-release.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Kubernetes Deployment File for Common Runtime for Applications (CRA) workers 3 | # 4 | # Before deployment, you need to specify your storage connection string to Kubernetes as follows: 5 | # kubectl create configmap cra-config --from-literal=azure_storage_conn_string=your_key 6 | # 7 | # where your_key is your Azure storage connection string. 8 | # 9 | 10 | kind : Deployment 11 | apiVersion : extensions/v1beta1 12 | metadata : 13 | name : crainst01 14 | namespace : default 15 | labels : 16 | app : crainst01 17 | spec : 18 | replicas : 1 19 | template : 20 | metadata : 21 | name : crainst01 22 | labels: 23 | app: crainst01 24 | spec: 25 | containers: 26 | - name : crainst01 27 | image : mydockeraccount/cra-0.1-release 28 | ports: 29 | - containerPort: 1500 30 | env: 31 | - name: AZURE_STORAGE_CONN_STRING 32 | valueFrom: 33 | configMapKeyRef: 34 | name: cra-config # Name of the ConfigMap containing the storage connection string 35 | key: azure_storage_conn_string # Key in the configmap associated with the storage connection string 36 | args : ["crainst01", "1500"] # CRA instance name: crainst01, exposed on port 1500 37 | --- 38 | kind : Deployment 39 | apiVersion : extensions/v1beta1 40 | metadata : 41 | name : crainst02 42 | namespace : default 43 | labels : 44 | app : crainst02 45 | spec : 46 | replicas : 1 47 | template : 48 | metadata : 49 | name : crainst02 50 | labels: 51 | app: crainst02 52 | spec: 53 | containers: 54 | - name : crainst02 55 | image : mydockeraccount/cra-0.1-release 56 | ports: 57 | - containerPort: 1501 58 | env: 59 | - name: AZURE_STORAGE_CONN_STRING 60 | valueFrom: 61 | configMapKeyRef: 62 | name: cra-config # Name of the ConfigMap containing the storage connection string 63 | key: azure_storage_conn_string # Key in the configmap associated with the storage connection string 64 | args : ["crainst02", "1501"] # CRA instance name: crainst02, exposed on port 1501 65 | -------------------------------------------------------------------------------- /src/CRA.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/CRA/0e9fab904bf1db055eaeaac5746be7534c86e8fd/src/CRA.snk -------------------------------------------------------------------------------- /src/CustomDictionary.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | CRA 5 | 6 | 7 | -------------------------------------------------------------------------------- /src/Samples/BandwidthTest/App.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /src/Samples/BandwidthTest/BandwidthTest.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | Exe 4 | netcoreapp2.0;net46 5 | AnyCPU 6 | true 7 | BandwidthTest 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | $(DefineConstants);DOTNETCORE 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /src/Samples/BandwidthTest/BandwidthTestVertex.cs: -------------------------------------------------------------------------------- 1 | using CRA.ClientLibrary; 2 | using System.Threading.Tasks; 3 | 4 | namespace BandwidthTest 5 | { 6 | public class BandwidthTestVertex : VertexBase 7 | { 8 | private int _chunkSize; 9 | 10 | public BandwidthTestVertex() : base() 11 | { 12 | } 13 | 14 | public override Task InitializeAsync(object vertexParameter) 15 | { 16 | _chunkSize = (int)vertexParameter; 17 | AddAsyncInputEndpoint("input1", new MyAsyncInput(this, _chunkSize)); 18 | AddAsyncOutputEndpoint("output1", new MyAsyncOutput(this, _chunkSize)); 19 | AddAsyncInputEndpoint("input2", new MyAsyncInput(this, _chunkSize)); 20 | AddAsyncOutputEndpoint("output2", new MyAsyncOutput(this, _chunkSize)); 21 | AddAsyncInputEndpoint("input3", new MyAsyncInput(this, _chunkSize)); 22 | AddAsyncOutputEndpoint("output3", new MyAsyncOutput(this, _chunkSize)); 23 | AddAsyncInputEndpoint("input4", new MyAsyncInput(this, _chunkSize)); 24 | AddAsyncOutputEndpoint("output4", new MyAsyncOutput(this, _chunkSize)); 25 | AddAsyncInputEndpoint("input5", new MyAsyncInput(this, _chunkSize)); 26 | AddAsyncOutputEndpoint("output5", new MyAsyncOutput(this, _chunkSize)); 27 | AddAsyncInputEndpoint("input6", new MyAsyncInput(this, _chunkSize)); 28 | AddAsyncOutputEndpoint("output6", new MyAsyncOutput(this, _chunkSize)); 29 | AddAsyncInputEndpoint("input7", new MyAsyncInput(this, _chunkSize)); 30 | AddAsyncOutputEndpoint("output7", new MyAsyncOutput(this, _chunkSize)); 31 | AddAsyncInputEndpoint("input8", new MyAsyncInput(this, _chunkSize)); 32 | AddAsyncOutputEndpoint("output8", new MyAsyncOutput(this, _chunkSize)); 33 | AddAsyncInputEndpoint("input9", new MyAsyncInput(this, _chunkSize)); 34 | AddAsyncOutputEndpoint("output9", new MyAsyncOutput(this, _chunkSize)); 35 | AddAsyncInputEndpoint("input10", new MyAsyncInput(this, _chunkSize)); 36 | AddAsyncOutputEndpoint("output10", new MyAsyncOutput(this, _chunkSize)); 37 | AddAsyncInputEndpoint("input11", new MyAsyncInput(this, _chunkSize)); 38 | AddAsyncOutputEndpoint("output11", new MyAsyncOutput(this, _chunkSize)); 39 | AddAsyncInputEndpoint("input12", new MyAsyncInput(this, _chunkSize)); 40 | AddAsyncOutputEndpoint("output12", new MyAsyncOutput(this, _chunkSize)); 41 | AddAsyncInputEndpoint("input13", new MyAsyncInput(this, _chunkSize)); 42 | AddAsyncOutputEndpoint("output13", new MyAsyncOutput(this, _chunkSize)); 43 | AddAsyncInputEndpoint("input14", new MyAsyncInput(this, _chunkSize)); 44 | AddAsyncOutputEndpoint("output14", new MyAsyncOutput(this, _chunkSize)); 45 | AddAsyncInputEndpoint("input15", new MyAsyncInput(this, _chunkSize)); 46 | AddAsyncOutputEndpoint("output15", new MyAsyncOutput(this, _chunkSize)); 47 | AddAsyncInputEndpoint("input16", new MyAsyncInput(this, _chunkSize)); 48 | AddAsyncOutputEndpoint("output16", new MyAsyncOutput(this, _chunkSize)); 49 | 50 | return base.InitializeAsync(vertexParameter); 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/Samples/BandwidthTest/MyAsyncInput.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using System.IO; 4 | using System.Threading; 5 | using CRA.ClientLibrary; 6 | using System.Diagnostics; 7 | 8 | namespace BandwidthTest 9 | { 10 | public class MyAsyncInput : IAsyncVertexInputEndpoint 11 | { 12 | bool _running = true; 13 | IVertex _vertex; 14 | int _chunkSize; 15 | byte[] _dataset; 16 | 17 | public MyAsyncInput(IVertex vertex, int chunkSize) 18 | { 19 | _vertex = vertex; 20 | _chunkSize = chunkSize; 21 | _dataset = new byte[chunkSize]; 22 | } 23 | 24 | public void Dispose() 25 | { 26 | Dispose(true); 27 | GC.SuppressFinalize(this); 28 | } 29 | 30 | protected virtual void Dispose(bool disposing) 31 | { 32 | if (disposing) 33 | { 34 | Console.WriteLine("Disposing MyInput"); 35 | _running = false; 36 | } 37 | } 38 | 39 | public async Task FromStreamAsync( 40 | Stream stream, 41 | string otherVertex, 42 | string otherEndpoint, 43 | CancellationToken token) 44 | { 45 | Console.WriteLine("Receiving data from vertex: " + otherVertex + ", endpoint: " + otherEndpoint); 46 | int bwCheckPeriod = 1000 * 1024 * 1024 / _chunkSize; 47 | 48 | Stopwatch sw = new Stopwatch(); 49 | sw.Start(); 50 | 51 | for (int i = 0; i < int.MaxValue; i++) 52 | { 53 | await stream.ReadAllRequiredBytesAsync(_dataset, 0, _chunkSize); 54 | if ((i > 0) && (i % bwCheckPeriod == 0)) 55 | { 56 | Console.WriteLine("Incoming bandwidth from vertex {0}, endpoint {1} = {2} MBps", otherVertex, otherEndpoint, _chunkSize * (double)bwCheckPeriod / (1000 * (double)sw.ElapsedMilliseconds)); 57 | sw.Restart(); 58 | } 59 | 60 | if (!_running) break; 61 | } 62 | } 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /src/Samples/BandwidthTest/MyAsyncOutput.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using System.IO; 4 | using System.Threading; 5 | using CRA.ClientLibrary; 6 | 7 | namespace BandwidthTest 8 | { 9 | public class MyAsyncOutput : IAsyncVertexOutputEndpoint 10 | { 11 | bool _running = true; 12 | IVertex _vertex; 13 | byte[] _dataset; 14 | int _chunkSize; 15 | 16 | public MyAsyncOutput(IVertex vertex, int chunkSize) 17 | { 18 | _vertex = vertex; 19 | _chunkSize = chunkSize; 20 | _dataset = new byte[chunkSize]; 21 | Random r = new Random(); 22 | r.NextBytes(_dataset); 23 | } 24 | 25 | public void Dispose() 26 | { 27 | Dispose(true); 28 | GC.SuppressFinalize(this); 29 | } 30 | 31 | protected virtual void Dispose(bool disposing) 32 | { 33 | if (disposing) 34 | { 35 | Console.WriteLine("Disposing MyOutput"); 36 | _running = false; 37 | } 38 | } 39 | 40 | public async Task ToStreamAsync(Stream stream, string otherVertex, string otherEndpoint, CancellationToken token) 41 | { 42 | Console.WriteLine("Sending data to vertex: " + otherVertex + ", endpoint: " + otherEndpoint); 43 | for (int i = 0; i < int.MaxValue; i += 1) 44 | { 45 | await stream.WriteAsync(_dataset, 0, _chunkSize); 46 | if (!_running) break; 47 | } 48 | } 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/Samples/BandwidthTest/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using CRA.ClientLibrary; 3 | using CRA.DataProvider.File; 4 | 5 | namespace BandwidthTest 6 | { 7 | class Program 8 | { 9 | static void Main(string[] args) 10 | { 11 | var client = new CRAClientLibrary(new FileDataProvider()); 12 | 13 | int chunkSize = 10*1024*1024; 14 | 15 | Console.ReadLine(); 16 | 17 | client.DefineVertexAsync("bandwidthtestvertex", () => new BandwidthTestVertex()).Wait(); 18 | client.InstantiateVertexAsync("crainst01", "bwvertex1", "bandwidthtestvertex", chunkSize).Wait(); 19 | client.InstantiateVertexAsync("crainst02", "bwvertex2", "bandwidthtestvertex", chunkSize).Wait(); 20 | // client.InstantiateVertexAsync("crainst03", "bwvertex3", "bandwidthtestvertex", chunkSize).Wait(); 21 | 22 | client.ConnectAsync("bwvertex1", "output1", "bwvertex2", "input1").Wait(); 23 | client.ConnectAsync("bwvertex1", "output2", "bwvertex2", "input2").Wait(); 24 | client.ConnectAsync("bwvertex1", "output3", "bwvertex2", "input3").Wait(); 25 | client.ConnectAsync("bwvertex1", "output4", "bwvertex2", "input4").Wait(); 26 | client.ConnectAsync("bwvertex1", "output5", "bwvertex2", "input5").Wait(); 27 | client.ConnectAsync("bwvertex1", "output6", "bwvertex2", "input6").Wait(); 28 | client.ConnectAsync("bwvertex1", "output7", "bwvertex2", "input7").Wait(); 29 | client.ConnectAsync("bwvertex1", "output8", "bwvertex2", "input8").Wait(); 30 | client.ConnectAsync("bwvertex2", "output9", "bwvertex1", "input9").Wait(); 31 | client.ConnectAsync("bwvertex2", "output10", "bwvertex1", "input10").Wait(); 32 | client.ConnectAsync("bwvertex2", "output11", "bwvertex1", "input11").Wait(); 33 | client.ConnectAsync("bwvertex2", "output12", "bwvertex1", "input12").Wait(); 34 | client.ConnectAsync("bwvertex2", "output13", "bwvertex1", "input13").Wait(); 35 | client.ConnectAsync("bwvertex2", "output14", "bwvertex1", "input14").Wait(); 36 | client.ConnectAsync("bwvertex2", "output15", "bwvertex1", "input15").Wait(); 37 | client.ConnectAsync("bwvertex2", "output16", "bwvertex1", "input16").Wait(); 38 | 39 | Console.ReadLine(); 40 | client.ResetAsync().Wait(); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/Samples/BandwidthTest/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | //[assembly: AssemblyTitle("BandwidthTest")] 9 | [assembly: AssemblyDescription("")] 10 | //[assembly: AssemblyConfiguration("")] 11 | //[assembly: AssemblyCompany("")] 12 | //[assembly: AssemblyProduct("BandwidthTest")] 13 | [assembly: AssemblyCopyright("Copyright © 2017")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("3c700a3b-9958-4470-9639-87964d20b11a")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | //[assembly: AssemblyVersion("1.0.0.0")] 36 | //[assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /src/Samples/ConnectionPair/App.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /src/Samples/ConnectionPair/ConnectionPair.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | Exe 4 | netcoreapp2.0;net46 5 | AnyCPU 6 | true 7 | ConnectionPair 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | $(DefineConstants);DOTNETCORE 19 | 20 | 21 | 22 | true 23 | true 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /src/Samples/ConnectionPair/ConnectionPairVertex.cs: -------------------------------------------------------------------------------- 1 | using CRA.ClientLibrary; 2 | using System.Threading.Tasks; 3 | 4 | namespace ConnectionPair 5 | { 6 | public class ConnectionPairVertex : VertexBase 7 | { 8 | public ConnectionPairVertex() : base() 9 | { 10 | } 11 | 12 | public override Task InitializeAsync(object vertexParameter) 13 | { 14 | AddAsyncInputEndpoint("input", new MyAsyncInput(this)); 15 | AddAsyncOutputEndpoint("output", new MyAsyncOutput(this)); 16 | return base.InitializeAsync(vertexParameter); 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/Samples/ConnectionPair/MyAsyncInput.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using System.IO; 4 | using System.Threading; 5 | using CRA.ClientLibrary; 6 | 7 | namespace ConnectionPair 8 | { 9 | public class MyAsyncInput : IAsyncVertexInputEndpoint 10 | { 11 | bool _running = true; 12 | IVertex _vertex; 13 | 14 | public MyAsyncInput(IVertex vertex) 15 | { 16 | _vertex = vertex; 17 | } 18 | 19 | public void Dispose() 20 | { 21 | Dispose(true); 22 | GC.SuppressFinalize(this); 23 | } 24 | 25 | protected virtual void Dispose(bool disposing) 26 | { 27 | if (disposing) 28 | { 29 | Console.WriteLine("Disposing MyInput"); 30 | _running = false; 31 | } 32 | } 33 | 34 | public async Task FromStreamAsync(Stream stream, string otherVertex, string otherEndpoint, CancellationToken token) 35 | { 36 | Console.WriteLine("Receiving data from vertex: " + otherVertex + ", endpoint: " + otherEndpoint); 37 | 38 | for (int i = 0; i < int.MaxValue; i++) 39 | { 40 | int val = stream.ReadInt32(); 41 | Console.WriteLine("Read value: " + val); 42 | token.ThrowIfCancellationRequested(); 43 | 44 | if (!_running) break; 45 | } 46 | } 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/Samples/ConnectionPair/MyAsyncOutput.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using System.IO; 4 | using System.Threading; 5 | using CRA.ClientLibrary; 6 | 7 | namespace ConnectionPair 8 | { 9 | public class MyAsyncOutput : IAsyncVertexOutputEndpoint 10 | { 11 | bool _running = true; 12 | IVertex _vertex; 13 | 14 | public MyAsyncOutput(IVertex vertex) 15 | { 16 | _vertex = vertex; 17 | } 18 | 19 | public void Dispose() 20 | { 21 | Dispose(true); 22 | GC.SuppressFinalize(this); 23 | } 24 | 25 | protected virtual void Dispose(bool disposing) 26 | { 27 | if (disposing) 28 | { 29 | Console.WriteLine("Disposing MyOutput"); 30 | _running = false; 31 | } 32 | } 33 | 34 | public async Task ToStreamAsync(Stream stream, string otherVertex, string otherEndpoint, CancellationToken token) 35 | { 36 | Console.WriteLine("Sending data to vertex: " + otherVertex + ", endpoint: " + otherEndpoint); 37 | 38 | for (int i = 0; i < int.MaxValue; i += 1) 39 | { 40 | for (int j = 0; j < 1; j++) 41 | { 42 | stream.WriteInt32(i + j); 43 | Console.WriteLine("Written value: " + (i + j)); 44 | } 45 | 46 | 47 | for (int j = 0; j < 1; j++) 48 | { 49 | await Task.Delay(1000); 50 | token.ThrowIfCancellationRequested(); 51 | } 52 | 53 | if (!_running) break; 54 | } 55 | } 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/Samples/ConnectionPair/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using CRA.ClientLibrary; 3 | 4 | namespace ConnectionPair 5 | { 6 | public struct MyParam 7 | { 8 | public int field1; 9 | public string field2; 10 | } 11 | 12 | class Program 13 | { 14 | static void Main(string[] args) 15 | { 16 | var client = new CRAClientLibrary(); 17 | 18 | client.DefineVertexAsync("connectionpairvertex", () => new ConnectionPairVertex()).Wait(); 19 | 20 | client.InstantiateVertexAsync("crainst01", "vertex1", "connectionpairvertex", new MyParam { field1 = 33, field2 = "foo" }).Wait(); 21 | 22 | 23 | client.InstantiateVertexAsync("crainst02", "vertex2", "connectionpairvertex", new MyParam { field1 = 34 }).Wait(); 24 | 25 | client.ConnectAsync("vertex1", "output", "vertex2", "input").Wait(); 26 | client.ConnectAsync("vertex2", "output", "vertex1", "input").Wait(); 27 | 28 | Console.ReadLine(); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/Samples/ConnectionPair/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.InteropServices; 3 | 4 | // General Information about an assembly is controlled through the following 5 | // set of attributes. Change these attribute values to modify the information 6 | // associated with an assembly. 7 | //[assembly: AssemblyTitle("ConnectionPair")] 8 | [assembly: AssemblyDescription("")] 9 | //[assembly: AssemblyConfiguration("")] 10 | //[assembly: AssemblyCompany("")] 11 | //[assembly: AssemblyProduct("ConnectionPair")] 12 | [assembly: AssemblyCopyright("Copyright © 2017")] 13 | [assembly: AssemblyTrademark("")] 14 | [assembly: AssemblyCulture("")] 15 | 16 | // Setting ComVisible to false makes the types in this assembly not visible 17 | // to COM components. If you need to access a type in this assembly from 18 | // COM, set the ComVisible attribute to true on that type. 19 | [assembly: ComVisible(false)] 20 | 21 | // The following GUID is for the ID of the typelib if this project is exposed to COM 22 | [assembly: Guid("00f97b37-965a-4f20-b5b2-e182d20509c2")] 23 | 24 | // Version information for an assembly consists of the following four values: 25 | // 26 | // Major Version 27 | // Minor Version 28 | // Build Number 29 | // Revision 30 | // 31 | // You can specify all the values or you can default the Build and Revision Numbers 32 | // by using the '*' as shown below: 33 | // [assembly: AssemblyVersion("1.0.*")] 34 | //[assembly: AssemblyVersion("1.0.0.0")] 35 | //[assembly: AssemblyFileVersion("1.0.0.0")] 36 | -------------------------------------------------------------------------------- /src/Samples/FusableConnectionPair/App.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /src/Samples/FusableConnectionPair/FusableConnectionPair.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | Exe 4 | netcoreapp2.0;net46 5 | AnyCPU 6 | true 7 | FusableConnectionPair 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | $(DefineConstants);DOTNETCORE 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /src/Samples/FusableConnectionPair/FusableConnectionPairVertex.cs: -------------------------------------------------------------------------------- 1 | using CRA.ClientLibrary; 2 | using System.Threading.Tasks; 3 | 4 | namespace FusableConnectionPair 5 | { 6 | public class FusableConnectionPairVertex : VertexBase 7 | { 8 | public FusableConnectionPairVertex() : base() 9 | { 10 | } 11 | 12 | public override Task InitializeAsync(object vertexParameter) 13 | { 14 | AddAsyncInputEndpoint("input", new MyAsyncFusableInput(this)); 15 | AddAsyncOutputEndpoint("output", new MyAsyncFusableOutput(this)); 16 | return base.InitializeAsync(vertexParameter); 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/Samples/FusableConnectionPair/MyAsyncFusableInput.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using System.IO; 4 | using System.Threading; 5 | using CRA.ClientLibrary; 6 | 7 | namespace FusableConnectionPair 8 | { 9 | public class MyAsyncFusableInput : IAsyncVertexInputEndpoint 10 | { 11 | bool _running = true; 12 | IVertex _vertex; 13 | 14 | public MyAsyncFusableInput(IVertex vertex) 15 | { 16 | _vertex = vertex; 17 | } 18 | 19 | public void Dispose() 20 | { 21 | Dispose(true); 22 | GC.SuppressFinalize(this); 23 | } 24 | 25 | protected virtual void Dispose(bool disposing) 26 | { 27 | if (disposing) 28 | { 29 | Console.WriteLine("Disposing MyInput"); 30 | _running = false; 31 | } 32 | } 33 | 34 | public async Task FromStreamAsync(Stream stream, string otherVertex, string otherEndpoint, CancellationToken token) 35 | { 36 | Console.WriteLine("Receiving data from vertex: " + otherVertex + ", endpoint: " + otherEndpoint); 37 | 38 | for (int i = 0; i < int.MaxValue; i++) 39 | { 40 | int val = stream.ReadInt32(); 41 | Console.WriteLine("Read value: " + val); 42 | token.ThrowIfCancellationRequested(); 43 | 44 | if (!_running) break; 45 | } 46 | } 47 | 48 | public void WriteInt32(int val) 49 | { 50 | Console.WriteLine("Read value: " + val); 51 | } 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/Samples/FusableConnectionPair/MyAsyncFusableOutput.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using System.IO; 4 | using System.Threading; 5 | using CRA.ClientLibrary; 6 | 7 | namespace FusableConnectionPair 8 | { 9 | public class MyAsyncFusableOutput : IAsyncFusableVertexOutputEndpoint 10 | { 11 | bool _running = true; 12 | IVertex _vertex; 13 | 14 | public MyAsyncFusableOutput(IVertex vertex) 15 | { 16 | _vertex = vertex; 17 | } 18 | 19 | public void Dispose() 20 | { 21 | Dispose(true); 22 | GC.SuppressFinalize(this); 23 | } 24 | 25 | protected virtual void Dispose(bool disposing) 26 | { 27 | if (disposing) 28 | { 29 | Console.WriteLine("Disposing MyOutput"); 30 | _running = false; 31 | } 32 | } 33 | 34 | public async Task ToStreamAsync(Stream stream, string otherVertex, string otherEndpoint, CancellationToken token) 35 | { 36 | Console.WriteLine("Sending data to vertex: " + otherVertex + ", endpoint: " + otherEndpoint); 37 | 38 | for (int i = 0; i < int.MaxValue; i += 1) 39 | { 40 | for (int j = 0; j < 1; j++) 41 | { 42 | stream.WriteInt32(i + j); 43 | Console.WriteLine("Written value: " + (i + j)); 44 | } 45 | 46 | 47 | for (int j = 0; j < 1; j++) 48 | { 49 | await Task.Delay(1000); 50 | token.ThrowIfCancellationRequested(); 51 | } 52 | 53 | if (!_running) break; 54 | } 55 | } 56 | 57 | public bool CanFuseWith(IAsyncVertexInputEndpoint endpoint, string otherVertex, string otherEndpoint) 58 | { 59 | if (endpoint as MyAsyncFusableInput != null) return true; 60 | return false; 61 | } 62 | 63 | public async Task ToInputAsync(IAsyncVertexInputEndpoint endpoint, string otherVertex, string otherEndpoint, CancellationToken token) 64 | { 65 | Console.WriteLine("Sending data to fused vertex: " + otherVertex + ", endpoint: " + otherEndpoint); 66 | 67 | MyAsyncFusableInput otherInstance = endpoint as MyAsyncFusableInput; 68 | 69 | for (int i = 0; i < int.MaxValue; i += 1) 70 | { 71 | for (int j = 0; j < 1; j++) 72 | { 73 | otherInstance.WriteInt32(i + j); 74 | Console.WriteLine("Written value: " + (i + j)); 75 | } 76 | 77 | 78 | for (int j = 0; j < 1; j++) 79 | { 80 | await Task.Delay(1000); 81 | token.ThrowIfCancellationRequested(); 82 | } 83 | 84 | if (!_running) break; 85 | } 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/Samples/FusableConnectionPair/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using CRA.ClientLibrary; 3 | 4 | namespace FusableConnectionPair 5 | { 6 | class Program 7 | { 8 | static void Main(string[] args) 9 | { 10 | var client = new CRAClientLibrary(); 11 | client.ResetAsync().Wait(); 12 | 13 | client.DefineVertexAsync("fusableconnectionpairvertex", () => new FusableConnectionPairVertex()).Wait(); 14 | 15 | client.InstantiateVertexAsync("crainst01", "fvertex1", "fusableconnectionpairvertex", null).Wait(); 16 | client.InstantiateVertexAsync("crainst01", "fvertex2", "fusableconnectionpairvertex", null).Wait(); 17 | 18 | client.ConnectAsync("fvertex1", "output", "fvertex2", "input").Wait(); 19 | 20 | Console.ReadLine(); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/Samples/FusableConnectionPair/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | //[assembly: AssemblyTitle("FusableConnectionPair")] 9 | [assembly: AssemblyDescription("")] 10 | //[assembly: AssemblyConfiguration("")] 11 | //[assembly: AssemblyCompany("")] 12 | //[assembly: AssemblyProduct("FusableConnectionPair")] 13 | [assembly: AssemblyCopyright("Copyright © 2017")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("73dddfaf-6075-405f-b171-c935ee38b44a")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | //[assembly: AssemblyVersion("1.0.0.0")] 36 | //[assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /src/Samples/ShardedConnectionPair/App.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /src/Samples/ShardedConnectionPair/MyAsyncInput.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using System.IO; 4 | using System.Threading; 5 | using CRA.ClientLibrary; 6 | 7 | namespace ShardedConnectionPair 8 | { 9 | public class MyAsyncInput : AsyncShardedVertexInputEndpointBase 10 | { 11 | bool _running = true; 12 | IVertex _vertex; 13 | 14 | public MyAsyncInput(IVertex vertex) 15 | { 16 | _vertex = vertex; 17 | } 18 | 19 | public override void Dispose() 20 | { 21 | Dispose(true); 22 | GC.SuppressFinalize(this); 23 | } 24 | 25 | protected virtual void Dispose(bool disposing) 26 | { 27 | if (disposing) 28 | { 29 | Console.WriteLine("Disposing MyInput"); 30 | _running = false; 31 | } 32 | } 33 | 34 | public override Task FromStreamAsync(Stream stream, string otherVertex, int otherShardId, string otherEndpoint, CancellationToken token) 35 | { 36 | Console.WriteLine("Receiving data from vertex: " + otherVertex + ", endpoint: " + otherEndpoint); 37 | 38 | for (int i = 0; i < int.MaxValue; i++) 39 | { 40 | int val = stream.ReadInt32(); 41 | Console.WriteLine("Read value: " + val); 42 | token.ThrowIfCancellationRequested(); 43 | 44 | if (!_running) break; 45 | } 46 | return Task.CompletedTask; 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/Samples/ShardedConnectionPair/MyAsyncOutput.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using System.IO; 4 | using System.Threading; 5 | using CRA.ClientLibrary; 6 | 7 | namespace ShardedConnectionPair 8 | { 9 | public class MyAsyncOutput : AsyncShardedVertexOutputEndpointBase 10 | { 11 | bool _running = true; 12 | IVertex _vertex; 13 | 14 | public MyAsyncOutput(IVertex vertex) 15 | { 16 | _vertex = vertex; 17 | } 18 | 19 | public override void Dispose() 20 | { 21 | Dispose(true); 22 | GC.SuppressFinalize(this); 23 | } 24 | 25 | protected virtual void Dispose(bool disposing) 26 | { 27 | if (disposing) 28 | { 29 | Console.WriteLine("Disposing MyOutput"); 30 | _running = false; 31 | } 32 | } 33 | 34 | public override async Task ToStreamAsync(Stream stream, string otherVertex, int otherShardId, string otherEndpoint, CancellationToken token) 35 | { 36 | Console.WriteLine("Sending data to vertex: " + otherVertex + ", endpoint: " + otherEndpoint); 37 | 38 | for (int i = 0; i < int.MaxValue; i += 1) 39 | { 40 | for (int j = 0; j < 1; j++) 41 | { 42 | stream.WriteInt32(i + j); 43 | Console.WriteLine("Written value: " + (i + j)); 44 | } 45 | 46 | 47 | for (int j = 0; j < 1; j++) 48 | { 49 | await Task.Delay(1000); 50 | token.ThrowIfCancellationRequested(); 51 | } 52 | 53 | if (!_running) break; 54 | } 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/Samples/ShardedConnectionPair/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using CRA.ClientLibrary; 3 | 4 | namespace ShardedConnectionPair 5 | { 6 | class Program 7 | { 8 | static void Main(string[] args) 9 | { 10 | var client = new CRAClientLibrary(); 11 | 12 | client.DefineVertexAsync("shardedconnectionpairvertex", (System.Linq.Expressions.Expression>)(() => new ShardedConnectionPairVertex())).Wait(); 13 | 14 | client.InstantiateVertexAsync( 15 | new[] { "crainst01", "crainst02" }, 16 | "vertex1", 17 | "shardedconnectionpairvertex", 18 | null, 1, e => e % 2).Wait(); 19 | 20 | client.InstantiateVertexAsync( 21 | new[] { "crainst01", "crainst02" }, 22 | "vertex2", 23 | "shardedconnectionpairvertex", 24 | null, 1, e => e % 2).Wait(); 25 | 26 | client.ConnectAsync("vertex1", "output", "vertex2", "input").Wait(); 27 | client.ConnectAsync("vertex2", "output", "vertex1", "input").Wait(); 28 | 29 | Console.ReadLine(); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/Samples/ShardedConnectionPair/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.InteropServices; 3 | 4 | // General Information about an assembly is controlled through the following 5 | // set of attributes. Change these attribute values to modify the information 6 | // associated with an assembly. 7 | //[assembly: AssemblyTitle("ConnectionPair")] 8 | [assembly: AssemblyDescription("")] 9 | //[assembly: AssemblyConfiguration("")] 10 | //[assembly: AssemblyCompany("")] 11 | //[assembly: AssemblyProduct("ConnectionPair")] 12 | [assembly: AssemblyCopyright("Copyright © 2017")] 13 | [assembly: AssemblyTrademark("")] 14 | [assembly: AssemblyCulture("")] 15 | 16 | // Setting ComVisible to false makes the types in this assembly not visible 17 | // to COM components. If you need to access a type in this assembly from 18 | // COM, set the ComVisible attribute to true on that type. 19 | [assembly: ComVisible(false)] 20 | 21 | // The following GUID is for the ID of the typelib if this project is exposed to COM 22 | [assembly: Guid("00f97b37-965a-4f20-b5b2-e182d20509c2")] 23 | 24 | // Version information for an assembly consists of the following four values: 25 | // 26 | // Major Version 27 | // Minor Version 28 | // Build Number 29 | // Revision 30 | // 31 | // You can specify all the values or you can default the Build and Revision Numbers 32 | // by using the '*' as shown below: 33 | // [assembly: AssemblyVersion("1.0.*")] 34 | //[assembly: AssemblyVersion("1.0.0.0")] 35 | //[assembly: AssemblyFileVersion("1.0.0.0")] 36 | -------------------------------------------------------------------------------- /src/Samples/ShardedConnectionPair/ShardedConnectionPair.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | Exe 4 | netcoreapp2.0;net46 5 | AnyCPU 6 | true 7 | ShardedConnectionPair 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | $(DefineConstants);DOTNETCORE 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /src/Samples/ShardedConnectionPair/ShardedConnectionPairVertex.cs: -------------------------------------------------------------------------------- 1 | using CRA.ClientLibrary; 2 | using System; 3 | using System.Threading.Tasks; 4 | 5 | namespace ShardedConnectionPair 6 | { 7 | public class ShardedConnectionPairVertex : ShardedVertexBase 8 | { 9 | public ShardedConnectionPairVertex() : base() 10 | { 11 | } 12 | 13 | public override Task InitializeAsync(int shardId, ShardingInfo shardingInfo, object vertexParameter) 14 | { 15 | Console.WriteLine("Sharded vertex name: {0}", GetVertexName()); 16 | 17 | AddAsyncInputEndpoint("input", new MyAsyncInput(this)); 18 | AddAsyncOutputEndpoint("output", new MyAsyncOutput(this)); 19 | 20 | return Task.CompletedTask; 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/Samples/ShardedDatasetTest/App.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /src/Samples/ShardedDatasetTest/ConfigurationsManager.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace ShardedDatasetTest 6 | { 7 | public static class ConfigurationsManager 8 | { 9 | //CRA instance prefix 10 | public static string CRA_WORKER_PREFIX = "crainst"; 11 | public static int INSTANCE_COUNT_PER_CRA_WORKER = 1; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/Samples/ShardedDatasetTest/IIntKeyedDatasetObserver.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace ShardedDatasetTest 6 | { 7 | public interface IIntKeyedDatasetObserver 8 | { 9 | void ProcessIntKeyedDataset(IntKeyedDataset source); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Samples/ShardedDatasetTest/IntKeyedDataset.cs: -------------------------------------------------------------------------------- 1 | using CRA.ClientLibrary; 2 | using CRA.ClientLibrary.DataProcessing; 3 | using System; 4 | using System.IO; 5 | using System.Linq.Expressions; 6 | 7 | namespace ShardedDatasetTest 8 | { 9 | public class IntKeyedDataset : DatasetBase 10 | { 11 | protected int _shardingCardinality; 12 | 13 | public int ShardingCardinality 14 | { 15 | get 16 | { 17 | return _shardingCardinality; 18 | } 19 | 20 | set 21 | { 22 | _shardingCardinality = value; 23 | } 24 | } 25 | 26 | protected int[] _keys; 27 | protected int[] _values; 28 | public int _shift; 29 | 30 | public IntKeyedDataset() 31 | { 32 | } 33 | 34 | public IntKeyedDataset(int shift) 35 | { 36 | _shift = shift; 37 | _keys = new int[20]; 38 | _values = new int[20]; 39 | for (int i = 0; i < _values.Length; i++) 40 | { 41 | _keys[i] = i + shift; 42 | _values[i] = i + shift; 43 | } 44 | } 45 | 46 | public IntKeyedDataset(IntKeyedDataset intDataset) : this(intDataset._shift) 47 | { 48 | _shardingCardinality = intDataset.ShardingCardinality; 49 | _shift = intDataset._shift; 50 | _keys = intDataset._keys; 51 | _values = intDataset._values; 52 | } 53 | 54 | public override IDataset ReKey(Expression> selector) 55 | { 56 | if (selector is Expression>) 57 | { 58 | return (IDataset)new StringKeyedDataset(_shift); 59 | } 60 | else 61 | { 62 | throw new InvalidCastException(); 63 | } 64 | } 65 | 66 | public override void ToStream(Stream stream) 67 | { 68 | byte[] intBytes = BitConverter.GetBytes(_shift); 69 | if (BitConverter.IsLittleEndian) 70 | Array.Reverse(intBytes); 71 | byte[] result = intBytes; 72 | 73 | stream.WriteAsync(result, 0, result.Length); 74 | } 75 | 76 | 77 | private IDataset CreateStreamableDatasetFromStream(Stream stream) 78 | { 79 | byte[] intBytes = new byte[BitConverter.GetBytes(_shift).Length]; 80 | stream.ReadAllRequiredBytes(intBytes, 0, intBytes.Length); 81 | if (BitConverter.IsLittleEndian) 82 | Array.Reverse(intBytes); 83 | _shift = BitConverter.ToInt32(intBytes, 0); 84 | 85 | return (IDataset)new IntKeyedDataset(_shift); 86 | } 87 | 88 | 89 | public override Expression>> CreateFromStreamDeserializer() 90 | { 91 | return stream => (new IntKeyedDataset()).CreateStreamableDatasetFromStream(stream); 92 | } 93 | 94 | public override void Subscribe(object observer) 95 | { 96 | if (observer is IIntKeyedDatasetObserver) 97 | { 98 | ((IIntKeyedDatasetObserver)observer).ProcessIntKeyedDataset(this); 99 | } 100 | else 101 | throw new InvalidCastException(); 102 | } 103 | 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /src/Samples/ShardedDatasetTest/IntKeyedDatasetExtensions.cs: -------------------------------------------------------------------------------- 1 | using CRA.ClientLibrary.DataProcessing; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | 6 | namespace ShardedDatasetTest 7 | { 8 | public static class IntKeyedDatasetExtensions 9 | { 10 | public static IntKeyedDataset ShiftUp(this IntKeyedDataset source, int upAmount) 11 | { 12 | return new IntKeyedDataset(source._shift + upAmount); 13 | } 14 | 15 | public static IntKeyedDataset BinaryShiftUp(this IntKeyedDataset source, IntKeyedDataset inputDS, int upAmount) 16 | { 17 | return new IntKeyedDataset(source._shift + inputDS._shift + upAmount); 18 | } 19 | 20 | 21 | public static IntKeyedDataset[] Splitter(this IntKeyedDataset source, IMoveDescriptor descriptor) 22 | { 23 | IntKeyedDataset[] outputs = new IntKeyedDataset[2]; 24 | outputs[0] = new IntKeyedDataset(source._shift + 100); 25 | outputs[1] = new IntKeyedDataset(source._shift + 200); 26 | return outputs; 27 | } 28 | 29 | public static IntKeyedDataset Merger(this IntKeyedDataset[] source, IMoveDescriptor descriptor) 30 | { 31 | return new IntKeyedDataset(source[0]._shift + source[1]._shift); 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/Samples/ShardedDatasetTest/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | //[assembly: AssemblyTitle("ShardedDatasetTest")] 9 | [assembly: AssemblyDescription("")] 10 | //[assembly: AssemblyConfiguration("")] 11 | //[assembly: AssemblyCompany("")] 12 | //[assembly: AssemblyProduct("ShardedDatasetTest")] 13 | [assembly: AssemblyCopyright("Copyright © 2017")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("3c700a3b-9958-4470-9639-87964d20b11a")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | //[assembly: AssemblyVersion("1.0.0.0")] 36 | //[assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /src/Samples/ShardedDatasetTest/ShardedDatasetTest.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | Exe 4 | netcoreapp2.0;net46 5 | AnyCPU 6 | true 7 | ShardedDatasetTest 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | $(DefineConstants);DOTNETCORE 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /src/Samples/ShardedDatasetTest/StringKeyedDataset.cs: -------------------------------------------------------------------------------- 1 | using CRA.ClientLibrary; 2 | using CRA.ClientLibrary.DataProcessing; 3 | using System; 4 | using System.IO; 5 | using System.Linq.Expressions; 6 | 7 | namespace ShardedDatasetTest 8 | { 9 | public class StringKeyedDataset : DatasetBase 10 | { 11 | protected int _shardingCardinality; 12 | 13 | public int ShardingCardinality 14 | { 15 | get 16 | { 17 | return _shardingCardinality; 18 | } 19 | 20 | set 21 | { 22 | _shardingCardinality = value; 23 | } 24 | } 25 | 26 | protected string[] _keys; 27 | protected int[] _values; 28 | public int _shift; 29 | 30 | public StringKeyedDataset() 31 | { 32 | } 33 | 34 | public StringKeyedDataset(int shift) 35 | { 36 | _shift = shift; 37 | _keys = new string[20]; 38 | _values = new int[20]; 39 | for (int i = 0; i < _values.Length; i++) 40 | { 41 | _keys[i] = (i + shift) + ""; 42 | _values[i] = i + shift; 43 | } 44 | } 45 | 46 | public StringKeyedDataset(StringKeyedDataset intDataset) : this(intDataset._shift) 47 | { 48 | _shardingCardinality = intDataset.ShardingCardinality; 49 | } 50 | 51 | public override IDataset ReKey(Expression> selector) 52 | { 53 | return null; 54 | } 55 | 56 | public override void ToStream(Stream stream) 57 | { 58 | byte[] intBytes = BitConverter.GetBytes(_shift); 59 | if (BitConverter.IsLittleEndian) 60 | Array.Reverse(intBytes); 61 | byte[] result = intBytes; 62 | 63 | stream.WriteAsync(result, 0, result.Length); 64 | } 65 | 66 | 67 | private IDataset CreateStreamableDatasetFromStream(Stream stream) 68 | { 69 | byte[] intBytes = new byte[BitConverter.GetBytes(_shift).Length]; 70 | stream.ReadAllRequiredBytes(intBytes, 0, intBytes.Length); 71 | if (BitConverter.IsLittleEndian) 72 | Array.Reverse(intBytes); 73 | _shift = BitConverter.ToInt32(intBytes, 0); 74 | 75 | return (IDataset)new StringKeyedDataset(_shift); 76 | } 77 | 78 | 79 | public override Expression>> CreateFromStreamDeserializer() 80 | { 81 | return stream => (new StringKeyedDataset()).CreateStreamableDatasetFromStream(stream); 82 | } 83 | 84 | public override void Subscribe(object observer) 85 | { 86 | throw new NotImplementedException(); 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/Samples/ShardedDatasetTest/observers/WriteToConsoleObserver.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace ShardedDatasetTest 6 | { 7 | public class WriteToConsoleObserver : IIntKeyedDatasetObserver 8 | { 9 | public WriteToConsoleObserver() 10 | { 11 | } 12 | 13 | public void ProcessIntKeyedDataset(IntKeyedDataset source) 14 | { 15 | Console.WriteLine("Shift value is: " + source._shift); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/privatesettings.config.example: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | --------------------------------------------------------------------------------