├── README.txt ├── .nuget ├── NuGet.exe └── NuGet.Config ├── SharedAssemblyInfo.cs ├── Frameworks ├── DifferentialDataflow │ ├── bin │ │ └── Debug │ │ │ └── NaiadLINQ.pdb │ ├── DifferentialDataflow.csproj.vspscc │ ├── App.config │ ├── Properties │ │ └── AssemblyInfo.cs │ ├── Operators │ │ ├── Abs.cs │ │ ├── Union.cs │ │ ├── Distinct.cs │ │ ├── Intersect.cs │ │ ├── SymmetricDifference.cs │ │ ├── Concat.cs │ │ ├── Select.cs │ │ ├── Except.cs │ │ ├── Where.cs │ │ ├── AdjustLattice.cs │ │ ├── Prioritize.cs │ │ ├── Aggregate.cs │ │ ├── GroupBy.cs │ │ ├── Count.cs │ │ ├── Monitor.cs │ │ ├── Max.cs │ │ ├── Sum.cs │ │ └── SelectMany.cs │ ├── OperatorImplementations │ │ └── UnaryStatefulWithAggregation.cs │ ├── Microsoft.Research.Naiad.DifferentialDataflow.nuspec │ ├── CollectionTrace │ │ ├── OffsetLength.cs │ │ └── Increments.cs │ ├── CoreGenerics.cs │ └── KeyIndices.cs ├── HdfsSupport │ ├── packages.config │ ├── app.config │ ├── Properties │ │ └── AssemblyInfo.cs │ └── Microsoft.Research.Naiad.HdfsSupport.nuspec ├── Lindi │ ├── Lindi.csproj.vspscc │ ├── Properties │ │ └── AssemblyInfo.cs │ └── Microsoft.Research.Naiad.Lindi.nuspec ├── WebHdfsSupport │ ├── packages.config │ ├── app.config │ ├── Properties │ │ └── AssemblyInfo.cs │ └── Microsoft.Research.Naiad.WebHdfsSupport.nuspec ├── Storage │ ├── packages.config │ ├── app.config │ ├── Properties │ │ └── AssemblyInfo.cs │ └── Microsoft.Research.Naiad.Storage.nuspec ├── AzureSupport │ ├── packages.config │ ├── Properties │ │ └── AssemblyInfo.cs │ ├── app.config │ ├── Console.cs │ └── Microsoft.Research.Naiad.AzureSupport.nuspec ├── WorkGenerator │ ├── Properties │ │ └── AssemblyInfo.cs │ └── Microsoft.Research.Naiad.WorkGenerator.nuspec └── GraphLINQ │ ├── Properties │ └── AssemblyInfo.cs │ └── Microsoft.Research.Naiad.GraphLINQ.nuspec ├── Documentation └── NaiadDocumentation │ ├── NaiadDocumentation │ ├── icons │ │ └── Help.png │ ├── Content │ │ └── VersionHistory │ │ │ ├── v0.5.aml │ │ │ ├── v0.4.aml │ │ │ ├── v0.1.aml │ │ │ ├── VersionHistory.aml │ │ │ ├── v0.2.aml │ │ │ └── v0.3.aml │ └── ContentLayout.content │ └── NaiadDocumentation.sln ├── .gitignore ├── Naiad ├── Naiad.csproj.vspscc ├── App.config ├── Frameworks │ └── LINQ.cs ├── Channels │ ├── ProcessBroadcastChannel.cs │ ├── BufferSegmentConsumer.cs │ └── PipelineChannel.cs ├── Properties │ └── AssemblyInfo.cs ├── Runtime │ ├── Progress │ │ ├── Update.cs │ │ ├── UpdateBuffer.cs │ │ └── PointstampCountSet.cs │ └── Controlling │ │ └── InternalController.cs ├── Microsoft.Research.Naiad.nuspec ├── Dataflow │ └── Stream.cs ├── Util │ └── DependencyLister.cs ├── DataStructures │ ├── CircularLogBuffer.cs │ └── Pair.cs └── NamespaceDocs.cs ├── Examples ├── Examples.csproj.vspscc ├── App.config ├── Properties │ └── AssemblyInfo.cs └── DifferentialDataflow │ └── WordCount.cs ├── ClusterSubmission ├── InstallConfigFile.ps1 ├── ConfigFile.targets ├── RunNaiad │ ├── AzureYarnSubmission.cs │ ├── packages.config │ ├── Properties │ │ └── AssemblyInfo.cs │ ├── Microsoft.Research.Naiad.ClusterSubmission.nuspec │ ├── YarnSubmission.cs │ ├── App.config │ └── LocalSubmission.cs ├── ClusterSubmission.sln └── NaiadPeloponneseSupport │ ├── packages.config │ ├── Properties │ └── AssemblyInfo.cs │ ├── PPMSubmission.csproj │ ├── PPMSubmission.cs │ └── App.config ├── AzureExamples ├── packages.config ├── Properties │ └── AssemblyInfo.cs ├── App.config ├── Azure │ ├── Repartition.cs │ └── GraphGenerator.cs └── Program.cs ├── NugetSample ├── packages.config ├── Properties │ └── AssemblyInfo.cs ├── App.config └── Microsoft.Research.Naiad.Sample.nuspec └── .gitattributes /README.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TimelyDataflow/Naiad/HEAD/README.txt -------------------------------------------------------------------------------- /.nuget/NuGet.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TimelyDataflow/Naiad/HEAD/.nuget/NuGet.exe -------------------------------------------------------------------------------- /SharedAssemblyInfo.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TimelyDataflow/Naiad/HEAD/SharedAssemblyInfo.cs -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/bin/Debug/NaiadLINQ.pdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TimelyDataflow/Naiad/HEAD/Frameworks/DifferentialDataflow/bin/Debug/NaiadLINQ.pdb -------------------------------------------------------------------------------- /Documentation/NaiadDocumentation/NaiadDocumentation/icons/Help.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TimelyDataflow/Naiad/HEAD/Documentation/NaiadDocumentation/NaiadDocumentation/icons/Help.png -------------------------------------------------------------------------------- /.nuget/NuGet.Config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.suo 2 | *.user 3 | *.sln.docstates 4 | Debug/ 5 | Release/ 6 | x64/ 7 | bin/ 8 | obj/ 9 | *.psess 10 | *.vsp 11 | *.vspx 12 | *.log 13 | Thumbs.db 14 | ehthumbs.db 15 | *~ 16 | Desktop.ini 17 | *.nupkg 18 | packages/ -------------------------------------------------------------------------------- /Frameworks/HdfsSupport/packages.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /Naiad/Naiad.csproj.vspscc: -------------------------------------------------------------------------------- 1 | "" 2 | { 3 | "FILE_VERSION" = "9237" 4 | "ENLISTMENT_CHOICE" = "NEVER" 5 | "PROJECT_FILE_RELATIVE_PATH" = "" 6 | "NUMBER_OF_EXCLUDED_FILES" = "0" 7 | "ORIGINAL_PROJECT_FILE_PATH" = "" 8 | "NUMBER_OF_NESTED_PROJECTS" = "0" 9 | "SOURCE_CONTROL_SETTINGS_PROVIDER" = "PROVIDER" 10 | } 11 | -------------------------------------------------------------------------------- /Examples/Examples.csproj.vspscc: -------------------------------------------------------------------------------- 1 | "" 2 | { 3 | "FILE_VERSION" = "9237" 4 | "ENLISTMENT_CHOICE" = "NEVER" 5 | "PROJECT_FILE_RELATIVE_PATH" = "" 6 | "NUMBER_OF_EXCLUDED_FILES" = "0" 7 | "ORIGINAL_PROJECT_FILE_PATH" = "" 8 | "NUMBER_OF_NESTED_PROJECTS" = "0" 9 | "SOURCE_CONTROL_SETTINGS_PROVIDER" = "PROVIDER" 10 | } 11 | -------------------------------------------------------------------------------- /Frameworks/Lindi/Lindi.csproj.vspscc: -------------------------------------------------------------------------------- 1 | "" 2 | { 3 | "FILE_VERSION" = "9237" 4 | "ENLISTMENT_CHOICE" = "NEVER" 5 | "PROJECT_FILE_RELATIVE_PATH" = "" 6 | "NUMBER_OF_EXCLUDED_FILES" = "0" 7 | "ORIGINAL_PROJECT_FILE_PATH" = "" 8 | "NUMBER_OF_NESTED_PROJECTS" = "0" 9 | "SOURCE_CONTROL_SETTINGS_PROVIDER" = "PROVIDER" 10 | } 11 | -------------------------------------------------------------------------------- /ClusterSubmission/InstallConfigFile.ps1: -------------------------------------------------------------------------------- 1 | param($installPath, $toolsPath, $package, $project) 2 | 3 | foreach ($targetsFile in Get-ChildItem (Join-Path $installPath "build") -Filter "*.targets") { 4 | (Get-Content $targetsFile.FullName) | Foreach-Object { $_ -replace 'SUBMISSION_PACKAGE_DIRECTORY', $installPath } | Set-Content $targetsFile.FullName 5 | } -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/DifferentialDataflow.csproj.vspscc: -------------------------------------------------------------------------------- 1 | "" 2 | { 3 | "FILE_VERSION" = "9237" 4 | "ENLISTMENT_CHOICE" = "NEVER" 5 | "PROJECT_FILE_RELATIVE_PATH" = "" 6 | "NUMBER_OF_EXCLUDED_FILES" = "0" 7 | "ORIGINAL_PROJECT_FILE_PATH" = "" 8 | "NUMBER_OF_NESTED_PROJECTS" = "0" 9 | "SOURCE_CONTROL_SETTINGS_PROVIDER" = "PROVIDER" 10 | } 11 | -------------------------------------------------------------------------------- /Frameworks/WebHdfsSupport/packages.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /Naiad/App.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/App.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /Frameworks/WebHdfsSupport/app.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /Frameworks/Storage/packages.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /ClusterSubmission/ConfigFile.targets: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /AzureExamples/packages.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /NugetSample/packages.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /Naiad/Frameworks/LINQ.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | -------------------------------------------------------------------------------- /Frameworks/Storage/app.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /Frameworks/HdfsSupport/app.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /Frameworks/AzureSupport/packages.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /Documentation/NaiadDocumentation/NaiadDocumentation.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 2013 4 | VisualStudioVersion = 12.0.30110.0 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{7CF6DF6D-3B04-46F8-A40B-537D21BCA0B4}") = "NaiadDocumentation", "NaiadDocumentation\NaiadDocumentation.shfbproj", "{3D7C87EB-E3D3-4DE1-A4B9-94AB66C99D36}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Release|Any CPU = Release|Any CPU 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {3D7C87EB-E3D3-4DE1-A4B9-94AB66C99D36}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 15 | {3D7C87EB-E3D3-4DE1-A4B9-94AB66C99D36}.Debug|Any CPU.Build.0 = Debug|Any CPU 16 | {3D7C87EB-E3D3-4DE1-A4B9-94AB66C99D36}.Release|Any CPU.ActiveCfg = Release|Any CPU 17 | {3D7C87EB-E3D3-4DE1-A4B9-94AB66C99D36}.Release|Any CPU.Build.0 = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(SolutionProperties) = preSolution 20 | HideSolutionNode = FALSE 21 | EndGlobalSection 22 | EndGlobal 23 | -------------------------------------------------------------------------------- /Naiad/Channels/ProcessBroadcastChannel.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | using Microsoft.Research.Naiad.DataStructures; 26 | using System.Diagnostics; 27 | using Microsoft.Research.Naiad.Serialization; 28 | using Microsoft.Research.Naiad.Scheduling; 29 | 30 | namespace Microsoft.Research.Naiad.Dataflow.Channels 31 | { 32 | } 33 | -------------------------------------------------------------------------------- /Documentation/NaiadDocumentation/NaiadDocumentation/Content/VersionHistory/v0.5.aml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | An update release of Naiad, version 0.5 includes support for reading and writing Hdfs files using either the Java-based 8 | protocol or REST-based WebHdfs, as well as bug fixes. 9 | 10 | 11 | 12 | 13 | 14 | Version 0.5 was released on October 17th, 2014 15 | 16 | 17 | 18 |
19 | Changes in This Release 20 | 21 | 22 | 23 | 24 | Frameworks for reading and writing data in Hdfs. 25 | 26 | 27 | 28 | 29 |
30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /Examples/App.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /AzureExamples/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | using System.Reflection; 21 | using System.Runtime.CompilerServices; 22 | using System.Runtime.InteropServices; 23 | 24 | // General Information about an assembly is controlled through the following 25 | // set of attributes. Change these attribute values to modify the information 26 | // associated with an assembly. 27 | [assembly: AssemblyTitle("AzureExamples")] 28 | [assembly: AssemblyConfiguration("")] 29 | 30 | // The following GUID is for the ID of the typelib if this project is exposed to COM 31 | [assembly: Guid("c628bffb-0c5e-4f55-b692-309b83782763")] 32 | -------------------------------------------------------------------------------- /Frameworks/HdfsSupport/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | using System.Reflection; 21 | using System.Runtime.CompilerServices; 22 | using System.Runtime.InteropServices; 23 | 24 | // General Information about an assembly is controlled through the following 25 | // set of attributes. Change these attribute values to modify the information 26 | // associated with an assembly. 27 | [assembly: AssemblyTitle("HdfsSupport")] 28 | [assembly: AssemblyConfiguration("")] 29 | 30 | // The following GUID is for the ID of the typelib if this project is exposed to COM 31 | [assembly: Guid("4659cd4b-1cea-424d-a4d9-70bda63ca19b")] 32 | -------------------------------------------------------------------------------- /Frameworks/WorkGenerator/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | using System.Reflection; 21 | using System.Runtime.CompilerServices; 22 | using System.Runtime.InteropServices; 23 | 24 | // General Information about an assembly is controlled through the following 25 | // set of attributes. Change these attribute values to modify the information 26 | // associated with an assembly. 27 | [assembly: AssemblyTitle("WorkGenerator")] 28 | [assembly: AssemblyConfiguration("")] 29 | 30 | // The following GUID is for the ID of the typelib if this project is exposed to COM 31 | [assembly: Guid("a53990d2-34a4-4e76-b08b-5428e64dc742")] 32 | -------------------------------------------------------------------------------- /Naiad/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System.Reflection; 22 | using System.Runtime.CompilerServices; 23 | using System.Runtime.InteropServices; 24 | 25 | // General Information about an assembly is controlled through the following 26 | // set of attributes. Change these attribute values to modify the information 27 | // associated with an assembly. 28 | [assembly: AssemblyTitle("Microsoft.Research.Naiad")] 29 | [assembly: AssemblyConfiguration("")] 30 | 31 | // The following GUID is for the ID of the typelib if this project is exposed to COM 32 | [assembly: Guid("e2d5f5f0-161a-46ef-9c5b-129a307527e3")] 33 | -------------------------------------------------------------------------------- /Frameworks/WebHdfsSupport/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | using System.Reflection; 21 | using System.Runtime.CompilerServices; 22 | using System.Runtime.InteropServices; 23 | 24 | // General Information about an assembly is controlled through the following 25 | // set of attributes. Change these attribute values to modify the information 26 | // associated with an assembly. 27 | [assembly: AssemblyTitle("WebHdfsSupport")] 28 | [assembly: AssemblyConfiguration("")] 29 | 30 | // The following GUID is for the ID of the typelib if this project is exposed to COM 31 | [assembly: Guid("83ee83cf-e684-425a-836e-0798594ce68e")] 32 | -------------------------------------------------------------------------------- /Examples/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System.Reflection; 22 | using System.Runtime.CompilerServices; 23 | using System.Runtime.InteropServices; 24 | 25 | // General Information about an assembly is controlled through the following 26 | // set of attributes. Change these attribute values to modify the information 27 | // associated with an assembly. 28 | [assembly: AssemblyTitle("Microsoft.Research.Naiad.Examples")] 29 | [assembly: AssemblyConfiguration("")] 30 | 31 | // The following GUID is for the ID of the typelib if this project is exposed to COM 32 | [assembly: Guid("43490246-380e-41db-a6fd-90112b8b6189")] 33 | -------------------------------------------------------------------------------- /NugetSample/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System.Reflection; 22 | using System.Runtime.CompilerServices; 23 | using System.Runtime.InteropServices; 24 | 25 | // General Information about an assembly is controlled through the following 26 | // set of attributes. Change these attribute values to modify the information 27 | // associated with an assembly. 28 | [assembly: AssemblyTitle("Microsoft.Research.Naiad.Sample")] 29 | [assembly: AssemblyConfiguration("")] 30 | 31 | // The following GUID is for the ID of the typelib if this project is exposed to COM 32 | [assembly: Guid("f9d9ae1f-d996-463a-ac0d-699c6da20e3c")] 33 | -------------------------------------------------------------------------------- /ClusterSubmission/RunNaiad/AzureYarnSubmission.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using Microsoft.Research.Peloponnese.ClusterUtils; 22 | using System; 23 | using System.Collections.Generic; 24 | using System.Linq; 25 | using System.Text; 26 | using System.Threading.Tasks; 27 | 28 | namespace Microsoft.Research.Naiad.Cluster.Submission 29 | { 30 | class AzureYarnSubmission : ClusterSubmission 31 | { 32 | public AzureYarnSubmission(AzureYarnClient yarnClient, Uri baseUri, int numberOfProcesses, string[] args) 33 | : base(yarnClient, baseUri, null, -1, numberOfProcesses, -1, args) 34 | { 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /Frameworks/Lindi/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System.Reflection; 22 | using System.Runtime.CompilerServices; 23 | using System.Runtime.InteropServices; 24 | 25 | // General Information about an assembly is controlled through the following 26 | // set of attributes. Change these attribute values to modify the information 27 | // associated with an assembly. 28 | [assembly: AssemblyTitle("Microsoft.Research.Naiad.Lindi")] 29 | [assembly: AssemblyConfiguration("")] 30 | 31 | // The following GUID is for the ID of the typelib if this project is exposed to COM 32 | [assembly: Guid("e37eabd1-ee4d-4c42-9d4b-282f96cbd2d4")] 33 | -------------------------------------------------------------------------------- /Frameworks/Storage/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | using System.Reflection; 21 | using System.Runtime.CompilerServices; 22 | using System.Runtime.InteropServices; 23 | 24 | // General Information about an assembly is controlled through the following 25 | // set of attributes. Change these attribute values to modify the information 26 | // associated with an assembly. 27 | [assembly: AssemblyTitle("Microsoft.Research.Naiad.Storage")] 28 | [assembly: AssemblyConfiguration("")] 29 | 30 | // The following GUID is for the ID of the typelib if this project is exposed to COM 31 | [assembly: Guid("5455f753-2668-4348-96c5-7101c521de71")] 32 | -------------------------------------------------------------------------------- /Frameworks/GraphLINQ/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System.Reflection; 22 | using System.Runtime.CompilerServices; 23 | using System.Runtime.InteropServices; 24 | 25 | // General Information about an assembly is controlled through the following 26 | // set of attributes. Change these attribute values to modify the information 27 | // associated with an assembly. 28 | [assembly: AssemblyTitle("Microsoft.Research.Naiad.GraphLINQ")] 29 | [assembly: AssemblyConfiguration("")] 30 | 31 | // The following GUID is for the ID of the typelib if this project is exposed to COM 32 | [assembly: Guid("28c42104-7500-405f-842b-d25d5e4c5dda")] 33 | -------------------------------------------------------------------------------- /Frameworks/AzureSupport/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System.Reflection; 22 | using System.Runtime.CompilerServices; 23 | using System.Runtime.InteropServices; 24 | 25 | // General Information about an assembly is controlled through the following 26 | // set of attributes. Change these attribute values to modify the information 27 | // associated with an assembly. 28 | [assembly: AssemblyTitle("Microsoft.Research.Naiad.AzureSupport")] 29 | [assembly: AssemblyConfiguration("")] 30 | 31 | // The following GUID is for the ID of the typelib if this project is exposed to COM 32 | [assembly: Guid("3d489523-573a-4d15-a7ef-ce273702c39d")] 33 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System.Reflection; 22 | using System.Runtime.CompilerServices; 23 | using System.Runtime.InteropServices; 24 | 25 | // General Information about an assembly is controlled through the following 26 | // set of attributes. Change these attribute values to modify the information 27 | // associated with an assembly. 28 | [assembly: AssemblyTitle("Microsoft.Research.Naiad.DifferentialDataflow")] 29 | [assembly: AssemblyConfiguration("")] 30 | 31 | // The following GUID is for the ID of the typelib if this project is exposed to COM 32 | [assembly: Guid("da38e2f3-2364-4d1b-9ee6-8ee21485aebe")] 33 | -------------------------------------------------------------------------------- /Documentation/NaiadDocumentation/NaiadDocumentation/Content/VersionHistory/v0.4.aml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | The first release of Naiad with a fully-documented SDK, version 0.4 includes many API simplifications, 8 | and a new framework with optimizations for graph-specific computations. 9 | 10 | 11 | 12 | 13 | Version 0.4 was released on April 17th 2014. 14 | 15 | 16 | 17 |
18 | Changes in This Release 19 | 20 | 21 | 22 | 23 | Comprehensive documentation of every public class, method, and property. 24 | 25 | 26 | 27 | The GraphLINQ framework contains data types and extension methods that add optimized graph-specific 28 | operators for streams. 29 | 30 | 31 | 32 | 33 | 34 |
35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /Documentation/NaiadDocumentation/NaiadDocumentation/Content/VersionHistory/v0.1.aml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 9 | The first version of Naiad, version 0.1 included an implementation of differential 10 | dataflow that ran on one or many computers. 11 | 12 | 13 | 14 | 15 | Version 0.1 was released on October 8th 2012. 16 | 17 | 18 |
19 | Changes in This Release 20 | 21 | 22 | 23 | 24 | Serial, multicore, and distributed implementations of differential dataflow. 25 | 26 | 27 | 28 | Example programs that illustrate how to use differential dataflow for a variety of 29 | graph and non-graph computations. 30 | 31 | 32 | 33 | 34 | 35 | 36 |
37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /Naiad/Runtime/Progress/Update.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | 26 | namespace Microsoft.Research.Naiad.Runtime.Progress 27 | { 28 | /// 29 | /// An update to a pointstamp count 30 | /// 31 | public struct Update 32 | { 33 | /// 34 | /// The pointstamp 35 | /// 36 | public Pointstamp Pointstamp; 37 | 38 | /// 39 | /// The change in the count of the pointstamp 40 | /// 41 | public Int64 Delta; 42 | 43 | internal Update(Pointstamp p, Int64 d) { this.Pointstamp = p; this.Delta = d; } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/Operators/Abs.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using Microsoft.Research.Naiad.Dataflow; 22 | using System; 23 | using System.Collections.Generic; 24 | using System.Linq; 25 | using System.Text; 26 | 27 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.Operators 28 | { 29 | internal class Abs : OperatorImplementations.UnaryStatefulOperator 30 | where S : IEquatable 31 | where T : Time 32 | { 33 | protected override Int64 WeightFunction(Int64 weight) 34 | { 35 | return Math.Abs(weight); 36 | } 37 | 38 | public Abs(int index, Stage collection, bool inputImmutable) 39 | : base(index, collection, inputImmutable) 40 | { 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /Frameworks/AzureSupport/app.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /Documentation/NaiadDocumentation/NaiadDocumentation/Content/VersionHistory/VersionHistory.aml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | This section describes the various changes made in each version of the Naiad project. 6 | 7 | 8 |
9 | Version History 10 | 11 | Select a version below to see a description of its changes. 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 |
36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /NugetSample/App.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /AzureExamples/App.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /Documentation/NaiadDocumentation/NaiadDocumentation/Content/VersionHistory/v0.2.aml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 9 | The first open-source release of Naiad, version 0.2 included support for multiple 10 | data-parallel frameworks on top of a common low-latency and high-throughput 11 | distributed runtime. 12 | 13 | 14 | 15 | 16 | 17 | Version 0.2 was released on October 29th 2013. 18 | 19 | 20 |
21 | Changes in This Release 22 | 23 | 24 | 25 | 26 | The license was changed to the Apache 2.0 open-source license. 27 | 28 | 29 | 30 | The Naiad runtime was split into the Naiad.dll runtime, 31 | and multiple frameworks. This release contained the Lindi (LINQ with declarative iteration) 32 | and Differential Dataflow frameworks. 33 | 34 | 35 | 36 | 37 | 38 |
39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/Operators/Union.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using Microsoft.Research.Naiad.Dataflow; 22 | using System; 23 | using System.Collections.Generic; 24 | using System.Linq; 25 | using System.Text; 26 | 27 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.Operators 28 | { 29 | class Union : OperatorImplementations.BinaryStatefulOperator 30 | where S : IEquatable 31 | where T : Microsoft.Research.Naiad.Time 32 | { 33 | protected override Int64 WeightFunction(Int64 weight1, Int64 weight2) 34 | { 35 | return Math.Max(weight1, weight2); 36 | } 37 | 38 | public Union(int index, Stage collection, bool input1Immutable, bool input2Immutable) 39 | : base(index, collection, input1Immutable, input2Immutable) 40 | { 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /NugetSample/Microsoft.Research.Naiad.Sample.nuspec: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Microsoft.Research.Naiad.Sample 5 | Naiad - Sample graph analysis program 6 | 0.5.0-beta 7 | naiadquestions@microsoft.com 8 | naiadquestions@microsoft.com,Microsoft 9 | http://www.apache.org/licenses/LICENSE-2.0.html 10 | http://research.microsoft.com/naiad/ 11 | 12 | true 13 | Illustrates how to perform an iterative graph analysis using Naiad and Differential Dataflow. 14 | Microsoft Corporation 15 | en-US 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/Operators/Distinct.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | 26 | using System.Collections.Concurrent; 27 | using Microsoft.Research.Naiad.Dataflow; 28 | 29 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.Operators 30 | { 31 | internal class Distinct : OperatorImplementations.UnaryStatefulOperator 32 | where S : IEquatable 33 | where T : Microsoft.Research.Naiad.Time 34 | { 35 | protected override Int64 WeightFunction(Int64 weight) 36 | { 37 | return weight > 0 ? 1 : 0; 38 | } 39 | 40 | public Distinct(int index, Stage collection, bool inputImmutable) 41 | : base(index, collection, inputImmutable) 42 | { 43 | } 44 | 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/Operators/Intersect.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using Microsoft.Research.Naiad.Dataflow; 22 | using System; 23 | using System.Collections.Generic; 24 | using System.Linq; 25 | using System.Text; 26 | 27 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.Operators 28 | { 29 | class Intersect : OperatorImplementations.BinaryStatefulOperator 30 | where S : IEquatable 31 | where T : Microsoft.Research.Naiad.Time 32 | { 33 | protected override Int64 WeightFunction(Int64 weight1, Int64 weight2) 34 | { 35 | return Math.Min(weight1, weight2); 36 | } 37 | 38 | public Intersect(int index, Stage collection, bool input1Immutable, bool input2Immutable) 39 | : base(index, collection, input1Immutable, input2Immutable) 40 | { 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/Operators/SymmetricDifference.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using Microsoft.Research.Naiad.Dataflow; 22 | using System; 23 | using System.Collections.Generic; 24 | using System.Linq; 25 | using System.Text; 26 | 27 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.Operators 28 | { 29 | class SymmetricDifference : OperatorImplementations.BinaryStatefulOperator 30 | where S : IEquatable 31 | where T : Microsoft.Research.Naiad.Time 32 | { 33 | protected override Int64 WeightFunction(Int64 weight1, Int64 weight2) 34 | { 35 | return Math.Abs(weight1 - weight2); 36 | } 37 | 38 | public SymmetricDifference(int index, Stage collection, bool input1Immutable, bool input2Immutable) 39 | : base(index, collection, input1Immutable, input2Immutable) { } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/Operators/Concat.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | using Microsoft.Research.Naiad; 26 | using Microsoft.Research.Naiad.Dataflow; 27 | using Microsoft.Research.Naiad.Dataflow.StandardVertices; 28 | 29 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.Operators 30 | { 31 | internal class Concat : BinaryVertex, Weighted, Weighted, T> 32 | where S : IEquatable 33 | where T : Time 34 | { 35 | public override void OnReceive1(Message, T> elements) 36 | { 37 | this.Output.Send(elements); 38 | } 39 | 40 | public override void OnReceive2(Message, T> elements) 41 | { 42 | this.Output.Send(elements); 43 | } 44 | 45 | public Concat(int index, Stage collection) 46 | : base(index, collection) 47 | { 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /ClusterSubmission/ClusterSubmission.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 2013 4 | VisualStudioVersion = 12.0.30723.0 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NaiadPeloponneseSupport", "NaiadPeloponneseSupport\NaiadPeloponneseSupport.csproj", "{271F7100-7AA3-4379-9C58-23618B73A5DD}" 7 | EndProject 8 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Nuget common files", "Nuget common files", "{B867D391-330A-4D87-9C81-8FA4D9EE05F8}" 9 | ProjectSection(SolutionItems) = preProject 10 | ConfigFile.targets = ConfigFile.targets 11 | InstallConfigFile.ps1 = InstallConfigFile.ps1 12 | EndProjectSection 13 | EndProject 14 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RunNaiad", "RunNaiad\RunNaiad.csproj", "{47D22F4A-8B47-4829-A896-E5318ECA4CC2}" 15 | EndProject 16 | Global 17 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 18 | Debug|x64 = Debug|x64 19 | Release|x64 = Release|x64 20 | EndGlobalSection 21 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 22 | {271F7100-7AA3-4379-9C58-23618B73A5DD}.Debug|x64.ActiveCfg = Debug|x64 23 | {271F7100-7AA3-4379-9C58-23618B73A5DD}.Debug|x64.Build.0 = Debug|x64 24 | {271F7100-7AA3-4379-9C58-23618B73A5DD}.Release|x64.ActiveCfg = Release|x64 25 | {271F7100-7AA3-4379-9C58-23618B73A5DD}.Release|x64.Build.0 = Release|x64 26 | {47D22F4A-8B47-4829-A896-E5318ECA4CC2}.Debug|x64.ActiveCfg = Debug|x64 27 | {47D22F4A-8B47-4829-A896-E5318ECA4CC2}.Debug|x64.Build.0 = Debug|x64 28 | {47D22F4A-8B47-4829-A896-E5318ECA4CC2}.Release|x64.ActiveCfg = Release|x64 29 | {47D22F4A-8B47-4829-A896-E5318ECA4CC2}.Release|x64.Build.0 = Release|x64 30 | EndGlobalSection 31 | GlobalSection(SolutionProperties) = preSolution 32 | HideSolutionNode = FALSE 33 | EndGlobalSection 34 | EndGlobal 35 | -------------------------------------------------------------------------------- /Documentation/NaiadDocumentation/NaiadDocumentation/Content/VersionHistory/v0.3.aml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 9 | The first release of Naiad with support for running computations on Microsoft Azure HDInsight and Hadoop 2.0 (YARN), 10 | version 0.3 was released alongside binary packages on NuGet. 11 | 12 | 13 | 14 | 15 | Version 0.3 was released on April 4th 2014. 16 | 17 | 18 |
19 | Changes in This Release 20 | 21 | 22 | 23 | 24 | 25 | Added the ClusterSubmission solution, which contains support 26 | for launching Naiad programs in a variety of distributed settings, including: 27 | 28 | 29 | 30 | Microsoft Azure HDInsight 3.0. 31 | 32 | 33 | Hadoop 2.0 (YARN) on Windows. 34 | 35 | 36 | 37 | 38 | 39 | The AzureSupport framework makes it easier to use Naiad programs with Microsoft Azure Storage 40 | Blobs and Tables. 41 | 42 | 43 | 44 | 45 | 46 |
47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/Operators/Select.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | using System.Linq.Expressions; 26 | using Microsoft.Research.Naiad.Dataflow; 27 | using Microsoft.Research.Naiad.Dataflow.StandardVertices; 28 | 29 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.Operators 30 | { 31 | internal class Select : UnaryVertex, Weighted, T> 32 | where S : IEquatable 33 | where T : Time 34 | where R : IEquatable 35 | { 36 | public Func selector; 37 | 38 | public override void OnReceive(Message, T> message) 39 | { 40 | var output = this.Output.GetBufferForTime(message.time); 41 | for (int i = 0; i < message.length; i++) 42 | output.Send(this.selector(message.payload[i].record).ToWeighted(message.payload[i].weight)); 43 | } 44 | 45 | public Select(int index, Stage collection, Expression> transformation) 46 | : base(index, collection) 47 | { 48 | selector = transformation.Compile(); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /Documentation/NaiadDocumentation/NaiadDocumentation/ContentLayout.content: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/Operators/Except.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using Microsoft.Research.Naiad.Dataflow; 22 | using System; 23 | using System.Collections.Generic; 24 | using System.Linq; 25 | using System.Text; 26 | using Microsoft.Research.Naiad.Dataflow.StandardVertices; 27 | 28 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.Operators 29 | { 30 | internal class Except : BinaryVertex, Weighted, Weighted, T> 31 | where S : IEquatable 32 | where T : Time 33 | { 34 | public override void OnReceive1(Message, T> message) 35 | { 36 | this.Output.Send(message); 37 | } 38 | 39 | public override void OnReceive2(Message, T> message) 40 | { 41 | var outputBuffer = this.Output.GetBufferForTime(message.time); 42 | 43 | for (int i = 0; i < message.length; i++) 44 | { 45 | var record = message.payload[i]; 46 | record.weight *= -1; 47 | 48 | outputBuffer.Send(record); 49 | } 50 | } 51 | 52 | public Except(int index, Stage collection) 53 | : base(index, collection) 54 | { 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/Operators/Where.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | 26 | using System.Collections.Concurrent; 27 | using System.Linq.Expressions; 28 | using Microsoft.Research.Naiad.Dataflow; 29 | using Microsoft.Research.Naiad.Dataflow.StandardVertices; 30 | 31 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.Operators 32 | { 33 | 34 | internal class Where : UnaryVertex, Weighted, T> 35 | where S : IEquatable 36 | where T : Microsoft.Research.Naiad.Time 37 | { 38 | protected Func predicate; 39 | 40 | public override void OnReceive(Message, T> message) 41 | { 42 | var output = this.Output.GetBufferForTime(message.time); 43 | for (int i = 0; i < message.length; i++) 44 | if (predicate(message.payload[i].record)) 45 | output.Send(message.payload[i]); 46 | } 47 | 48 | public Where(int index, Stage collection, Expression> pred) 49 | : base(index, collection) 50 | { 51 | predicate = pred.Compile(); 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/Operators/AdjustLattice.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using Microsoft.Research.Naiad.Dataflow; 22 | using Microsoft.Research.Naiad.Dataflow.StandardVertices; 23 | 24 | using System; 25 | using System.Collections.Generic; 26 | using System.Linq; 27 | using System.Text; 28 | 29 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.Operators 30 | { 31 | internal class AdjustTime : UnaryVertex, Weighted, T> 32 | where T : Time 33 | where R : IEquatable 34 | { 35 | public Func adjustment; 36 | 37 | public override void OnReceive(Message, T> message) 38 | { 39 | for (int i = 0; i < message.length; i++) 40 | { 41 | var record = message.payload[i]; 42 | var adjusted = adjustment(record.record, message.time); 43 | var newTime = adjusted.Join(message.time); 44 | this.Output.GetBufferForTime(newTime).Send(message.payload[i]); 45 | } 46 | } 47 | 48 | public AdjustTime(int index, Stage collection, Func transformation) 49 | : base(index, collection) 50 | { 51 | adjustment = transformation; 52 | } 53 | 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /ClusterSubmission/RunNaiad/packages.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /ClusterSubmission/NaiadPeloponneseSupport/packages.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /Frameworks/Lindi/Microsoft.Research.Naiad.Lindi.nuspec: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Microsoft.Research.Naiad.Lindi 5 | Naiad - Lindi framework 6 | 0.5.0-beta 7 | naiadquestions@microsoft.com 8 | naiadquestions@microsoft.com,Microsoft 9 | http://www.apache.org/licenses/LICENSE-2.0.html 10 | http://research.microsoft.com/naiad/ 11 | 12 | true 13 | Lindi is a simple LINQ-like programming framework for Naiad. 14 | Microsoft Corporation 15 | en-US 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /Naiad/Microsoft.Research.Naiad.nuspec: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Microsoft.Research.Naiad 5 | Naiad - Core 6 | 0.5.0-beta 7 | naiadquestions@microsoft.com 8 | naiadquestions@microsoft.com,Microsoft 9 | http://www.apache.org/licenses/LICENSE-2.0.html 10 | http://research.microsoft.com/naiad/ 11 | 12 | true 13 | Naiad: a distributed framework for incremental, iterative, and interactive programs. 14 | Microsoft Corporation 15 | en-US 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /Naiad/Dataflow/Stream.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Linq.Expressions; 25 | using System.Text; 26 | 27 | using Microsoft.Research.Naiad.Dataflow; 28 | 29 | namespace Microsoft.Research.Naiad 30 | { 31 | /// 32 | /// Represents a stream of records each tagged with a time. 33 | /// 34 | /// record type 35 | /// time type 36 | public class Stream where TTime : Time 37 | { 38 | internal readonly StageOutput StageOutput; 39 | 40 | /// 41 | /// Expression indicating a partitioning property the stream obeys, or null if none exists. 42 | /// 43 | public Expression> PartitionedBy { get { return this.StageOutput.PartitionedBy; } } 44 | 45 | /// 46 | /// Stage the stream is produced by. 47 | /// 48 | public Dataflow.Stage ForStage { get { return this.StageOutput.ForStage; } } 49 | 50 | /// 51 | /// Time context for the stream. 52 | /// 53 | public Dataflow.TimeContext Context { get { return this.StageOutput.Context; } } 54 | 55 | internal Stream(StageOutput stageOutput) 56 | { 57 | this.StageOutput = stageOutput; 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /Frameworks/WorkGenerator/Microsoft.Research.Naiad.WorkGenerator.nuspec: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Microsoft.Research.Naiad.WorkGenerator 5 | Naiad - Work queue library 6 | 0.5.0-beta 7 | naiadquestions@microsoft.com 8 | naiadquestions@microsoft.com,Microsoft 9 | http://www.apache.org/licenses/LICENSE-2.0.html 10 | http://research.microsoft.com/naiad/ 11 | 12 | true 13 | Lindi is a simple LINQ-like programming framework for Naiad. 14 | Microsoft Corporation 15 | en-US 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /ClusterSubmission/RunNaiad/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | using System.Reflection; 21 | using System.Runtime.CompilerServices; 22 | using System.Runtime.InteropServices; 23 | 24 | // General Information about an assembly is controlled through the following 25 | // set of attributes. Change these attribute values to modify the information 26 | // associated with an assembly. 27 | [assembly: AssemblyTitle("RunNaiad")] 28 | [assembly: AssemblyDescription("")] 29 | [assembly: AssemblyConfiguration("")] 30 | [assembly: AssemblyCompany("")] 31 | [assembly: AssemblyProduct("RunNaiad")] 32 | [assembly: AssemblyCopyright("Copyright © 2014")] 33 | [assembly: AssemblyTrademark("")] 34 | [assembly: AssemblyCulture("")] 35 | 36 | // Setting ComVisible to false makes the types in this assembly not visible 37 | // to COM components. If you need to access a type in this assembly from 38 | // COM, set the ComVisible attribute to true on that type. 39 | [assembly: ComVisible(false)] 40 | 41 | // The following GUID is for the ID of the typelib if this project is exposed to COM 42 | [assembly: Guid("9bc369a3-3588-49fc-9ef1-f325702c8175")] 43 | 44 | // Version information for an assembly consists of the following four values: 45 | // 46 | // Major Version 47 | // Minor Version 48 | // Build Number 49 | // Revision 50 | // 51 | // You can specify all the values or you can default the Build and Revision Numbers 52 | // by using the '*' as shown below: 53 | // [assembly: AssemblyVersion("1.0.*")] 54 | [assembly: AssemblyVersion("0.5.0.0")] 55 | [assembly: AssemblyFileVersion("0.5.0.0")] 56 | -------------------------------------------------------------------------------- /Frameworks/GraphLINQ/Microsoft.Research.Naiad.GraphLINQ.nuspec: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Microsoft.Research.Naiad.GraphLINQ 5 | Naiad - GraphLINQ framework 6 | 0.5.0-beta 7 | naiadquestions@microsoft.com 8 | naiadquestions@microsoft.com,Microsoft 9 | http://www.apache.org/licenses/LICENSE-2.0.html 10 | http://research.microsoft.com/naiad/ 11 | 12 | true 13 | GraphLINQ is a LINQ-like programming framework for Naiad supporting graph-specific optimizations for iterative applications. 14 | Microsoft Corporation 15 | en-US 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /ClusterSubmission/NaiadPeloponneseSupport/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System.Reflection; 22 | using System.Runtime.CompilerServices; 23 | using System.Runtime.InteropServices; 24 | 25 | // General Information about an assembly is controlled through the following 26 | // set of attributes. Change these attribute values to modify the information 27 | // associated with an assembly. 28 | [assembly: AssemblyTitle("Microsoft.Research.Naiad.Cluster")] 29 | [assembly: AssemblyDescription("")] 30 | [assembly: AssemblyConfiguration("")] 31 | [assembly: AssemblyCompany("")] 32 | [assembly: AssemblyProduct("Microsoft.Research.Naiad.Cluster")] 33 | [assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")] 34 | [assembly: AssemblyTrademark("")] 35 | [assembly: AssemblyCulture("")] 36 | 37 | // Setting ComVisible to false makes the types in this assembly not visible 38 | // to COM components. If you need to access a type in this assembly from 39 | // COM, set the ComVisible attribute to true on that type. 40 | [assembly: ComVisible(false)] 41 | 42 | // The following GUID is for the ID of the typelib if this project is exposed to COM 43 | [assembly: Guid("971b8836-7751-4a6c-8721-50a0db985a43")] 44 | 45 | // Version information for an assembly consists of the following four values: 46 | // 47 | // Major Version 48 | // Minor Version 49 | // Build Number 50 | // Revision 51 | // 52 | // You can specify all the values or you can default the Build and Revision Numbers 53 | // by using the '*' as shown below: 54 | // [assembly: AssemblyVersion("1.0.*")] 55 | [assembly: AssemblyVersion("0.5.0.9")] 56 | [assembly: AssemblyFileVersion("0.5.0.0")] 57 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/Operators/Prioritize.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | using Microsoft.Research.Naiad; 26 | 27 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.Operators.Prioritize 28 | { 29 | #if false 30 | internal class Ingress : OperatorImplementations.UnaryStatelessOperator> 31 | where S : IEquatable 32 | where T : Naiad.Lattice 33 | { 34 | Func priorityFunction; 35 | 36 | public override void OnRecv(NaiadRecord element) 37 | { 38 | Send(element.record.ToNaiadRecord(element.weight, new IntTotalLattice(element.time, priorityFunction(element.record)))); 39 | } 40 | 41 | public Ingress(int index, UnaryCollectionVertex> collection, Func p) 42 | : base(index, collection, x => new IntTotalLattice(x, 0)) 43 | { 44 | priorityFunction = p; 45 | } 46 | } 47 | 48 | internal class Egress : OperatorImplementations.UnaryStatelessOperator, S, T> 49 | where S : IEquatable 50 | where T : Naiad.Lattice 51 | { 52 | public override void OnRecv(NaiadRecord> element) 53 | { 54 | Send(element.record.ToNaiadRecord(element.weight, element.time.s)); 55 | } 56 | 57 | public Egress(int index, UnaryCollectionVertex collection) 58 | : base(index, collection, x => x.s) 59 | { 60 | } 61 | } 62 | #endif 63 | } 64 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/Operators/Aggregate.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | using System.Linq.Expressions; 26 | using Microsoft.Research.Naiad.DataStructures; 27 | using Microsoft.Research.Naiad.Dataflow; 28 | 29 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.Operators 30 | { 31 | internal class Aggregate : OperatorImplementations.UnaryStatefulOperatorWithAggregation 32 | where S : IEquatable 33 | where R : IEquatable 34 | where T : Time 35 | where K : IEquatable 36 | where V : IEquatable 37 | { 38 | private readonly Func reducer; 39 | 40 | protected override void Reduce(K key, UnaryKeyIndices keyIndices, int time) 41 | { 42 | collection.Clear(); 43 | inputTrace.EnumerateCollectionAt(keyIndices.processed, time, collection); 44 | 45 | for (int i = 0; i < collection.Count; i++) 46 | if (collection.Array[i].weight != 0) 47 | { 48 | var result = reducer(key, collection.Array[i].record); 49 | 50 | outputTrace.Introduce(ref outputWorkspace, result, 1, time); 51 | } 52 | } 53 | 54 | public Aggregate(int i, Stage c, bool inputImmutable, Expression> k, Expression> v, Expression> axpy, Expression> isZero, Expression> r) 55 | : base(i, c, inputImmutable, k, v, axpy, isZero) 56 | { 57 | reducer = r.Compile(); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/OperatorImplementations/UnaryStatefulWithAggregation.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | 26 | using System.Linq.Expressions; 27 | 28 | using Microsoft.Research.Naiad.DataStructures; 29 | using Microsoft.Research.Naiad.Dataflow.Channels; 30 | using Microsoft.Research.Naiad.Scheduling; 31 | using Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.CollectionTrace; 32 | using Microsoft.Research.Naiad; 33 | using Microsoft.Research.Naiad.Dataflow; 34 | 35 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.OperatorImplementations 36 | { 37 | internal class UnaryStatefulOperatorWithAggregation : OperatorImplementations.UnaryStatefulOperator 38 | where K : IEquatable 39 | where V : IEquatable 40 | where S : IEquatable 41 | where T : Time 42 | where R : IEquatable 43 | { 44 | Func axpy; 45 | Func isZero; 46 | 47 | protected override CollectionTraceCheckpointable createInputTrace() 48 | { 49 | return new CollectionTraceWithAggregation((i, j) => this.internTable.LessThan(i, j), i => this.internTable.UpdateTime(i), axpy, isZero); 50 | } 51 | 52 | public UnaryStatefulOperatorWithAggregation(int index, Stage collection, bool immutableInput, Expression> k, Expression> v, Expression> a, Expression> i) 53 | : base(index, collection, immutableInput, k, v) 54 | { 55 | axpy = a.Compile(); 56 | isZero = i.Compile(); 57 | 58 | inputTrace = createInputTrace(); 59 | } 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/Microsoft.Research.Naiad.DifferentialDataflow.nuspec: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Microsoft.Research.Naiad.DifferentialDataflow 5 | Naiad - Differential Dataflow framework 6 | 0.5.0-beta 7 | naiadquestions@microsoft.com 8 | naiadquestions@microsoft.com,Microsoft 9 | http://www.apache.org/licenses/LICENSE-2.0.html 10 | http://research.microsoft.com/naiad/ 11 | 12 | true 13 | Differential dataflow is a LINQ-like programming framework for Naiad supporting incremental and iterative programs. 14 | Microsoft Corporation 15 | en-US 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /ClusterSubmission/RunNaiad/Microsoft.Research.Naiad.ClusterSubmission.nuspec: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Microsoft.Research.Naiad.ClusterSubmission 5 | 0.5.0-beta 6 | Naiad - cluster submission tool 7 | naiadquestions@microsoft.com 8 | naiadquestions@microsoft.com,Microsoft 9 | http://www.apache.org/licenses/LICENSE-2.0.html 10 | http://research.microsoft.com/naiad/ 11 | true 12 | 13 | Launcher for submitting Naiad programs to clusters including YARN and Azure HDInsight. 14 | 15 | Microsoft Corporation 16 | en-US 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/Operators/GroupBy.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | 26 | using System.Collections.Concurrent; 27 | using System.Linq.Expressions; 28 | using Microsoft.Research.Naiad.DataStructures; 29 | using Microsoft.Research.Naiad.Dataflow; 30 | 31 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.Operators 32 | { 33 | internal class GroupBy : OperatorImplementations.ConservativeUnaryStatefulOperator 34 | where K : IEquatable 35 | where V : IEquatable 36 | where S : IEquatable 37 | where T : Microsoft.Research.Naiad.Time 38 | where R : IEquatable 39 | { 40 | public Func, IEnumerable> reducer; 41 | 42 | protected IEnumerable EnumerateCollection() 43 | { 44 | for (int i = 0; i < collection.Count; i++) 45 | for (int j = 0; j < collection.Array[i].weight; j++) 46 | yield return collection.Array[i].record; 47 | } 48 | 49 | protected override void Reduce(K key, UnaryKeyIndices keyIndices, int time) 50 | { 51 | collection.Clear(); 52 | inputTrace.EnumerateCollectionAt(keyIndices.processed, time, collection); 53 | 54 | if (collection.Count > 0) 55 | foreach (var r in reducer(key, EnumerateCollection())) 56 | outputTrace.Introduce(ref outputWorkspace, r, 1, time); 57 | } 58 | 59 | public GroupBy(int index, Stage collection, bool immutableInput, Expression> k, Expression> v, Func, IEnumerable> r) 60 | : base(index, collection, immutableInput, k, v) 61 | { 62 | reducer = r; 63 | } 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /Frameworks/Storage/Microsoft.Research.Naiad.Storage.nuspec: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Microsoft.Research.Naiad.Storage 5 | Naiad - Shared Storage support 6 | 0.5.0-beta 7 | naiadquestions@microsoft.com 8 | naiadquestions@microsoft.com,Microsoft 9 | http://www.apache.org/licenses/LICENSE-2.0.html 10 | http://research.microsoft.com/naiad/ 11 | 12 | true 13 | Provides convenient access to Windows Azure Storage APIs from Naiad programs. 14 | Microsoft Corporation 15 | en-US 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /Frameworks/HdfsSupport/Microsoft.Research.Naiad.HdfsSupport.nuspec: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Microsoft.Research.Naiad.HdfsSupport 5 | Naiad - Hdfs Storage support 6 | 0.5.0-beta 7 | naiadquestions@microsoft.com 8 | naiadquestions@microsoft.com,Microsoft 9 | http://www.apache.org/licenses/LICENSE-2.0.html 10 | http://research.microsoft.com/naiad/ 11 | 12 | true 13 | Provides convenient access to Windows Azure Storage APIs from Naiad programs. 14 | Microsoft Corporation 15 | en-US 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/Operators/Count.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | using System.Linq.Expressions; 26 | using Microsoft.Research.Naiad.DataStructures; 27 | using Microsoft.Research.Naiad.Dataflow; 28 | 29 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.Operators 30 | { 31 | internal class Count : OperatorImplementations.UnaryStatefulOperator 32 | where K : IEquatable 33 | where S : IEquatable 34 | where T : Microsoft.Research.Naiad.Time 35 | where R : IEquatable 36 | { 37 | public Func reducer; 38 | protected override void NewOutputMinusOldOutput(K key, UnaryKeyIndices keyIndices, int timeIndex) 39 | { 40 | var newSum = 0L; 41 | collection.Clear(); 42 | inputTrace.EnumerateCollectionAt(keyIndices.processed, timeIndex, collection); 43 | for (int i = 0; i < collection.Count; i++) 44 | newSum += collection.Array[i].weight; 45 | 46 | var oldSum = newSum; 47 | difference.Clear(); 48 | inputTrace.EnumerateCollectionAt(keyIndices.unprocessed, timeIndex, difference); 49 | for (int i = 0; i < difference.Count; i++) 50 | oldSum -= difference.Array[i].weight; 51 | 52 | if (oldSum != newSum) 53 | { 54 | if (oldSum > 0) 55 | outputTrace.Introduce(ref outputWorkspace, reducer(key, oldSum), -1, timeIndex); 56 | if (newSum > 0) 57 | outputTrace.Introduce(ref outputWorkspace, reducer(key, newSum), +1, timeIndex); 58 | } 59 | } 60 | 61 | public Count(int index, Stage collection, bool inputImmutable, Expression> k, Expression> r) 62 | : base(index, collection, inputImmutable, k, k) 63 | { 64 | reducer = r.Compile(); 65 | } 66 | } 67 | } 68 | 69 | -------------------------------------------------------------------------------- /Frameworks/AzureSupport/Console.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using Microsoft.WindowsAzure.Storage.Blob; 22 | using System; 23 | using System.Collections.Generic; 24 | using System.Linq; 25 | using System.Text; 26 | 27 | namespace Microsoft.Research.Naiad.Frameworks.Azure 28 | { 29 | /// 30 | /// Extension methods 31 | /// 32 | public static class ConsoleExtensionMethods 33 | { 34 | /// 35 | /// Redirects Console.Out to an Azure blob, with the process id in the filename. 36 | /// 37 | /// Naiad controller 38 | /// Azure container 39 | /// Format string expecting the Naiad process id 40 | public static void SetConsoleOut(this Microsoft.Research.Naiad.Controller controller, CloudBlobContainer container, string format) 41 | { 42 | var filename = string.Format(format, controller.Configuration.ProcessID); 43 | 44 | var writer = new System.IO.StreamWriter(container.GetBlockBlobReference(filename).OpenWrite()); 45 | writer.AutoFlush = false; 46 | 47 | Console.SetOut(writer); 48 | } 49 | 50 | /// 51 | /// Redirects Console.Error to an Azure blob, with the process id in the filename. 52 | /// 53 | /// Naiad controller 54 | /// Azure container 55 | /// Format string expecting the Naiad process id 56 | public static void SetConsoleError(this Microsoft.Research.Naiad.Controller controller, CloudBlobContainer container, string format) 57 | { 58 | var filename = string.Format(format, controller.Configuration.ProcessID); 59 | 60 | var writer = new System.IO.StreamWriter(container.GetBlockBlobReference(filename).OpenWrite()); 61 | writer.AutoFlush = false; 62 | 63 | Console.SetError(writer); 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /Frameworks/WebHdfsSupport/Microsoft.Research.Naiad.WebHdfsSupport.nuspec: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Microsoft.Research.Naiad.WebHdfs 5 | Naiad - WebHdfs Storage support 6 | 0.5.0-beta 7 | naiadquestions@microsoft.com 8 | naiadquestions@microsoft.com,Microsoft 9 | http://www.apache.org/licenses/LICENSE-2.0.html 10 | http://research.microsoft.com/naiad/ 11 | 12 | true 13 | Provides convenient access to Windows Azure Storage APIs from Naiad programs. 14 | Microsoft Corporation 15 | en-US 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/CollectionTrace/OffsetLength.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | 26 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.CollectionTrace 27 | { 28 | /// 29 | /// Munge together an index into the heaps array (length) and an offset 30 | /// in that array (offset) into a single integer. 31 | /// 32 | /// The top bit is reserved for flagging merged state offsets. 33 | /// 34 | internal struct OffsetLength : IEquatable 35 | { 36 | public int offsetLength; 37 | 38 | public void GetOffsetLength(out int offset, out int length) 39 | { 40 | length = 0; 41 | while ((offsetLength >> length) % 2 == 0) 42 | length++; 43 | 44 | offset = offsetLength >> (length + 1); 45 | } 46 | 47 | int Offset 48 | { 49 | get 50 | { 51 | int o, l; 52 | GetOffsetLength(out o, out l); 53 | return o; 54 | } 55 | } 56 | 57 | public int Length 58 | { 59 | get 60 | { 61 | int o, l; 62 | GetOffsetLength(out o, out l); 63 | return l; 64 | } 65 | } 66 | 67 | public override string ToString() 68 | { 69 | if (this.offsetLength == 0) 70 | return "Null"; 71 | else 72 | return String.Format("[o:{0}, l:{1}]", Offset, Length); 73 | } 74 | 75 | public bool Equals(OffsetLength other) 76 | { 77 | return this.offsetLength == other.offsetLength; 78 | } 79 | 80 | public bool IsEmpty { get { return offsetLength == 0; } } 81 | 82 | public OffsetLength(int ol) 83 | { 84 | offsetLength = ol; 85 | } 86 | 87 | public OffsetLength(int o, int l) 88 | { 89 | offsetLength = (o << (l + 1)) + (1 << l); 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/Operators/Monitor.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | 26 | using System.Diagnostics; 27 | using System.Runtime.InteropServices; 28 | using System.Threading; 29 | 30 | using Microsoft.Research.Naiad.Utilities; 31 | using Microsoft.Research.Naiad.Dataflow; 32 | using Microsoft.Research.Naiad.Dataflow.StandardVertices; 33 | 34 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.Operators 35 | { 36 | internal class Monitor : UnaryVertex, Weighted, T> 37 | where R : IEquatable 38 | where T : Time 39 | { 40 | public Action,T>>> action; 41 | public List,T>> list; 42 | 43 | Int64 count; 44 | T leastTime; 45 | 46 | 47 | public override void OnReceive(Message, T> message) 48 | { 49 | this.NotifyAt(message.time); 50 | if (count == 0 || leastTime.CompareTo(message.time) > 0) 51 | leastTime = message.time; 52 | 53 | for (int i = 0; i < message.length; i++) 54 | { 55 | if (action != null) 56 | list.Add(message.payload[i].PairWith(message.time)); 57 | 58 | count++; 59 | } 60 | 61 | this.Output.Send(message); 62 | } 63 | 64 | public override void OnNotify(T time) 65 | { 66 | if (action != null) 67 | { 68 | action(VertexId, list); 69 | list.Clear(); 70 | } 71 | else 72 | { 73 | Console.WriteLine("{0}\t{1}\t{2}\t{3}", this.VertexId, count, leastTime, System.DateTime.Now); 74 | 75 | count = 0; 76 | } 77 | } 78 | 79 | public Monitor(int index, Stage collection, bool immutableInput, Action,T>>> a) : base(index, collection) 80 | { 81 | action = a; 82 | list = new List,T>>(); 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /ClusterSubmission/RunNaiad/YarnSubmission.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using Microsoft.Research.Peloponnese.Yarn; 22 | using Microsoft.Research.Peloponnese.Shared; 23 | using System; 24 | using System.Collections.Generic; 25 | using System.IO; 26 | using System.Linq; 27 | using System.Text; 28 | using System.Threading.Tasks; 29 | 30 | namespace Microsoft.Research.Naiad.Cluster.Submission 31 | { 32 | class NativeYarnSubmission : ClusterSubmission 33 | { 34 | public NativeYarnSubmission( 35 | string rmNode, int wsPort, DfsClient dfsClient, string queueName, Uri stagingUri, Uri jobUri, string launcherNode, int launcherPort, 36 | int amMemoryInMB, int numberOfProcesses, int workerMemoryInMB, string[] args) 37 | : base(new NativeYarnClient(rmNode, wsPort, dfsClient, jobUri, launcherNode, launcherPort), 38 | stagingUri, queueName, amMemoryInMB, numberOfProcesses, workerMemoryInMB, args) 39 | { 40 | } 41 | 42 | public NativeYarnSubmission( 43 | string rmNode, int wsPort, DfsClient dfsClient, string queueName, Uri stagingUri, Uri jobUri, string peloponneseDirectory, 44 | int amMemoryInMB, int numberOfProcesses, int workerMemoryInMB, string[] args) 45 | : base( 46 | new NativeYarnClient(rmNode, wsPort, dfsClient, jobUri, LauncherJarFile(peloponneseDirectory), YarnDirectory()), 47 | stagingUri, queueName, amMemoryInMB, numberOfProcesses, workerMemoryInMB, args) 48 | { 49 | } 50 | 51 | private static string LauncherJarFile(string peloponneseDirectory) 52 | { 53 | return Path.Combine(peloponneseDirectory, "Microsoft.Research.Peloponnese.YarnLauncher.jar"); 54 | } 55 | 56 | private static string YarnDirectory() 57 | { 58 | string yarnDirectory = Environment.GetEnvironmentVariable("HADOOP_COMMON_HOME"); 59 | 60 | if (yarnDirectory == null) 61 | { 62 | throw new ApplicationException("No HADOOP_COMMON_HOME defined"); 63 | } 64 | 65 | return yarnDirectory; 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /Frameworks/AzureSupport/Microsoft.Research.Naiad.AzureSupport.nuspec: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Microsoft.Research.Naiad.AzureSupport 5 | Naiad - Windows Azure Storage support 6 | 0.5.0-beta 7 | naiadquestions@microsoft.com 8 | naiadquestions@microsoft.com,Microsoft 9 | http://www.apache.org/licenses/LICENSE-2.0.html 10 | http://research.microsoft.com/naiad/ 11 | 12 | true 13 | Provides convenient access to Windows Azure Storage APIs from Naiad programs. 14 | Microsoft Corporation 15 | en-US 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /Naiad/Runtime/Controlling/InternalController.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Diagnostics; 24 | using System.IO; 25 | using System.Linq; 26 | using System.Linq.Expressions; 27 | using System.Net.Sockets; 28 | using System.Text; 29 | using Microsoft.Research.Naiad.Dataflow.Channels; 30 | using Microsoft.Research.Naiad.Serialization; 31 | using Microsoft.Research.Naiad.Frameworks; 32 | using Microsoft.Research.Naiad.Scheduling; 33 | using System.Collections.Concurrent; 34 | using Microsoft.Research.Naiad.Runtime.Networking; 35 | using System.Threading; 36 | 37 | namespace Microsoft.Research.Naiad.Runtime.Controlling 38 | { 39 | internal interface InternalWorkerGroup : WorkerGroup 40 | { 41 | void Start(); 42 | void WakeUp(); 43 | void Abort(); 44 | void Activate(); 45 | void Pause(); 46 | void Resume(); 47 | 48 | Scheduler this[int index] { get; } 49 | 50 | void NotifyWorkerStarting(Scheduler scheduler); 51 | void NotifyWorkerWaking(Scheduler scheduler); 52 | void NotifyVertexStarting(Scheduler scheduler, Scheduler.WorkItem work); 53 | void NotifyVertexEnding(Scheduler scheduler, Scheduler.WorkItem work); 54 | void NotifyVertexEnqueued(Scheduler scheduler, Scheduler.WorkItem work); 55 | void NotifySchedulerSleeping(Scheduler scheduler); 56 | void NotifySchedulerTerminating(Scheduler scheduler); 57 | #if false 58 | void NotifyOperatorReceivedRecords(Dataflow.Vertex op, int channelId, int recordsReceived); 59 | void NotifyOperatorSentRecords(Dataflow.Vertex op, int channelId, int recordsSent); 60 | #endif 61 | 62 | /// 63 | /// Blocks the scheduler waiting for the event to be signalled. 64 | /// Used in broadcast wakeup implementation. 65 | /// 66 | /// 67 | /// 68 | /// 69 | long BlockScheduler(AutoResetEvent selectiveEvent, long val); 70 | } 71 | 72 | #if false 73 | public interface InternalController : IDisposable 74 | #endif 75 | } 76 | -------------------------------------------------------------------------------- /ClusterSubmission/RunNaiad/App.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/Operators/Max.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | using System.Linq.Expressions; 26 | using Microsoft.Research.Naiad.DataStructures; 27 | using Microsoft.Research.Naiad.Dataflow; 28 | 29 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.Operators 30 | { 31 | internal class Max : OperatorImplementations.ConservativeUnaryStatefulOperator 32 | where K : IEquatable 33 | where V : IEquatable 34 | where S : IEquatable 35 | where M : IComparable 36 | where T : Microsoft.Research.Naiad.Time 37 | { 38 | Func valueSelector; 39 | Func resultSelector; 40 | 41 | protected override void Reduce(K key, UnaryKeyIndices keyIndices, int timeIndex) 42 | { 43 | var maxFound = false; 44 | var maxValue = default(M); 45 | var maxEntry = default(V); 46 | 47 | collection.Clear(); 48 | inputTrace.EnumerateCollectionAt(keyIndices.processed, timeIndex, collection); 49 | 50 | for (int i = 0; i < collection.Count; i++) 51 | { 52 | var element = collection.Array[i]; 53 | 54 | if (element.weight > 0) 55 | { 56 | var value = valueSelector(key, element.record); 57 | if (maxFound == false || maxValue.CompareTo(value) < 0) 58 | { 59 | maxFound = true; 60 | maxValue = value; 61 | maxEntry = element.record; 62 | } 63 | } 64 | } 65 | 66 | if (maxFound) 67 | outputTrace.Introduce(ref outputWorkspace, resultSelector(key, maxEntry), 1, timeIndex); 68 | } 69 | 70 | public Max(int index, Stage collection, bool inputImmutable, Expression> k, Expression> e, Expression> v, Expression> r) 71 | : base(index, collection, inputImmutable, k, e) 72 | { 73 | valueSelector = v.Compile(); 74 | resultSelector = r.Compile(); 75 | } 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/CoreGenerics.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | 26 | using System.IO; 27 | using Microsoft.Research.Naiad; 28 | 29 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow 30 | { 31 | /// 32 | /// A record with a signed 64-bit weight, which corresponds to the multiplicity of the record 33 | /// in a multiset. 34 | /// 35 | /// The type of the record. 36 | public struct Weighted : IEquatable> where TRecord : IEquatable 37 | { 38 | /// 39 | /// The record. 40 | /// 41 | public TRecord record; 42 | 43 | /// 44 | /// The weight. 45 | /// 46 | public Int64 weight; 47 | 48 | /// 49 | /// Returns a string representation of this weighted record. 50 | /// 51 | /// A string representation of this weighted record. 52 | public override string ToString() 53 | { 54 | if (weight > 0) 55 | return String.Format("[ {0}, +{1} ]", record.ToString(), weight); 56 | else 57 | return String.Format("[ {0}, {1} ]", record.ToString(), weight); 58 | } 59 | 60 | /// 61 | /// Returns true if and only if this and the object have equal records and weights. 62 | /// 63 | /// The other object. 64 | /// true if and only if this and the object have equal records and weights. 65 | public bool Equals(Weighted other) 66 | { 67 | return this.weight == other.weight && this.record.Equals(other.record); 68 | } 69 | 70 | /// 71 | /// Constructs a new weighted object from the given and . 72 | /// 73 | /// The record. 74 | /// The weight. 75 | public Weighted(TRecord record, Int64 weight) { this.record = record; this.weight = weight; } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /ClusterSubmission/NaiadPeloponneseSupport/PPMSubmission.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {271F7100-7AA3-4379-9C58-23618B73A5DD} 8 | Exe 9 | Properties 10 | Naiad.Util 11 | PPMSubmission 12 | v4.5 13 | 512 14 | 15 | 16 | true 17 | bin\x64\Debug\ 18 | DEBUG;TRACE 19 | full 20 | x64 21 | prompt 22 | MinimumRecommendedRules.ruleset 23 | true 24 | 25 | 26 | bin\x64\Release\ 27 | TRACE 28 | true 29 | pdbonly 30 | x64 31 | prompt 32 | MinimumRecommendedRules.ruleset 33 | true 34 | 35 | 36 | 37 | ..\..\peloponnese\bin\Release\Microsoft.Research.Peloponnese.Utils.dll 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 66 | -------------------------------------------------------------------------------- /Naiad/Util/DependencyLister.cs: -------------------------------------------------------------------------------- 1 | /* Naiad ver. 0.4 2 | * Copyright (c) Microsoft Corporation 3 | * All rights reserved. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 12 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 13 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 14 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 15 | * 16 | * See the Apache Version 2.0 License for specific language governing 17 | * permissions and limitations under the License. 18 | */ 19 | 20 | using System; 21 | using System.Collections.Generic; 22 | using System.Linq; 23 | using System.Reflection; 24 | using System.Text; 25 | using System.Threading.Tasks; 26 | 27 | namespace Microsoft.Research.Naiad.Utilities 28 | { 29 | internal class DependencyLister : MarshalByRefObject 30 | { 31 | private static string[] FrameworkAssemblyNames = { "System", "System.Core", "mscorlib", "System.Xml" }; 32 | 33 | /// 34 | /// Returns the non-framework assemblies on which a given assembly depends. 35 | /// 36 | /// The initial assembly 37 | /// A set of non-framework assemblies on which the given assembly depends 38 | private static HashSet Dependencies(Assembly source) 39 | { 40 | HashSet visited = new HashSet(); 41 | Queue assemblyQueue = new Queue(); 42 | assemblyQueue.Enqueue(source); 43 | visited.Add(source); 44 | 45 | while (assemblyQueue.Count > 0) 46 | { 47 | Assembly currentAssembly = assemblyQueue.Dequeue(); 48 | 49 | foreach (AssemblyName name in currentAssembly.GetReferencedAssemblies()) 50 | { 51 | Assembly referencedAssembly = Assembly.Load(name); 52 | if (!visited.Contains(referencedAssembly) && !FrameworkAssemblyNames.Contains(name.Name) && !(name.Name.StartsWith("System"))) 53 | { 54 | visited.Add(referencedAssembly); 55 | assemblyQueue.Enqueue(referencedAssembly); 56 | } 57 | } 58 | } 59 | return visited; 60 | } 61 | 62 | /// 63 | /// Returns the locations of non-framework assemblies on which the assembly with the given filename depends. 64 | /// 65 | /// The filename of the assembly 66 | /// An array of filenames for non-framework assemblies on which the given assembly depends 67 | public string[] ListDependencies(string assemblyFilename) 68 | { 69 | Assembly assembly = Assembly.LoadFrom(assemblyFilename); 70 | return DependencyLister.Dependencies(assembly).Select(x => x.Location).ToArray(); 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /Examples/DifferentialDataflow/WordCount.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | 26 | using Microsoft.Research.Naiad; 27 | using Microsoft.Research.Naiad.Frameworks.DifferentialDataflow; 28 | 29 | namespace Microsoft.Research.Naiad.Examples.DifferentialDataflow 30 | { 31 | /// 32 | /// Demonstrates an interactive Naiad computation. 33 | /// 34 | public class WordCount : Example 35 | { 36 | /// 37 | /// Executes a word counting Naiad program. 38 | /// 39 | /// Naiad controller configuration 40 | /// Remaining arguments 41 | public void Execute(string[] args) 42 | { 43 | // first, construct a Naiad controller. 44 | using (var computation = NewComputation.FromArgs(ref args)) 45 | { 46 | // create an incrementally updateable collection 47 | var text = computation.NewInputCollection(); 48 | 49 | // segment strings, count, and print 50 | text.SelectMany(x => x.Split(' ')) 51 | .Count(y => y, (k, c) => k + ":" + c) // yields "word:count" for each word 52 | .Subscribe(l => { foreach (var element in l) Console.WriteLine(element); }); 53 | 54 | computation.Activate(); 55 | 56 | if (computation.Configuration.ProcessID == 0) 57 | { 58 | Console.WriteLine("Start entering lines of text. An empty line will exit the program."); 59 | Console.WriteLine("Naiad will display counts (and changes in counts) of words you type."); 60 | 61 | var line = Console.ReadLine(); 62 | for (int i = 0; line != ""; i++) 63 | { 64 | text.OnNext(line); 65 | computation.Sync(i); 66 | line = Console.ReadLine(); 67 | } 68 | } 69 | 70 | text.OnCompleted(); // closes input 71 | computation.Join(); 72 | } 73 | 74 | } 75 | 76 | public string Usage { get { return ""; } } 77 | 78 | 79 | public string Help 80 | { 81 | get { return "Demonstrates a simple differential dataflow program for interactively counting words in lines of text."; } 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /AzureExamples/Azure/Repartition.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | 26 | using Microsoft.Research.Naiad.Frameworks.Azure; 27 | using Microsoft.Research.Naiad.Dataflow.PartitionBy; 28 | 29 | namespace Microsoft.Research.Naiad.AzureExamples 30 | { 31 | public class Repartition : Example 32 | { 33 | public string Usage 34 | { 35 | get { return "containerName inputDirectory outputDirectory"; } 36 | } 37 | 38 | public string Help 39 | { 40 | get { return "repartitions the text contexts of inputDirectory into as many parts as there are workers, writing the outputs to outputDirectory."; } 41 | } 42 | 43 | public void Execute(string[] args) 44 | { 45 | using (var computation = NewComputation.FromArgs(ref args)) 46 | { 47 | computation.Controller.SetConsoleOut(computation.DefaultBlobContainer("naiad-outputs"), "out-{0}.txt"); 48 | computation.Controller.SetConsoleError(computation.DefaultBlobContainer("naiad-outputs"), "err-{0}.txt"); 49 | 50 | if (args.Length == 4) 51 | { 52 | var containerName = args[1]; 53 | var inputDirectory = args[2]; 54 | var outputDirectory = args[3]; 55 | 56 | if (!inputDirectory.Equals(outputDirectory)) 57 | { 58 | var container = computation.DefaultBlobContainer(containerName); 59 | 60 | computation.ReadTextFromAzureBlobs(container, inputDirectory) 61 | .PartitionBy(x => x.GetHashCode()) 62 | .WriteTextToAzureBlobs(container, outputDirectory + "/part-{0}-{1}.txt"); 63 | } 64 | else 65 | { 66 | Console.Error.WriteLine("ERROR: Input directory name ({0}) equals output directory name ({1})", inputDirectory, outputDirectory); 67 | } 68 | } 69 | else 70 | { 71 | Console.Error.WriteLine("repartition requires three additional arguments: " + this.Usage); 72 | } 73 | 74 | computation.Activate(); 75 | computation.Join(); 76 | 77 | Console.Out.Close(); 78 | Console.Error.Close(); 79 | } 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/Operators/Sum.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | using System.Linq.Expressions; 26 | using Microsoft.Research.Naiad; 27 | using Microsoft.Research.Naiad.Dataflow; 28 | 29 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.Operators 30 | { 31 | internal class SumInt32 : Aggregate 32 | where K : IEquatable 33 | where S : IEquatable 34 | where T : Microsoft.Research.Naiad.Time 35 | where R : IEquatable 36 | { 37 | public SumInt32(int index, Stage collection, bool inputImmutable, Expression> k, Expression> v, Expression> r) 38 | : base(index, collection, inputImmutable, k, v, (Int64 a, int x, int y) => (int)(a * x + y), x => x == 0, r) 39 | { 40 | } 41 | } 42 | 43 | internal class SumInt64 : Aggregate 44 | where K : IEquatable 45 | where S : IEquatable 46 | where T : Time 47 | where R : IEquatable 48 | { 49 | public SumInt64(int index, Stage collection, bool inputImmutable, Expression> k, Expression> v, Expression> r) 50 | : base(index, collection, inputImmutable, k, v, (a, x, y) => a * x + y, x => x == 0, r) 51 | { 52 | } 53 | } 54 | 55 | internal class SumFloat : Aggregate 56 | where K : IEquatable 57 | where S : IEquatable 58 | where T : Time 59 | where R : IEquatable 60 | { 61 | public SumFloat(int index, Stage collection, bool inputImmutable, Expression> k, Expression> v, Expression> r) 62 | : base(index, collection, inputImmutable, k, v, (a, x, y) => a * x + y, x => x == 0.0, r) 63 | { 64 | } 65 | } 66 | 67 | internal class SumDouble : Aggregate 68 | where K : IEquatable 69 | where S : IEquatable 70 | where T : Time 71 | where R : IEquatable 72 | { 73 | public SumDouble(int index, Stage collection, bool inputImmutable, Expression> k, Expression> v, Expression> r) 74 | : base(index, collection, inputImmutable, k, v, (a, x, y) => a * x + y, x => x == 0.0, r) 75 | { 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/Operators/SelectMany.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | using System.Linq.Expressions; 26 | using Microsoft.Research.Naiad.Dataflow; 27 | using Microsoft.Research.Naiad.Dataflow.StandardVertices; 28 | 29 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.Operators 30 | { 31 | internal class SelectMany : UnaryVertex, Weighted, T> 32 | where S : IEquatable 33 | where T : Time 34 | where R : IEquatable 35 | { 36 | protected Func> selector; 37 | 38 | public override void OnReceive(Message, T> message) 39 | { 40 | var output = this.Output.GetBufferForTime(message.time); 41 | for (int i = 0; i < message.length; i++) 42 | { 43 | var record = message.payload[i]; 44 | foreach (var r in selector(record.record)) 45 | output.Send(new Weighted(r, record.weight)); 46 | } 47 | } 48 | 49 | public override string ToString() 50 | { 51 | return "SelectMany"; 52 | } 53 | 54 | public SelectMany(int index, Stage collection, Expression>> transformation) 55 | : base(index, collection) 56 | { 57 | selector = transformation.Compile(); 58 | } 59 | } 60 | 61 | internal class SelectManyBatch : UnaryVertex, Weighted, T> 62 | where S : IEquatable 63 | where T : Microsoft.Research.Naiad.Time 64 | where R : IEquatable 65 | { 66 | protected Func>> selector; 67 | 68 | public override void OnReceive(Message, T> message) 69 | { 70 | var output = this.Output.GetBufferForTime(message.time); 71 | for (int i = 0; i < message.length; i++) 72 | { 73 | var record = message.payload[i]; 74 | foreach (var r in selector(record.record)) 75 | for (int ii = 0; ii < r.Count; ii++) 76 | output.Send(new Weighted(r.Array[r.Offset + i], record.weight)); 77 | } 78 | } 79 | 80 | public SelectManyBatch(int index, Stage collection, Expression>>> transformation) 81 | : base(index, collection) 82 | { 83 | selector = transformation.Compile(); 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /Naiad/Runtime/Progress/UpdateBuffer.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | using Microsoft.Research.Naiad.Scheduling; 26 | using Microsoft.Research.Naiad.Dataflow; 27 | 28 | namespace Microsoft.Research.Naiad.Runtime.Progress 29 | { 30 | internal class ProgressUpdateBuffer 31 | where T : Time 32 | { 33 | public Dictionary Updates; 34 | Runtime.Progress.ProgressUpdateProducer producer; 35 | 36 | int stageID; 37 | 38 | T lastSentTime; 39 | Int64 lastSentCount; 40 | 41 | Pointstamp version; 42 | 43 | public void PushCachedToDelta() 44 | { 45 | if (lastSentCount != 0) 46 | { 47 | Int64 value; 48 | if (!Updates.TryGetValue(lastSentTime, out value)) 49 | Updates.Add(lastSentTime, lastSentCount); 50 | else 51 | { 52 | if (value + lastSentCount == 0) 53 | Updates.Remove(lastSentTime); 54 | else 55 | Updates[lastSentTime] = value + lastSentCount; 56 | } 57 | 58 | lastSentCount = 0; 59 | } 60 | } 61 | 62 | public void Update(T time, Int64 weight) 63 | { 64 | if (!time.Equals(lastSentTime)) 65 | { 66 | PushCachedToDelta(); 67 | lastSentTime = time; 68 | } 69 | 70 | lastSentCount += weight; 71 | } 72 | 73 | public void Flush() 74 | { 75 | PushCachedToDelta(); 76 | 77 | if (Updates.Count > 0) 78 | { 79 | foreach (var pair in Updates) 80 | { 81 | if (pair.Value != 0) 82 | { 83 | pair.Key.Populate(ref version); // do the type conversion to pointstamp 84 | producer.UpdateRecordCounts(version, pair.Value); 85 | } 86 | } 87 | 88 | Updates.Clear(); 89 | } 90 | } 91 | 92 | public ProgressUpdateBuffer(int name, Runtime.Progress.ProgressUpdateProducer p) 93 | { 94 | stageID = name; 95 | 96 | Updates = new Dictionary(); 97 | producer = p; 98 | 99 | var temp = new Pointstamp(0, new int[] { }); 100 | version = new Pointstamp(name, new int[default(T).Populate(ref temp)]); 101 | } 102 | } 103 | 104 | } 105 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/KeyIndices.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | using Microsoft.Research.Naiad.Serialization; 26 | 27 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow 28 | { 29 | internal struct UnaryKeyIndices : IEquatable 30 | { 31 | public int unprocessed; 32 | public int processed; 33 | public int output; 34 | 35 | public bool Equals(UnaryKeyIndices other) 36 | { 37 | return this.unprocessed == other.unprocessed && this.processed == other.processed && this.output == other.output; 38 | } 39 | 40 | public bool IsEmpty { get { return unprocessed == 0 && processed == 0 && output == 0; } } 41 | 42 | public UnaryKeyIndices(int u, int p, int r) { unprocessed = u; processed = p; output = r; } 43 | } 44 | 45 | internal struct BinaryKeyIndices : IEquatable 46 | { 47 | //public K key; 48 | public int unprocessed1; 49 | public int unprocessed2; 50 | public int processed1; 51 | public int processed2; 52 | public int output; 53 | 54 | public bool Equals(BinaryKeyIndices other) 55 | { 56 | return this.unprocessed1 == other.unprocessed1 57 | && this.unprocessed2 == other.unprocessed2 58 | && this.processed1 == other.processed1 59 | && this.processed2 == other.processed2 60 | && this.output == other.output; 61 | } 62 | 63 | public BinaryKeyIndices(int u1, int u2, int p1, int p2, int r) { unprocessed1 = u1; unprocessed2 = u2; processed1 = p1; processed2 = p2; output = r; } 64 | } 65 | 66 | // Join doesn't need all the KeyIndices cruft 67 | internal struct JoinKeyIndices : IEquatable 68 | { 69 | public int processed1; 70 | public int processed2; 71 | 72 | public bool IsEmpty { get { return processed1 == 0 && processed2 == 0; } } 73 | 74 | public bool Equals(JoinKeyIndices other) 75 | { 76 | return this.processed1 == other.processed1 && this.processed2 == other.processed2; 77 | } 78 | 79 | public JoinKeyIndices(int p1, int p2) { processed1 = p1; processed2 = p2; } 80 | } 81 | 82 | // Join doesn't need all the KeyIndices cruft 83 | internal struct JoinIntKeyIndices 84 | { 85 | public int processed1; 86 | public int processed2; 87 | 88 | public bool IsEmpty { get { return processed1 == 0 && processed2 == 0; } } 89 | 90 | public JoinIntKeyIndices(int p1, int p2) { processed1 = p1; processed2 = p2; } 91 | 92 | } 93 | 94 | } 95 | -------------------------------------------------------------------------------- /Frameworks/DifferentialDataflow/CollectionTrace/Increments.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | 26 | namespace Microsoft.Research.Naiad.Frameworks.DifferentialDataflow.CollectionTrace 27 | { 28 | 29 | internal struct CollectionTraceWithHeapIncrement : IEquatable 30 | { 31 | public OffsetLength OffsetLength; 32 | public int TimeIndex; 33 | 34 | public bool IsEmpty { get { return OffsetLength.IsEmpty; } } 35 | 36 | public bool Equals(CollectionTraceWithHeapIncrement other) 37 | { 38 | return this.OffsetLength.Equals(other.OffsetLength) && this.TimeIndex == other.TimeIndex; 39 | } 40 | 41 | public CollectionTraceWithHeapIncrement(OffsetLength ol, int t) { OffsetLength = ol; TimeIndex = t; } 42 | public CollectionTraceWithHeapIncrement(int t) { OffsetLength = new OffsetLength(); TimeIndex = t; } 43 | } 44 | 45 | internal struct CollectionTraceWithAggregationIncrement : IEquatable> 46 | where S : IEquatable 47 | { 48 | public Int64 Weight; 49 | public int TimeIndex; 50 | public S Value; 51 | 52 | public bool IsEmpty(Func isZero) { return Weight == 0 && isZero(Value); } 53 | 54 | public CollectionTraceWithAggregationIncrement Add(CollectionTraceWithAggregationIncrement that, Func axpy, int scale = 1) 55 | { 56 | this.Weight += scale * that.Weight; 57 | this.Value = axpy(scale, that.Value, this.Value); 58 | return this; 59 | } 60 | 61 | public bool Equals(CollectionTraceWithAggregationIncrement other) 62 | { 63 | return this.Weight == other.Weight && this.TimeIndex == other.TimeIndex && this.Value.Equals(other.Value); 64 | } 65 | 66 | public CollectionTraceWithAggregationIncrement(Int64 w, int t, S v) { Weight = w; TimeIndex = t; Value = v; } 67 | public CollectionTraceWithAggregationIncrement(int t) { Weight = 0; TimeIndex = t; Value = default(S); } 68 | } 69 | 70 | internal struct CollectionTraceWithoutHeapIncrement : IEquatable 71 | { 72 | public Int64 Weight; 73 | public int TimeIndex; 74 | 75 | public bool IsEmpty { get { return Weight == 0; } } 76 | 77 | public bool Equals(CollectionTraceWithoutHeapIncrement other) 78 | { 79 | return this.Weight == other.Weight && this.TimeIndex == other.TimeIndex; 80 | } 81 | 82 | public CollectionTraceWithoutHeapIncrement(Int64 w, int t) { Weight = w; TimeIndex = t; } 83 | public CollectionTraceWithoutHeapIncrement(int t) { Weight = 0; TimeIndex = t; } 84 | } 85 | 86 | } 87 | -------------------------------------------------------------------------------- /Naiad/Runtime/Progress/PointstampCountSet.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | using Microsoft.Research.Naiad.Dataflow.Channels; 26 | using Microsoft.Research.Naiad.Serialization; 27 | using Microsoft.Research.Naiad.DataStructures; 28 | using Microsoft.Research.Naiad.Scheduling; 29 | 30 | namespace Microsoft.Research.Naiad.Runtime.Progress 31 | { 32 | internal class PointstampCountSet 33 | { 34 | internal readonly Dictionary Counts = new Dictionary(); 35 | private readonly PointstampFrontier actualFrontier; 36 | 37 | internal PointstampCountSet(Reachability reachability) 38 | { 39 | actualFrontier = new PointstampFrontier(reachability); 40 | } 41 | 42 | public Pointstamp[] Frontier = new Pointstamp[0]; 43 | 44 | internal bool UpdatePointstampCount(Pointstamp version, Int64 delta) 45 | { 46 | var oldFrontier = Frontier; 47 | var count = 0L; 48 | if (!Counts.TryGetValue(version, out count)) 49 | { 50 | version = new Pointstamp(version); 51 | 52 | Counts.Add(version, delta); 53 | 54 | // Potentially add this version to the frontier 55 | if (actualFrontier.Add(version)) 56 | Frontier = actualFrontier.Antichain.ToArray(); 57 | } 58 | else 59 | { 60 | if (count + delta == 0) 61 | { 62 | Counts.Remove(version); 63 | if (actualFrontier.Remove(version)) 64 | Frontier = actualFrontier.Antichain.ToArray(); 65 | } 66 | else 67 | { 68 | Counts[version] = count + delta; 69 | } 70 | } 71 | 72 | return Frontier != oldFrontier; 73 | } 74 | 75 | public void Checkpoint(NaiadWriter writer, NaiadSerialization longSerializer, NaiadSerialization pointstampSerializer, NaiadSerialization intSerializer) 76 | { 77 | 78 | this.Counts.Checkpoint(writer); 79 | this.actualFrontier.Checkpoint(writer); 80 | this.Frontier.Checkpoint(this.Frontier.Length, writer); 81 | } 82 | 83 | public void Restore(NaiadReader reader, NaiadSerialization longSerializer, NaiadSerialization pointstampSerializer, NaiadSerialization intSerializer) 84 | { 85 | this.Counts.Restore(reader); 86 | this.actualFrontier.Restore(reader); 87 | this.Frontier = CheckpointRestoreExtensionMethods.RestoreArray(reader, n => new Pointstamp[n]); 88 | } 89 | 90 | public bool Stateful { get { return true; } } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /Naiad/Channels/BufferSegmentConsumer.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using Microsoft.Research.Naiad.Runtime.Networking; 22 | using System; 23 | using System.Collections.Generic; 24 | using System.IO; 25 | using System.Linq; 26 | using System.Text; 27 | 28 | namespace Microsoft.Research.Naiad.Serialization 29 | { 30 | internal interface SerializedMessageSender : IDisposable 31 | { 32 | int SendBufferSegment(MessageHeader hdr, BufferSegment segment); 33 | } 34 | 35 | internal class StreamSerializedMessageSender : SerializedMessageSender 36 | { 37 | private readonly Stream stream; 38 | private int nextPageIndex; 39 | 40 | private readonly int pageSize; 41 | 42 | private bool disposed = false; 43 | 44 | public int SendBufferSegment(MessageHeader hdr, BufferSegment segment) 45 | { 46 | ArraySegment byteArraySegment = segment.ToArraySegment(); 47 | if (byteArraySegment.Offset == 0) 48 | { 49 | this.stream.Write(byteArraySegment.Array, 0, this.pageSize); 50 | } 51 | else 52 | { 53 | this.stream.Write(byteArraySegment.Array, byteArraySegment.Offset, byteArraySegment.Count); 54 | // Now pad the write to the full page size. 55 | this.stream.Seek(this.pageSize - byteArraySegment.Count, SeekOrigin.Current); 56 | } 57 | return this.nextPageIndex++; 58 | } 59 | 60 | internal StreamSerializedMessageSender(Stream stream, int pageSize) 61 | { 62 | this.nextPageIndex = 0; 63 | this.stream = stream; 64 | 65 | this.pageSize = pageSize; 66 | } 67 | 68 | public void Dispose() 69 | { 70 | if (!this.disposed) 71 | { 72 | this.stream.Dispose(); 73 | this.disposed = true; 74 | } 75 | } 76 | } 77 | 78 | internal class NetworkChannelSerializedMessageSender : SerializedMessageSender 79 | { 80 | private readonly NetworkChannel networkChannel; 81 | private int destProcessID; 82 | private int nextPageIndex; 83 | 84 | public int SendBufferSegment(MessageHeader hdr, BufferSegment segment) 85 | { 86 | segment.Copy(); 87 | this.networkChannel.SendBufferSegment(hdr, this.destProcessID, segment, true); 88 | return this.nextPageIndex++; 89 | } 90 | 91 | public NetworkChannelSerializedMessageSender(NetworkChannel networkChannel, int destProcessID) 92 | { 93 | this.nextPageIndex = 0; 94 | this.networkChannel = networkChannel; 95 | this.destProcessID = destProcessID; 96 | } 97 | 98 | public void Dispose() { } 99 | } 100 | 101 | 102 | 103 | } 104 | -------------------------------------------------------------------------------- /Naiad/Channels/PipelineChannel.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | using Microsoft.Research.Naiad.DataStructures; 26 | using System.Diagnostics; 27 | using Microsoft.Research.Naiad.Scheduling; 28 | 29 | namespace Microsoft.Research.Naiad.Dataflow.Channels 30 | { 31 | internal class PipelineChannel : Cable 32 | where T : Time 33 | { 34 | private class Fiber : SendChannel 35 | { 36 | private readonly PipelineChannel bundle; 37 | private readonly int index; 38 | private VertexInput receiver; 39 | 40 | public Fiber(PipelineChannel bundle, VertexInput receiver, int index) 41 | { 42 | this.bundle = bundle; 43 | this.index = index; 44 | this.receiver = receiver; 45 | } 46 | 47 | public void Send(Message records) 48 | { 49 | this.receiver.OnReceive(records, new ReturnAddress()); 50 | } 51 | 52 | public override string ToString() 53 | { 54 | return string.Format("Pipeline({0} => {1})", this.bundle.SourceStage, this.bundle.DestinationStage); 55 | } 56 | 57 | public void Flush() 58 | { 59 | this.receiver.Flush(); 60 | } 61 | } 62 | 63 | private readonly StageOutput sender; 64 | private readonly StageInput receiver; 65 | 66 | private readonly Dictionary subChannels; 67 | 68 | private readonly int channelId; 69 | public int ChannelId { get { return channelId; } } 70 | 71 | public PipelineChannel(StageOutput sender, StageInput receiver, int channelId) 72 | { 73 | this.sender = sender; 74 | this.receiver = receiver; 75 | 76 | this.channelId = channelId; 77 | 78 | this.subChannels = new Dictionary(); 79 | foreach (VertexLocation loc in sender.ForStage.Placement) 80 | if (loc.ProcessId == sender.ForStage.InternalComputation.Controller.Configuration.ProcessID) 81 | this.subChannels[loc.VertexId] = new Fiber(this, receiver.GetPin(loc.VertexId), loc.VertexId); 82 | } 83 | 84 | public SendChannel GetSendChannel(int i) 85 | { 86 | return this.subChannels[i]; 87 | } 88 | 89 | public Dataflow.Stage SourceStage { get { return this.sender.ForStage; } } 90 | public Dataflow.Stage DestinationStage { get { return this.receiver.ForStage; } } 91 | 92 | public override string ToString() 93 | { 94 | return String.Format("Pipeline channel: {0} -> {1}", this.sender.ForStage, this.receiver.ForStage); 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /ClusterSubmission/NaiadPeloponneseSupport/PPMSubmission.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.IO; 24 | using System.Linq; 25 | using System.Text; 26 | using System.Threading.Tasks; 27 | using System.Xml.Linq; 28 | 29 | using Microsoft.Research.Peloponnese.ClusterUtils; 30 | 31 | namespace Microsoft.Research.Naiad.Cluster 32 | { 33 | public interface PPMSubmission : IDisposable 34 | { 35 | void Submit(); 36 | int Join(); 37 | } 38 | 39 | public class Helpers 40 | { 41 | public static void MakeCommandLine(string[] args, bool stripPath, out string exeName, out IEnumerable commandLineArgs) 42 | { 43 | if (stripPath) 44 | { 45 | if (args[0].ToLower().StartsWith("hdfs://")) 46 | { 47 | exeName = args[0].Substring(args[0].LastIndexOf('/') + 1); 48 | } 49 | else 50 | { 51 | exeName = Path.GetFileName(args[0]); 52 | } 53 | } 54 | else 55 | { 56 | exeName = Path.GetFullPath(args[0]); 57 | } 58 | 59 | List commandLineArgsList = new List(args.Length); 60 | for (int i = 1; i < args.Length ;++i) 61 | { 62 | commandLineArgsList.Add(args[i]); 63 | } 64 | commandLineArgsList.Add("--ppm"); 65 | 66 | commandLineArgs = commandLineArgsList; 67 | } 68 | 69 | public static XDocument MakePeloponneseConfig( 70 | int numberOfProcesses, int workerMemoryInMB, string type, 71 | string commandLine, IEnumerable commandLineArgs, bool addRedirects, IEnumerable resources) 72 | { 73 | XDocument configDoc = new XDocument(); 74 | 75 | XElement docElement = new XElement("PeloponneseConfig"); 76 | 77 | XElement serverElement = new XElement("PeloponneseServer"); 78 | 79 | XElement portElement = new XElement("Port"); 80 | portElement.Value = "8471"; 81 | serverElement.Add(portElement); 82 | 83 | XElement prefixElement = new XElement("Prefix"); 84 | prefixElement.Value = "/peloponnese/server/"; 85 | serverElement.Add(prefixElement); 86 | 87 | XElement workers = ConfigHelpers.MakeProcessGroup( 88 | "Worker", type, 1, numberOfProcesses, workerMemoryInMB, true, 89 | commandLine, commandLineArgs, "LOG_DIRS", "stdout.txt", "stderr.txt", 90 | resources, null); 91 | serverElement.Add(workers); 92 | 93 | docElement.Add(serverElement); 94 | 95 | configDoc.Add(docElement); 96 | 97 | return configDoc; 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /AzureExamples/Program.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | using System.Threading.Tasks; 26 | 27 | using Microsoft.Research.Naiad; 28 | using Microsoft.Research.Naiad.Diagnostics; 29 | 30 | namespace Microsoft.Research.Naiad.AzureExamples 31 | { 32 | /// 33 | /// An example runnable as a sample Naiad program. 34 | /// 35 | public interface Example 36 | { 37 | /// 38 | /// Describes arguments used by the example 39 | /// 40 | string Usage { get; } 41 | 42 | /// 43 | /// Describes the intended behavior for the example, with rich descriptive text. 44 | /// 45 | string Help { get; } 46 | 47 | /// 48 | /// Executes the example with all of the supplied arguments (including the example name). 49 | /// 50 | /// 51 | void Execute(string[] args); 52 | } 53 | 54 | class Program 55 | { 56 | static void Main(string[] args) 57 | { 58 | // map from example names to code to run in each case 59 | var examples = new Dictionary(); 60 | 61 | // some GraphLINQ examples 62 | examples.Add("connectedcomponents", new AzureExamples.ConnectedComponents()); 63 | examples.Add("graphgenerator", new AzureExamples.GraphGenerator()); 64 | examples.Add("repartition", new AzureExamples.Repartition()); 65 | examples.Add("graphlinq-pagerank", new AzureExamples.GraphLINQ.PageRank()); 66 | 67 | // determine which exmample was asked for 68 | if (args.Length == 0 || !examples.ContainsKey(args[0].ToLower())) 69 | { 70 | Console.Error.WriteLine("First argument not found in list of examples"); 71 | Console.Error.WriteLine("Choose from the following exciting options:"); 72 | foreach (var pair in examples) 73 | Console.Error.WriteLine("\tAzureExamples.exe {0} {1} [naiad options]", pair.Key, pair.Value.Usage); 74 | 75 | Console.Error.WriteLine(); 76 | Configuration.Usage(); 77 | } 78 | else 79 | { 80 | var example = args[0].ToLower(); 81 | if (args.Contains("--help") || args.Contains("/?") || args.Contains("--usage")) 82 | { 83 | Console.Error.WriteLine("Usage: AzureExamples.exe {0} {1} [naiad options]", example, examples[example].Usage); 84 | Configuration.Usage(); 85 | } 86 | else 87 | { 88 | Logging.LogLevel = LoggingLevel.Error; 89 | Logging.LogStyle = LoggingStyle.Console; 90 | 91 | examples[example].Execute(args); 92 | } 93 | } 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /Naiad/DataStructures/CircularLogBuffer.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | 26 | namespace Microsoft.Research.Naiad 27 | { 28 | // Simple circular buffer for logging 29 | // Not thread safe 30 | internal class CircularLogBuffer 31 | { 32 | public int Head = 0; 33 | public int Tail = 0; 34 | public int Length; 35 | public int NumElements = 0; 36 | public bool OverwriteIfFull = false; 37 | public T emptyslot; 38 | 39 | public T[] data; 40 | 41 | // Add an element to the head. 42 | public void Add(T element) 43 | { 44 | data[Head] = element; 45 | if (NumElements == Length && !OverwriteIfFull) 46 | { 47 | throw (new Exception("Tried to add element to full buffer")); 48 | } 49 | if (NumElements == Length && Head == Tail) 50 | { 51 | Tail++; 52 | } 53 | Head++; 54 | if (Head == Length) Head = 0; 55 | if (NumElements < Length) NumElements++; 56 | } 57 | 58 | // Remove the tail element and return it. 59 | public T Remove() 60 | { 61 | if (NumElements == 0) 62 | { 63 | throw (new Exception("Tried to remove from empty buffer")); 64 | } 65 | T element = data[Tail]; 66 | data[Tail] = emptyslot; 67 | Tail++; 68 | if (Tail == Length) Tail = 0; 69 | NumElements--; 70 | return element; 71 | } 72 | 73 | public bool isFull() 74 | { 75 | return (NumElements == Length); 76 | } 77 | 78 | public bool isEmpty() 79 | { 80 | return (NumElements == 0); 81 | } 82 | 83 | public void Clear() 84 | { 85 | NumElements = 0; 86 | Head = 0; 87 | Tail = 0; 88 | for (int i = 0; i < Length; i++) 89 | { 90 | data[i] = emptyslot; 91 | } 92 | } 93 | 94 | public override string ToString() 95 | { 96 | string s = String.Format("Head={0} Tail={1} Length={2} NumElements={3} data=[", Head, Tail, Length, NumElements); 97 | for (int i = 0; i < Length; i++) 98 | { 99 | if (i > 0) s = s + ","; 100 | s = s + data[i]; 101 | } 102 | s = s + "]"; 103 | 104 | return s; 105 | } 106 | 107 | #region Constructors 108 | public CircularLogBuffer(int size, T empty) 109 | { 110 | data = new T[size]; 111 | Length = size; 112 | emptyslot = empty; 113 | for (int i = 0; i < Length; i++) 114 | { 115 | data[i] = emptyslot; 116 | } 117 | } 118 | #endregion 119 | 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /AzureExamples/Azure/GraphGenerator.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using Microsoft.Research.Naiad; 22 | using Microsoft.Research.Naiad.Input; 23 | using Microsoft.Research.Naiad.Frameworks.Lindi; 24 | using Microsoft.Research.Naiad.Frameworks.Azure; 25 | using Microsoft.Research.Naiad.Dataflow.PartitionBy; 26 | using System; 27 | using System.Collections.Generic; 28 | using System.Linq; 29 | using System.Text; 30 | using Microsoft.WindowsAzure.Storage; 31 | 32 | namespace Microsoft.Research.Naiad.AzureExamples 33 | { 34 | public struct GraphProperties 35 | { 36 | public int NodeCount; 37 | public int EdgeCount; 38 | 39 | public GraphProperties(int nodeCount, int edgeCount) 40 | { 41 | this.NodeCount = nodeCount; 42 | this.EdgeCount = edgeCount; 43 | } 44 | } 45 | 46 | class GraphGenerator : Example 47 | { 48 | public string Usage 49 | { 50 | get { return "containername directoryname nodecount edgecount"; } 51 | } 52 | 53 | public void Execute(string[] args) 54 | { 55 | var containerName = args[1]; 56 | var directoryName = args[2]; 57 | 58 | var nodeCount = int.Parse(args[3]); 59 | var edgeCount = int.Parse(args[4]); 60 | 61 | CloudStorageAccount storageAccount = CloudStorageAccount.DevelopmentStorageAccount; 62 | 63 | var container = storageAccount.CreateCloudBlobClient() 64 | .GetContainerReference(containerName); 65 | 66 | container.CreateIfNotExists(); 67 | 68 | // allocate a new computation from command line arguments. 69 | using (var computation = NewComputation.FromArgs(ref args)) 70 | { 71 | // create a new input from a constant data source 72 | var source = new ConstantDataSource(new GraphProperties(nodeCount, edgeCount)); 73 | var input = computation.NewInput(source); 74 | 75 | // generate the graph, partition by edge source, and write to Azure. 76 | input.SelectMany(x => GenerateGraph(x.NodeCount, x.EdgeCount)) 77 | .PartitionBy(x => x.First) 78 | .WriteBinaryToAzureBlobs(container, directoryName + "/edges-{0}"); 79 | 80 | // start job and wait. 81 | computation.Activate(); 82 | computation.Join(); 83 | } 84 | 85 | } 86 | 87 | private static IEnumerable> GenerateGraph(int nodeCount, int edgeCount) 88 | { 89 | var random = new Random(0); 90 | for (int i = 0; i < edgeCount; i++) 91 | yield return new Pair(random.Next(nodeCount), random.Next(nodeCount)); 92 | } 93 | 94 | 95 | public string Help 96 | { 97 | get { return "Generates a random graph and writes it to a set of files in an Azure directory. For use with other Azure example."; } 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /ClusterSubmission/NaiadPeloponneseSupport/App.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | -------------------------------------------------------------------------------- /ClusterSubmission/RunNaiad/LocalSubmission.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Diagnostics; 24 | using System.IO; 25 | using System.Linq; 26 | using System.Text; 27 | using System.Threading.Tasks; 28 | using System.Xml.Linq; 29 | 30 | using Microsoft.Research.Peloponnese.ClusterUtils; 31 | 32 | namespace Microsoft.Research.Naiad.Cluster 33 | { 34 | class LocalSubmission : PPMSubmission 35 | { 36 | private readonly string ppmHome; 37 | private readonly string workingDirectory; 38 | private readonly string configPath; 39 | 40 | private Process ppmProcess; 41 | 42 | private string CreateDirectory(string wdBase) 43 | { 44 | if (!Directory.Exists(wdBase)) 45 | { 46 | Directory.CreateDirectory(wdBase); 47 | } 48 | 49 | var existingDirs = Directory.EnumerateDirectories(wdBase); 50 | var existingJobs = existingDirs.Select(x => Path.GetFileName(x)) 51 | .Select(x => { int jobId; if (int.TryParse(x, out jobId)) return jobId; else return -1; }); 52 | 53 | int nextJob = 0; 54 | if (existingJobs.Count() > 0) 55 | { 56 | nextJob = existingJobs.Max() + 1; 57 | } 58 | 59 | var wd = Path.Combine(wdBase, nextJob.ToString()); 60 | 61 | Directory.CreateDirectory(wd); 62 | 63 | return wd; 64 | } 65 | 66 | public LocalSubmission(int numberOfProcesses, string[] args, string localDirectory) 67 | { 68 | string commandLine; 69 | IEnumerable commandLineArgs; 70 | Helpers.MakeCommandLine(args, false, out commandLine, out commandLineArgs); 71 | 72 | this.ppmHome = ConfigHelpers.GetPPMHome(null); 73 | 74 | XDocument config = Helpers.MakePeloponneseConfig(numberOfProcesses, -1, "local", commandLine, commandLineArgs, true, null); 75 | 76 | this.workingDirectory = CreateDirectory(localDirectory); 77 | 78 | this.configPath = "config.xml"; 79 | config.Save(Path.Combine(this.workingDirectory, this.configPath)); 80 | } 81 | 82 | public void Dispose() 83 | { 84 | } 85 | 86 | public void Submit() 87 | { 88 | ProcessStartInfo psi = new ProcessStartInfo(); 89 | psi.FileName = Path.Combine(this.ppmHome, "Microsoft.Research.Peloponnese.PersistentProcessManager.exe"); 90 | psi.Arguments = this.configPath; 91 | psi.UseShellExecute = false; 92 | psi.WorkingDirectory = this.workingDirectory; 93 | 94 | this.ppmProcess = new Process(); 95 | this.ppmProcess.StartInfo = psi; 96 | this.ppmProcess.EnableRaisingEvents = true; 97 | 98 | this.ppmProcess.Start(); 99 | } 100 | 101 | public int Join() 102 | { 103 | this.ppmProcess.WaitForExit(); 104 | return this.ppmProcess.ExitCode; 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /Naiad/DataStructures/Pair.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | 26 | namespace Microsoft.Research.Naiad 27 | { 28 | /// 29 | /// Extension methods for the Pair type 30 | /// 31 | public static class PairExtensionMethods 32 | { 33 | /// 34 | /// Returns a new Pair of two elements 35 | /// 36 | /// first element type 37 | /// second element type 38 | /// first element 39 | /// second element 40 | /// pair of first and second elements 41 | public static Pair PairWith(this V1 first, V2 second) 42 | { 43 | return new Pair(first, second); 44 | } 45 | } 46 | 47 | /// 48 | /// Pair of two elements 49 | /// 50 | /// The first element type. 51 | /// The second element type. 52 | public struct Pair : IEquatable> 53 | { 54 | /// 55 | /// First element. 56 | /// 57 | public TFirst First; 58 | 59 | /// 60 | /// Second element. 61 | /// 62 | public TSecond Second; 63 | 64 | /// 65 | /// Constructs a pair from two elements 66 | /// 67 | /// The first element. 68 | /// The second element. 69 | public Pair(TFirst first, TSecond second) { First = first; Second = second; } 70 | 71 | /// 72 | /// Returns a string representation of this pair. 73 | /// 74 | /// A string representation of this pair. 75 | public override string ToString() 76 | { 77 | return "[" + First.ToString() + " " + Second.ToString() + "]"; 78 | } 79 | 80 | /// 81 | /// Returns a 32-bit signed integer hashcode for this pair. 82 | /// 83 | /// A 32-bit signed integer hashcode for this pair. 84 | public override int GetHashCode() 85 | { 86 | return (this.First == null ? 0 : this.First.GetHashCode()) + 123412324 * (this.Second == null ? 0 : this.Second.GetHashCode()); 87 | } 88 | 89 | 90 | /// 91 | /// Compares this pair and the pair for element-wise equality. 92 | /// 93 | /// The other pair. 94 | /// true, if and only if both pairs are element-wise equal. 95 | public bool Equals(Pair other) 96 | { 97 | return (EqualityComparer.Default.Equals(this.First, other.First) && EqualityComparer.Default.Equals(this.Second, other.Second)); 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /Naiad/NamespaceDocs.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Naiad ver. 0.5 3 | * Copyright (c) Microsoft Corporation 4 | * All rights reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT 14 | * LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR 15 | * A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. 16 | * 17 | * See the Apache Version 2.0 License for specific language governing 18 | * permissions and limitations under the License. 19 | */ 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Text; 25 | 26 | namespace Microsoft.Research.Naiad 27 | { 28 | /// 29 | /// The Microsoft.Research.Naiad namespace provides the basic classes that are used in all Naiad programs. 30 | /// 31 | class NamespaceDoc 32 | { 33 | } 34 | 35 | namespace Dataflow 36 | { 37 | /// 38 | /// The Dataflow namespace provides classes that enable the programmer to construct dataflow graphs manually. 39 | /// 40 | class NamespaceDoc 41 | { 42 | } 43 | 44 | namespace Iteration 45 | { 46 | /// 47 | /// The Dataflow.Iteration namespace provides classes that enable adding cycles to a dataflow graph. 48 | /// 49 | class NamespaceDoc 50 | { 51 | } 52 | } 53 | 54 | namespace PartitionBy 55 | { 56 | /// 57 | /// The Dataflow.PartitionBy namespace provides extension methods that enable manual repartitioning of data between vertices. 58 | /// 59 | class NamespaceDoc 60 | { 61 | } 62 | } 63 | 64 | namespace StandardVertices 65 | { 66 | /// 67 | /// The Dataflow.StandardVertices namespace provides base implementations of commonly-used dataflow vertex types, such as and . 68 | /// 69 | class NamespaceDoc 70 | { 71 | } 72 | } 73 | } 74 | 75 | namespace Diagnostics 76 | { 77 | /// 78 | /// The Diagnostics namespace provides classes that support , tracing, and observing various events in the Naiad runtime. 79 | /// 80 | class NamespaceDoc 81 | { 82 | } 83 | } 84 | 85 | namespace Input 86 | { 87 | /// 88 | /// The Input namespace provides classes and extension methods for ingesting data into a Naiad computation. 89 | /// 90 | class NamespaceDoc 91 | { 92 | } 93 | } 94 | 95 | namespace Runtime.Progress 96 | { 97 | /// 98 | /// The Runtime.Progress namespace provides classes that support tracking the progress of a Naiad computation. 99 | /// 100 | class NamespaceDoc 101 | { 102 | } 103 | } 104 | 105 | namespace Serialization 106 | { 107 | /// 108 | /// The Serialization namespace contains classes that support serialization of Naiad data for network and file I/O. 109 | /// 110 | class NamespaceDoc 111 | { 112 | } 113 | } 114 | 115 | namespace Utilities 116 | { 117 | /// 118 | /// The Utilities namespace contains miscellaneous classes that support Naiad framework authors. 119 | /// 120 | class NamespaceDoc 121 | { 122 | } 123 | } 124 | } 125 | --------------------------------------------------------------------------------