├── ScientificDataSet ├── Images │ └── open.png ├── Utilities │ ├── Images │ │ └── open.png │ └── GeoConventions.cs ├── Core │ ├── IDependantVariable.cs │ ├── EmptyValue.cs │ ├── Interpolation.cs │ ├── Providers.cs │ ├── MetadataChanges.cs │ ├── Exceptions │ │ ├── ResourceNotFoundException.cs │ │ ├── DataSetException.cs │ │ ├── DistributedCommitFailedException.cs │ │ ├── ConstraintsFailedException.cs │ │ ├── Exceptions.cs │ │ └── DataSetCreatingException.cs │ ├── MetadataContainerVariable.cs │ ├── IChunkSizesAdjustable.cs │ ├── Dimension.cs │ ├── IndexTransformVariable.cs │ ├── IndexResolver.cs │ ├── DataSetLink.cs │ ├── Factory │ │ └── Attributes.cs │ ├── DataSetChangeset.cs │ ├── Schemas.cs │ ├── LambdaTransformVariable.cs │ ├── Range.cs │ ├── OrderedArray1d.cs │ ├── RefVariable.cs │ ├── Rectangle.cs │ ├── RefVariableMetadata.cs │ └── CollectionCombination.cs ├── Providers │ ├── NetCDF │ │ ├── Interop │ │ │ ├── ResultCode.cs │ │ │ ├── NcConst.cs │ │ │ ├── CreateMode.cs │ │ │ └── NcType.cs │ │ ├── NetCDFException.cs │ │ ├── NetCDFGlobalMetadataVariable.cs │ │ ├── ChunkSizeSelector.cs │ │ └── NetCDFUri.cs │ ├── CSV │ │ ├── CsvParsingFailedException.cs │ │ ├── CsvVariablesScalar.cs │ │ ├── CsvVariables1d.cs │ │ └── CsvVariables2d.cs │ └── Memory │ │ ├── MemoryVariable1d.cs │ │ └── MemoryDataSet.cs └── ScientificDataSet.csproj ├── Common └── Version.proj ├── sdsutil ├── sdsutil.csproj └── MetadataConflictResolver.cs ├── ScientificDataSet.DataSetReplicator ├── ScientificDataSet.DataSetReplicator.csproj └── DataSetCommittedEventManager.cs ├── .gitignore ├── SDSLiteTests ├── SDSLiteTests.csproj ├── NetCDFTests.cs ├── Repro.cs └── CsvTests.cs ├── FetchClimate1 ├── ClimateServiceClient │ ├── HandlerClass.cs │ ├── Properties │ │ └── AssemblyInfo.cs │ ├── ServiceLocationConfSection.cs │ ├── CredentialsManager.cs │ ├── NonDisposingServicePort.cs │ ├── RestProcessingClient.cs │ ├── ClimateService.Client.csproj │ └── WcfProcessingClient.cs └── ClimateService.Common │ ├── Exceptions.cs │ ├── RestApiNamings.cs │ ├── Properties │ └── AssemblyInfo.cs │ ├── DataScale.cs │ ├── Extensions.cs │ ├── ClimateService.Common.csproj │ ├── Hash.cs │ ├── RestApiUtilities.cs │ ├── PrioritySemaphore.cs │ └── RequestDescriptors.cs ├── Licence.txt ├── CHANGELOG.md ├── .github └── workflows │ └── build-test.yml ├── SDSlite.sln └── README.md /ScientificDataSet/Images/open.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/predictionmachines/SDSlite/HEAD/ScientificDataSet/Images/open.png -------------------------------------------------------------------------------- /ScientificDataSet/Utilities/Images/open.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/predictionmachines/SDSlite/HEAD/ScientificDataSet/Utilities/Images/open.png -------------------------------------------------------------------------------- /Common/Version.proj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 3.0.1 4 | (c) Microsoft Corporation. All rights reserved 5 | 6 | 7 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/IDependantVariable.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | 7 | namespace Microsoft.Research.Science.Data 8 | { 9 | internal interface IDependantVariable 10 | { 11 | void UpdateChanges(DataSet.Changes changes); 12 | } 13 | } 14 | 15 | -------------------------------------------------------------------------------- /sdsutil/sdsutil.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net60 5 | Exe 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/EmptyValue.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | 7 | namespace Microsoft.Research.Science.Data 8 | { 9 | /// 10 | /// Represents empty value for DataSet variables to contain 11 | /// metadata dictionary only. 12 | /// 13 | public struct EmptyValueType 14 | { 15 | } 16 | } 17 | 18 | -------------------------------------------------------------------------------- /ScientificDataSet/Providers/NetCDF/Interop/ResultCode.cs: -------------------------------------------------------------------------------- 1 | #pragma warning disable 1591 2 | 3 | namespace NetCDFInterop 4 | { 5 | public enum ResultCode : int 6 | { 7 | /// No Error 8 | NC_NOERR = 0, 9 | /// Invalid dimension id or name 10 | NC_EBADDIM = -46, 11 | /// Attribute not found 12 | NC_ENOTATT = -43, 13 | /// Variable name is in use 14 | NC_ENAMEINUSE = -42 15 | }; 16 | } 17 | -------------------------------------------------------------------------------- /ScientificDataSet/Providers/NetCDF/Interop/NcConst.cs: -------------------------------------------------------------------------------- 1 | #pragma warning disable 1591 2 | 3 | namespace NetCDFInterop 4 | { 5 | public static class NcConst 6 | { 7 | /// 8 | /// 'size' argument to ncdimdef for an unlimited dimension 9 | /// 10 | public const int NC_UNLIMITED = 0; 11 | /// 12 | /// Variable id to put/get a global attribute 13 | /// 14 | public const int NC_GLOBAL = -1; 15 | }; 16 | } 17 | -------------------------------------------------------------------------------- /ScientificDataSet.DataSetReplicator/ScientificDataSet.DataSetReplicator.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net471 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.userosscache 8 | *.sln.docstates 9 | 10 | # User-specific files (MonoDevelop/Xamarin Studio) 11 | *.userprefs 12 | 13 | # Build results 14 | [Dd]ebug/ 15 | [Dd]ebugPublic/ 16 | [Rr]elease/ 17 | [Rr]eleases/ 18 | x64/ 19 | x86/ 20 | bld/ 21 | [Bb]in/ 22 | [Oo]bj/ 23 | 24 | # Visual Studio 2015 cache/options directory 25 | .vs/ 26 | 27 | /packages 28 | -------------------------------------------------------------------------------- /SDSLiteTests/SDSLiteTests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net6.0;net48 5 | 8 6 | false 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/Interpolation.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | 7 | namespace Microsoft.Research.Science.Data 8 | { 9 | internal static class Interpolation 10 | { 11 | public static double Interpolate(double x, double x1, double x2, double a, double b) 12 | { 13 | return a + (b - a) * (x - x1) / (x2 - x1); 14 | } 15 | 16 | public static object Interpolate(object x, object x1, object x2, object a, object b) 17 | { 18 | return (double)a + ((double)b - (double)a) * ((double)x - (double)x1) / ((double)x2 - (double)x1); 19 | } 20 | } 21 | } 22 | 23 | -------------------------------------------------------------------------------- /ScientificDataSet/Providers/NetCDF/Interop/CreateMode.cs: -------------------------------------------------------------------------------- 1 | #pragma warning disable 1591 2 | namespace NetCDFInterop 3 | { 4 | public enum CreateMode : int 5 | { 6 | NC_NOWRITE = 0, 7 | /// read & write 8 | NC_WRITE = 0x0001, 9 | NC_CLOBBER = 0, 10 | /// Don't destroy existing file on create 11 | NC_NOCLOBBER = 0x0004, 12 | /// argument to ncsetfill to clear NC_NOFILL 13 | NC_FILL = 0, 14 | /// Don't fill data section an records 15 | NC_NOFILL = 0x0100, 16 | /// Use locking if available 17 | NC_LOCK = 0x0400, 18 | /// Share updates, limit cacheing 19 | NC_SHARE = 0x0800, 20 | NC_64BIT_OFFSET = 0x0200, 21 | /// Enforce strict netcdf-3 rules 22 | NC_CLASSIC = 0x0100, 23 | /// causes netCDF to create a HDF5/NetCDF-4 file 24 | NC_NETCDF4 = 0x1000 25 | }; 26 | } 27 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/Providers.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.IO; 7 | 8 | namespace Microsoft.Research.Science.Data 9 | { 10 | /// 11 | /// Determines how to find out the value. 12 | /// 13 | public enum ReverseIndexSelection 14 | { 15 | /// 16 | /// Return an exact value when given coordinates are exactly found; 17 | /// otherwise, throw an exception. 18 | /// 19 | Exact, 20 | /// 21 | /// If no exact value is found for given coordinates, take nearest exact value. 22 | /// 23 | Nearest, 24 | /// 25 | /// If no exact value is found for given coordinates, interpolate the value from 26 | /// its exact neighbours. 27 | /// 28 | Interpolation 29 | } 30 | } 31 | 32 | -------------------------------------------------------------------------------- /FetchClimate1/ClimateServiceClient/HandlerClass.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading; 7 | 8 | namespace Microsoft.Research.Science.Data 9 | { 10 | internal sealed class OnCommittedHandler 11 | { 12 | private string hash; 13 | private EventWaitHandle waitHandle; 14 | private Action CustomHandler; 15 | 16 | public OnCommittedHandler(EventWaitHandle waitHandle, Action handler) 17 | { 18 | this.waitHandle = waitHandle; 19 | this.CustomHandler = handler; 20 | } 21 | 22 | public void Handler(object sender, DataSetCommittedEventArgs arg) 23 | { 24 | if (CustomHandler != null) 25 | CustomHandler(arg, this); 26 | } 27 | 28 | public string RequestHash 29 | { 30 | get { return hash; } 31 | } 32 | 33 | public EventWaitHandle Completed 34 | { 35 | get { return waitHandle; } 36 | } 37 | } 38 | } 39 | 40 | -------------------------------------------------------------------------------- /ScientificDataSet/Providers/CSV/CsvParsingFailedException.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | 7 | namespace Microsoft.Research.Science.Data.CSV 8 | { 9 | /// 10 | /// The exception that is thrown when cannot parse input file. 11 | /// 12 | [global::System.Serializable] 13 | public class CsvParsingFailedException : ApplicationException 14 | { 15 | /// 16 | public CsvParsingFailedException() { } 17 | 18 | /// 19 | public CsvParsingFailedException(string message) : base(message) { } 20 | /// 21 | public CsvParsingFailedException(string message, Exception inner) : base(message, inner) { } 22 | /// 23 | protected CsvParsingFailedException( 24 | System.Runtime.Serialization.SerializationInfo info, 25 | System.Runtime.Serialization.StreamingContext context) 26 | : base(info, context) { } 27 | } 28 | } 29 | 30 | -------------------------------------------------------------------------------- /Licence.txt: -------------------------------------------------------------------------------- 1 | SDSLite 2 | 3 | Copyright (c) Microsoft Corporation 4 | 5 | All rights reserved. 6 | 7 | MIT License 8 | 9 | Permission is hereby granted, free of charge, to any person obtaining a copy 10 | of this software and associated documentation files (the Software), to deal 11 | in the Software without restriction, including without limitation the rights 12 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 13 | copies of the Software, and to permit persons to whom the Software is 14 | furnished to do so, subject to the following conditions: 15 | 16 | The above copyright notice and this permission notice shall be included in all 17 | copies or substantial portions of the Software. 18 | 19 | THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 22 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 24 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 25 | SOFTWARE. 26 | -------------------------------------------------------------------------------- /ScientificDataSet/ScientificDataSet.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | true 6 | false 7 | false 8 | true 9 | SDSLite 10 | A cross platform .NET library for manipulating netCDF, CSV and TSV files. 11 | Microsoft 12 | Microsoft Research, Computational Science group 13 | http://research.microsoft.com/en-us/projects/sds 14 | https://github.com/predictionmachines/SDSlite 15 | git 16 | MIT 17 | README.md 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /sdsutil/MetadataConflictResolver.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | 6 | namespace sdsutil 7 | { 8 | interface IMetadataConflictResolver 9 | { 10 | /// 11 | /// Resolves conflicting attribute presented in two merged DataSets. 12 | /// 13 | /// Conflicting attribute value. 14 | /// First alternative. 15 | /// Second alternative. 16 | /// 17 | object Resolve(string attribute, object value1, object value2); 18 | } 19 | 20 | class WarningConflictResolver : IMetadataConflictResolver 21 | { 22 | #region IMetadataConflictResolver Members 23 | 24 | public object Resolve(string attribute, object value1, object value2) 25 | { 26 | Console.ForegroundColor = ConsoleColor.Yellow; 27 | Console.Error.WriteLine("Attribute " + attribute + " is conflicting; first value is chosen."); 28 | Console.ResetColor(); 29 | return value1; 30 | } 31 | 32 | #endregion 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /FetchClimate1/ClimateService.Common/Exceptions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | 6 | namespace Microsoft.Research.Science.Data.Climate.Common 7 | { 8 | public class DataProcComputationException : Exception 9 | { 10 | public DataProcComputationException(string mess) 11 | : base(mess) 12 | { } 13 | } 14 | public class NoProcessorAvailableException : Exception 15 | { 16 | public NoProcessorAvailableException(string mess) 17 | : base(mess) 18 | { } 19 | } 20 | 21 | public class MissingValuePresentException : DataProcComputationException 22 | { 23 | public MissingValuePresentException(string mess) 24 | : base(mess) 25 | { } 26 | } 27 | 28 | public class TooLargeDataException : DataProcComputationException 29 | { 30 | public TooLargeDataException(string mess) 31 | : base(mess) 32 | { } 33 | } 34 | 35 | public class DataAggregationException : DataProcComputationException 36 | { 37 | public DataAggregationException(string mess) 38 | : base(mess) 39 | { } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | ## [Unreleased] 9 | 10 | ## [3.0.1] - 2023-06-02 11 | 12 | ### Added 13 | - Exposed `nc_inq_libvers` function as `string NetCDFInterop.NetCDF.nc_inq_libvers()`. 14 | ## [3.0.0] - 2023-06-01 15 | 16 | ### Added 17 | - Tests for `.NETFramework,Version=v4.8`. 18 | 19 | ### Removed 20 | - Obsolete `CoordinateSystem` class and all related functionality. 21 | - Obsolete `AddVariableByValue` methods. 22 | 23 | ### Changed 24 | - `CsvDataset` now writes floating point values using the `R` format specifier. 25 | This ensures that the value doesn't change when it is read back. 26 | 27 | ## [2.0.4] - 2023-05-14 28 | 29 | ### Added 30 | - Tests for CSV dataset. 31 | - This `CHANGELOG.md` 32 | 33 | ### Changed 34 | - `netCDF` cross-platform P/Invoke is now done by .NET. The dependency on `DynamicInterop` has been removed. 35 | - `CsvDataset` handles absence of `MissingValue` attribute. This closes #38. 36 | - `README.md` now contains sample code. It is now packaged in `.nuget`. -------------------------------------------------------------------------------- /ScientificDataSet/Providers/NetCDF/NetCDFException.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using NetCDFInterop; 7 | 8 | namespace Microsoft.Research.Science.Data.NetCDF4 9 | { 10 | /// 11 | /// A fault within the unmanaged code of the NetCDF library. 12 | /// 13 | public class NetCDFException : Exception 14 | { 15 | private int resultCode; 16 | /// 17 | /// 18 | /// 19 | /// 20 | public NetCDFException(int resultCode) : 21 | base(NetCDF.nc_strerror(resultCode)) 22 | { 23 | this.resultCode = resultCode; 24 | } 25 | /// 26 | /// 27 | /// 28 | /// 29 | public NetCDFException(string message) : base(message) 30 | { 31 | this.resultCode = -1; 32 | } 33 | /// 34 | /// Gets the unmanaged NetCDF result code, describing the fault. 35 | /// 36 | public int ResultCode 37 | { 38 | get { return resultCode; } 39 | } 40 | } 41 | } 42 | 43 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/MetadataChanges.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | 7 | namespace Microsoft.Research.Science.Data 8 | { 9 | /// 10 | /// Stores changes of the metadata dictionary. 11 | /// 12 | public class MetadataChanges : MetadataDictionary 13 | { 14 | internal MetadataChanges(Dictionary dictionary) 15 | : base(true, dictionary) 16 | { 17 | } 18 | /// 19 | /// Gets the attribute value for the given . 20 | /// 21 | /// Key of the attribute to get. 22 | /// Attribute value. 23 | public override object this[string key] 24 | { 25 | get 26 | { 27 | return base[key]; 28 | } 29 | set 30 | { 31 | CheckKey(key); 32 | CheckValue(value); 33 | 34 | // Constraints on name 35 | if (key == KeyForName) 36 | { 37 | if (value != null && !(value is string)) 38 | throw new Exception("Name of a variable must be a string"); 39 | } 40 | 41 | dictionary[key] = value; 42 | } 43 | } 44 | } 45 | } 46 | 47 | -------------------------------------------------------------------------------- /FetchClimate1/ClimateService.Common/RestApiNamings.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | 6 | namespace Microsoft.Research.Science.Data.Climate 7 | { 8 | /// 9 | /// Static class with different constants, used in Fetch Climate REST API. 10 | /// 11 | public static class RestApiNamings 12 | { 13 | /// 14 | /// Binary request type name. 15 | /// 16 | public const string binaryRequestTypeName = "application/x-www-form-urlencoded"; 17 | 18 | /// 19 | /// Csv request type name. 20 | /// 21 | public const string csvRequestTypeName = "text/csv"; 22 | 23 | /// 24 | /// Text request type name. 25 | /// 26 | public const string textRequestTypeName = "text/plain"; 27 | 28 | /// 29 | /// Xml request type name. 30 | /// 31 | public const string xmlRequestTypeName = "application/xml"; 32 | 33 | /// 34 | /// POST request method name. 35 | /// 36 | public const string postRequestMethodName = "POST"; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /ScientificDataSet/Providers/NetCDF/Interop/NcType.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | 6 | namespace NetCDFInterop 7 | { 8 | /// 9 | /// The netcdf external data types 10 | /// 11 | public enum NcType : int 12 | { 13 | /// signed 1 byte integer 14 | NC_BYTE = 1, 15 | /// unsigned 1 byte int 16 | NC_UBYTE = 7, 17 | /// ISO/ASCII character 18 | NC_CHAR = 2, 19 | /// signed 2 byte integer 20 | NC_SHORT = 3, 21 | /// unsigned 2 byte integer 22 | NC_USHORT = 8, 23 | /// signed 4 byte integer 24 | NC_INT = 4, 25 | /// unsigned 4-byte int 26 | NC_UINT = 9, 27 | /// single precision floating point number 28 | NC_FLOAT = 5, 29 | /// double precision floating point number 30 | NC_DOUBLE = 6, 31 | /// signed 8-byte int 32 | NC_INT64 = 10, 33 | /// unsigned 8-byte int 34 | NC_UINT64 = 11, 35 | /// string 36 | NC_STRING = 12 37 | }; 38 | } 39 | -------------------------------------------------------------------------------- /ScientificDataSet/Providers/NetCDF/NetCDFGlobalMetadataVariable.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using NetCDFInterop; 7 | 8 | namespace Microsoft.Research.Science.Data.NetCDF4 9 | { 10 | internal class NetCDFGlobalMetadataVariable : NetCdfVariable 11 | { 12 | public NetCDFGlobalMetadataVariable(NetCDFDataSet ds) 13 | : base(ds, NcConst.NC_GLOBAL, String.Empty, new string[0], false) 14 | { 15 | this.ID = NetCDFDataSet.GlobalMetadataVariableID; 16 | } 17 | 18 | public override Array GetData(int[] origin, int[] shape) 19 | { 20 | if ((origin == null || origin.Length == 0) && 21 | (shape == null || shape.Length == 0)) 22 | return new EmptyValueType[] { new EmptyValueType() }; 23 | throw new ArgumentException("The variable is scalar therefore given arguments are incorrect"); 24 | } 25 | 26 | protected override int[] ReadShape() 27 | { 28 | return new int[0]; 29 | } 30 | 31 | public override void PutData(int[] origin, Array a) 32 | { 33 | throw new NotSupportedException("MetadataContainerVariable contains metadata only"); 34 | } 35 | 36 | public override void Append(Array a, int dimToAppend) 37 | { 38 | throw new NotSupportedException("MetadataContainerVariable contains metadata only"); 39 | } 40 | } 41 | } 42 | 43 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/Exceptions/ResourceNotFoundException.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | 7 | namespace Microsoft.Research.Science.Data 8 | { 9 | /// 10 | /// The exception that is thrown when a particular resource is not found. 11 | /// 12 | [Serializable] 13 | public class ResourceNotFoundException : DataSetException 14 | { 15 | /// 16 | /// 17 | /// 18 | public ResourceNotFoundException() { } 19 | /// 20 | /// 21 | /// 22 | /// 23 | public ResourceNotFoundException(string resourceName) 24 | : base(String.Format("Resource {0} not found", resourceName)) { } 25 | /// 26 | /// 27 | /// 28 | /// 29 | /// 30 | public ResourceNotFoundException(string resourceName, Exception inner) 31 | : base(String.Format("Resource {0} not found", resourceName), inner) { } 32 | /// 33 | /// 34 | /// 35 | /// 36 | /// 37 | protected ResourceNotFoundException( 38 | System.Runtime.Serialization.SerializationInfo info, 39 | System.Runtime.Serialization.StreamingContext context) 40 | : base(info, context) { } 41 | } 42 | } 43 | 44 | -------------------------------------------------------------------------------- /SDSLiteTests/NetCDFTests.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Research.Science.Data; 2 | using Microsoft.Research.Science.Data.NetCDF4; 3 | using NUnit.Framework; 4 | using System; 5 | using System.Linq; 6 | 7 | namespace SDSLiteTests 8 | { 9 | public class NetCDFTests: GenericFileTests 10 | { 11 | protected override DataSetUri dsUri(string fileName, params object[] attributes) 12 | { 13 | var dsUri = new NetCDFUri() 14 | { 15 | FileName = fileName, 16 | }; 17 | if (attributes != null) { 18 | dsUri.Deflate = attributes 19 | .Select(a => a as DeflateLevel?) 20 | .FirstOrDefault(d => d != null) ?? DeflateLevel.Normal; 21 | } 22 | return dsUri; 23 | } 24 | 25 | [Test] 26 | public void Deflate_reduces_file_size() 27 | { 28 | var data = Enumerable.Repeat(Math.PI, 2048).ToArray(); 29 | var len_uncompressed = Empty_AddVariable(data, DeflateLevel.Store); 30 | var len_compressed = Empty_AddVariable(data, DeflateLevel.Best); 31 | Assert.IsTrue(len_compressed < len_uncompressed); 32 | } 33 | 34 | [Test] 35 | public void Test_inq_libvers() 36 | { 37 | var ver = NetCDFInterop.NetCDF.nc_inq_libvers(); 38 | Assert.IsNotNull(ver); 39 | Assert.IsTrue(ver.StartsWith("4.")); 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /FetchClimate1/ClimateService.Common/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("ClimateService.Common")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("Microsoft")] 12 | [assembly: AssemblyProduct("ClimateService.Common")] 13 | [assembly: AssemblyTrademark("")] 14 | [assembly: AssemblyCulture("")] 15 | 16 | // Setting ComVisible to false makes the types in this assembly not visible 17 | // to COM components. If you need to access a type in this assembly from 18 | // COM, set the ComVisible attribute to true on that type. 19 | [assembly: ComVisible(false)] 20 | 21 | // The following GUID is for the ID of the typelib if this project is exposed to COM 22 | [assembly: Guid("bd2da9ee-4417-482e-a2aa-6e13d41a9a17")] 23 | 24 | // Version information for an assembly consists of the following four values: 25 | // 26 | // Major Version 27 | // Minor Version 28 | // Build Number 29 | // Revision 30 | // 31 | // You can specify all the values or you can default the Build and Revision Numbers 32 | // by using the '*' as shown below: 33 | // [assembly: AssemblyVersion("1.0.*")] 34 | //[assembly: AssemblyVersion("1.0.0.0")] 35 | //[assembly: AssemblyFileVersion("1.0.0.0")] 36 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/MetadataContainerVariable.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | 7 | namespace Microsoft.Research.Science.Data 8 | { 9 | /// 10 | /// Designated to contain metadata dictionary without data. 11 | /// 12 | internal class MetadataContainerVariable : Variable 13 | { 14 | private MetadataDictionary metadata; 15 | 16 | public MetadataContainerVariable(DataSet dataSet) : 17 | base(dataSet, new string[0] { }) 18 | { 19 | metadata = new MetadataDictionary(); 20 | 21 | Initialize(); 22 | } 23 | 24 | public override MetadataDictionary Metadata 25 | { 26 | get { return metadata; } 27 | } 28 | 29 | public override Array GetData(int[] origin, int[] shape) 30 | { 31 | if ((origin == null || origin.Length == 0) && 32 | (shape == null || shape.Length == 0)) 33 | return new EmptyValueType[] { new EmptyValueType() }; 34 | throw new ArgumentException("The variable is scalar therefore given arguments are incorrect"); 35 | } 36 | 37 | protected override int[] ReadShape() 38 | { 39 | return new int[0]; 40 | } 41 | 42 | public override void PutData(int[] origin, Array a) 43 | { 44 | throw new NotSupportedException("MetadataContainerVariable contains metadata only"); 45 | } 46 | 47 | public override void Append(Array a, int dimToAppend) 48 | { 49 | throw new NotSupportedException("MetadataContainerVariable contains metadata only"); 50 | } 51 | } 52 | } 53 | 54 | -------------------------------------------------------------------------------- /FetchClimate1/ClimateServiceClient/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System.Reflection; 3 | using System.Runtime.CompilerServices; 4 | using System.Runtime.InteropServices; 5 | 6 | // General Information about an assembly is controlled through the following 7 | // set of attributes. Change these attribute values to modify the information 8 | // associated with an assembly. 9 | [assembly: AssemblyTitle("ClimateClient")] 10 | [assembly: AssemblyDescription("")] 11 | [assembly: AssemblyConfiguration("")] 12 | [assembly: AssemblyCompany("")] 13 | [assembly: AssemblyProduct("ClimateClient")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("7d895d45-acf4-4613-bfb2-65188f85d3c1")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | //[assembly: AssemblyVersion("1.0.0.0")] 36 | //[assembly: AssemblyFileVersion("1.0.0.0")] 37 | 38 | -------------------------------------------------------------------------------- /ScientificDataSet/Utilities/GeoConventions.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | #pragma warning disable 1591 3 | using System; 4 | 5 | namespace Microsoft.Research.Science.Data.Utilities 6 | { 7 | public static class GeoConventions 8 | { 9 | public static bool IsLatitude(Variable v) 10 | { 11 | string units = v.Metadata.GetUnits(); 12 | if (!String.IsNullOrEmpty(units)) 13 | { 14 | units = units.ToLower(); 15 | 16 | // The recommended unit of latitude is degrees_north. 17 | // Also acceptable are degree_north, degree_N, degrees_N, degreeN, and degreesN. 18 | if (units.Contains("degree") && (units.EndsWith("north") || units.EndsWith("n"))) 19 | return true; 20 | } 21 | // Check if name indicates latitute 22 | string name = v.Name.ToLower(); 23 | return (name.StartsWith("lat") || name.StartsWith("_lat") || name.Contains("latitude")); 24 | } 25 | 26 | public static bool IsLongitude(Variable v) 27 | { 28 | string units = v.Metadata.GetUnits(); 29 | if (!String.IsNullOrEmpty(units)) 30 | { 31 | units = units.ToLower(); 32 | 33 | // The recommended unit of longitude is degrees_east. 34 | // Also acceptable are degree_east, degree_E, degrees_E, degreeE, and degreesE. 35 | if (units.Contains("degree") && (units.EndsWith("east") || units.EndsWith("e"))) 36 | return true; 37 | } 38 | // Check if name indicates longitude 39 | string name = v.Name.ToLower(); 40 | return (name.StartsWith("lon") || name.StartsWith("_lon") || name.Contains("longitude")); 41 | } 42 | } 43 | } 44 | 45 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/Exceptions/DataSetException.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | 4 | namespace Microsoft.Research.Science.Data 5 | { 6 | /// 7 | /// The exception that is thrown when a DataSet-related exceptional case occurs. 8 | /// 9 | [Serializable] 10 | public class DataSetException : ApplicationException 11 | { 12 | /// 13 | /// Initializes a new instance of the DataSetException class. 14 | /// 15 | public DataSetException() 16 | { 17 | } 18 | /// 19 | /// Initializes a new instance of the DataSetException class. 20 | /// 21 | /// Error message 22 | public DataSetException(string message) 23 | : base(message) 24 | { 25 | } 26 | /// 27 | /// Initializes a new instance of the DataSetException class. 28 | /// 29 | /// 30 | /// Error message 31 | public DataSetException(string message, Exception innerException) 32 | : base(message, innerException) 33 | { 34 | } 35 | /// Initializes a new instance of the class 36 | /// with serialized data. 37 | /// 38 | /// The object that holds the serialized object data. 39 | /// The contextual information about the source or destination. 40 | protected DataSetException(System.Runtime.Serialization.SerializationInfo info, 41 | System.Runtime.Serialization.StreamingContext context) 42 | : base(info, context) { } 43 | } 44 | } 45 | 46 | -------------------------------------------------------------------------------- /FetchClimate1/ClimateService.Common/DataScale.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Microsoft.Research.Science.Data.Climate.Common 4 | { 5 | public struct DataScale 6 | { 7 | public double AddOffset, ScaleFactor, MissingValue; 8 | public DataScale(Variable v) 9 | { 10 | double add_offset = 0; 11 | double scale_factor = 1.0; 12 | double missingValue = double.NaN; 13 | 14 | string[] AddOffsetKeys = new string[] { "add_offset", "AddOffset" }; 15 | string[] MissingValueKeys = new string[] { "missing_value", "MissingValue" }; 16 | string[] scaleFactorKeys = new string[] { "scale_factor", "ScaleFactor" }; 17 | 18 | foreach (string ao_key in AddOffsetKeys) 19 | if (v.Metadata.ContainsKey(ao_key)) 20 | { 21 | add_offset = Convert.ToDouble(v.Metadata[ao_key]); 22 | break; 23 | } 24 | 25 | foreach (string sf_key in scaleFactorKeys) 26 | if (v.Metadata.ContainsKey(sf_key)) 27 | { 28 | scale_factor = Convert.ToDouble(v.Metadata[sf_key]); 29 | break; 30 | } 31 | 32 | foreach (string mv_key in MissingValueKeys) 33 | if (v.Metadata.ContainsKey(mv_key)) 34 | { 35 | missingValue = Convert.ToDouble(v.Metadata[mv_key]); 36 | break; 37 | } 38 | 39 | MissingValue = missingValue; 40 | AddOffset = add_offset; 41 | ScaleFactor = scale_factor; 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /FetchClimate1/ClimateServiceClient/ServiceLocationConfSection.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Configuration; 6 | 7 | namespace Microsoft.Research.Science.Data.Climate 8 | { 9 | public class ServiceLocationConfiguration : ConfigurationSection 10 | { 11 | private static ServiceLocationConfiguration section = null; 12 | 13 | 14 | public static ServiceLocationConfiguration Current 15 | { 16 | get 17 | { 18 | if (section == null) 19 | { 20 | var conf = ConfigurationManager.OpenExeConfiguration(ConfigurationUserLevel.None); 21 | section = conf.GetSection("ServiceLocationConfiguration") as ServiceLocationConfiguration; 22 | } 23 | return section; 24 | } 25 | } 26 | 27 | [ConfigurationProperty("ServiceURL", IsRequired = false)] 28 | public string ServiceURL 29 | { 30 | get 31 | { 32 | return (string)this["ServiceURL"]; 33 | } 34 | set 35 | { 36 | this["ServiceURL"] = value; 37 | } 38 | } 39 | 40 | [ConfigurationProperty("CommunicationProtocol", IsRequired = false)] 41 | public string CommunicationProtocol 42 | { 43 | get 44 | { 45 | return (string)this["CommunicationProtocol"]; 46 | } 47 | set 48 | { 49 | this["CommunicationProtocol"] = value; 50 | } 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/Exceptions/DistributedCommitFailedException.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | 7 | namespace Microsoft.Research.Science.Data 8 | { 9 | /// 10 | /// The exception that is thrown if during distributed commit of DataSets linked by reference 11 | /// variables, some of these DataSets failed to commit its changes. 12 | /// 13 | [Serializable] 14 | public class DistributedCommitFailedException : DataSetException 15 | { 16 | private DataSet failed; 17 | /// 18 | /// 19 | /// 20 | /// 21 | public DistributedCommitFailedException(DataSet failedDataSet) : base("DataSet " + failedDataSet.URI + " commit failed") 22 | { 23 | failed = failedDataSet; 24 | } 25 | /// 26 | /// 27 | /// 28 | /// 29 | /// 30 | public DistributedCommitFailedException(DataSet failedDataSet, Exception inner) 31 | : base("DataSet " + failedDataSet.URI + " commit failed", inner) 32 | { 33 | failed = failedDataSet; 34 | } 35 | /// 36 | /// 37 | /// 38 | /// 39 | /// 40 | protected DistributedCommitFailedException( 41 | System.Runtime.Serialization.SerializationInfo info, 42 | System.Runtime.Serialization.StreamingContext context) 43 | : base(info, context) { } 44 | 45 | /// 46 | /// Gets the data set that is unable to commit. 47 | /// 48 | public DataSet FailedDataSet 49 | { 50 | get { return failed; } 51 | } 52 | } 53 | } 54 | 55 | -------------------------------------------------------------------------------- /ScientificDataSet/Providers/Memory/MemoryVariable1d.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | 6 | namespace Microsoft.Research.Science.Data.Memory 7 | { 8 | internal class MemoryVariable1d : MemoryVariable 9 | { 10 | public MemoryVariable1d(DataSet sds, string name, string[] dims) 11 | : base(sds, name, dims, new /*OrderedArray1d*/ArrayWrapper(1, typeof(DataType))) 12 | { 13 | if (dims.Length != 1) 14 | throw new Exception("Expected for 1d variable"); 15 | } 16 | 17 | /*protected override void CheckConstraints(Variable.Changes proposedChanges) 18 | { 19 | base.CheckConstraints(proposedChanges); 20 | 21 | OrderedArray1d ordered = (OrderedArray1d)array; 22 | if (ordered.Order == ArrayOrder.None) 23 | return; 24 | 25 | ArrayOrder order = ordered.Order; 26 | if (proposedChanges.DataPieces != null) 27 | foreach (DataPiece piece in proposedChanges.DataPieces) 28 | { 29 | ArrayOrder inFact = OrderedArray1d.IsOrdered((DataType[])piece.Data); 30 | if (inFact == ArrayOrder.Unknown) 31 | continue; 32 | if (inFact == ArrayOrder.None) 33 | throw new Exception("Array " + Name + " is not ordered."); 34 | 35 | if (order == ArrayOrder.Unknown) 36 | order = inFact; 37 | else if (order != inFact) 38 | throw new Exception("Appended array " + Name + " has wrong order."); 39 | } 40 | }*/ 41 | } 42 | } 43 | 44 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/Exceptions/ConstraintsFailedException.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | 4 | namespace Microsoft.Research.Science.Data 5 | { 6 | /// 7 | /// The exception that is thrown when the consistency constraints are failed for the data set. 8 | /// 9 | [Serializable] 10 | public class ConstraintsFailedException : DataSetException 11 | { 12 | /// Initializes a new instance of the class. 13 | public ConstraintsFailedException() { } 14 | /// Initializes a new instance of the 15 | /// class with a specified error message. 16 | /// Error message 17 | public ConstraintsFailedException(string message) : base(message) { } 18 | /// Initializes a new instance of the class 19 | /// with a specified error message and a reference to the inner exception that is the cause of this exception. 20 | /// 21 | /// Error message 22 | /// Exception that causes this exception 23 | public ConstraintsFailedException(string message, Exception inner) : base(message, inner) { } 24 | /// Initializes a new instance of the class 25 | /// with serialized data. 26 | /// 27 | /// The object that holds the serialized object data. 28 | /// The contextual information about the source or destination. 29 | protected ConstraintsFailedException( 30 | System.Runtime.Serialization.SerializationInfo info, 31 | System.Runtime.Serialization.StreamingContext context) 32 | : base(info, context) { } 33 | } 34 | } 35 | 36 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/IChunkSizesAdjustable.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | 6 | namespace Microsoft.Research.Science.Data 7 | { 8 | /// 9 | /// The instance, implementing this interface, allows to adjust chunk sizes. 10 | /// 11 | public interface IChunkSizesAdjustable 12 | { 13 | /// 14 | /// Sets the chunk sizes (in data elements) to be used when new variables of the rank equal to length of the are added. 15 | /// 16 | /// 17 | /// 18 | /// If a instance implements this interface, it stores data by chunks and allows a user to change the chunk sizes. 19 | /// The argument of the method affects new variables when they are being added to the , 20 | /// if their rank is equal to the length of the array. 21 | /// If not, uses default chunk sizes. 22 | /// 23 | /// 24 | /// Each i-th value of the array is a number of elements in a single chunk by index i. 25 | /// 26 | /// 27 | /// 28 | /// DataSet ds = DataSet.Open("data.nc?openMode=create"); 29 | /// IChunkSizesAdjustable adjustChunkSize = (IChunkSizesAdjustable)ds; 30 | /// adjustChunkSize.SetChunkSizes(new int[] { 100000 }); 31 | /// ds.AddVariable<double>("var1", "i"); // chunk size for variable "var1" is 100000 of numbers 32 | /// adjustChunkSize.SetChunkSizes(new int[] { 10000, 10 }); 33 | /// ds.AddVariable<double>("var2", "i", "j"); // chunk size for variable "var1" is 10000 x 10 of numbers 34 | /// 35 | /// 36 | /// 37 | void SetChunkSizes(int[] sizes); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /ScientificDataSet/Providers/CSV/CsvVariablesScalar.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections; 4 | using System.Collections.Generic; 5 | using System.Diagnostics; 6 | using System.Globalization; 7 | using System.IO; 8 | using System.Linq; 9 | using System.Reflection; 10 | using System.Text; 11 | using System.Threading; 12 | 13 | namespace Microsoft.Research.Science.Data.CSV 14 | { 15 | internal sealed class CsvVariableScalar : CsvVariable 16 | { 17 | internal CsvVariableScalar(CsvDataSet dataSet, CsvColumn column) 18 | : base(dataSet, column) 19 | { 20 | if (column.Rank != 0) 21 | throw new Exception("This is a scalar variable and is being created with different rank."); 22 | 23 | data = new ArrayWrapper(0, typeof(DataType));//new OrderedArray1d(typeof(DataType), false); 24 | 25 | Initialize(); 26 | } 27 | 28 | private CsvVariableScalar(CsvDataSet dataSet, int id, MetadataDictionary metadata, ArrayWrapper data, string[] dims) 29 | : base(dataSet, id, metadata, dims) 30 | { 31 | this.data = data; 32 | Initialize(); 33 | } 34 | 35 | public override Variable CloneAndRenameDims(string[] newDims) 36 | { 37 | if (newDims != null && newDims.Length != 0) 38 | throw new Exception("New dimensions are wrong"); 39 | Variable var = new CsvVariableScalar((CsvDataSet)DataSet, ID, Metadata, data, newDims); 40 | return var; 41 | } 42 | 43 | /// 44 | /// Builds and returns inner data to be presented as a single column. 45 | /// 46 | protected override Array GetInnerData() 47 | { 48 | return data.Data; 49 | } 50 | 51 | protected override void InnerInitialize(Array data, int[] shape) 52 | { 53 | this.data.PutData(null, data); 54 | ChangesUpdateShape(this.changes, ReadShape()); 55 | } 56 | } 57 | } 58 | 59 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/Dimension.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | 7 | namespace Microsoft.Research.Science.Data 8 | { 9 | /// 10 | /// Represents a dimension of a variable. 11 | /// 12 | public struct Dimension 13 | { 14 | private string name; 15 | private int length; 16 | 17 | /// 18 | /// 19 | /// 20 | /// 21 | /// 22 | internal Dimension(string name, int length) 23 | { 24 | this.name = name; 25 | this.length = length; 26 | } 27 | 28 | /// 29 | /// Gets the name of the dimension. 30 | /// 31 | public string Name 32 | { 33 | get { return name; } 34 | internal set { name = value; } 35 | } 36 | 37 | /// 38 | /// Gets the length of the dimension. 39 | /// 40 | /// 41 | /// This is a length of data of all variables depending on it, by the corresponding dimension. 42 | /// 43 | public int Length 44 | { 45 | get 46 | { 47 | return length; 48 | } 49 | internal set 50 | { 51 | length = value; 52 | } 53 | } 54 | 55 | /// 56 | /// Represents the dimension as a text. 57 | /// 58 | /// 59 | public override string ToString() 60 | { 61 | return string.Format("({0}:{1})", String.IsNullOrEmpty(name) ? "_" : name, length); 62 | } 63 | 64 | /// 65 | /// Returns the hash code for this instance. 66 | /// 67 | /// The hash code for this instance. 68 | public override int GetHashCode() 69 | { 70 | if (name != null) 71 | return name.GetHashCode(); 72 | return 0; 73 | } 74 | } 75 | } 76 | 77 | -------------------------------------------------------------------------------- /FetchClimate1/ClimateServiceClient/CredentialsManager.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using Microsoft.Win32; 6 | 7 | namespace Microsoft.Research.Science.Data 8 | { 9 | public class CredentialsManager 10 | { 11 | private const string DefaultLogin = "anonymous"; 12 | private const string DefaultPassword = "anonymous"; 13 | 14 | public static void GetCredentials(out string login, out string password) 15 | { 16 | RegistryKey rkey = null; 17 | RegistryKey rsubkey = null; 18 | 19 | try 20 | { 21 | rkey = Registry.CurrentUser; 22 | rsubkey = rkey.OpenSubKey("SOFTWARE\\Microsoft"); 23 | 24 | if (rsubkey == null || 25 | !rsubkey.GetSubKeyNames().Contains("FetchClimate")) 26 | { 27 | login = CredentialsManager.DefaultLogin; 28 | password = CredentialsManager.DefaultPassword; 29 | return; 30 | } 31 | 32 | rsubkey.Close(); 33 | rsubkey = rkey.OpenSubKey("SOFTWARE\\Microsoft\\FetchClimate"); 34 | 35 | string privilegedLogin = rsubkey.GetValue("login") as string; 36 | string privilegedPassword = rsubkey.GetValue("password") as string; 37 | 38 | if (string.IsNullOrEmpty(privilegedLogin) || string.IsNullOrEmpty(privilegedPassword)) 39 | { 40 | login = CredentialsManager.DefaultLogin; 41 | password = CredentialsManager.DefaultPassword; 42 | return; 43 | } 44 | 45 | login = privilegedLogin; 46 | password = privilegedPassword; 47 | 48 | return; 49 | } 50 | finally 51 | { 52 | if (rkey != null) 53 | rkey.Close(); 54 | if (rsubkey != null) 55 | rsubkey.Close(); 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /ScientificDataSet/Providers/CSV/CsvVariables1d.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections; 4 | using System.Collections.Generic; 5 | using System.Diagnostics; 6 | using System.Globalization; 7 | using System.IO; 8 | using System.Linq; 9 | using System.Reflection; 10 | using System.Text; 11 | using System.Threading; 12 | 13 | namespace Microsoft.Research.Science.Data.CSV 14 | { 15 | internal sealed class CsvVariable1d : CsvVariable 16 | { 17 | internal CsvVariable1d(CsvDataSet dataSet, CsvColumn column) 18 | : base(dataSet, column) 19 | { 20 | if (column.Rank != 1) 21 | throw new Exception("This is 1d-variable and is being created with different rank."); 22 | 23 | data = new ArrayWrapper(1, typeof(DataType));//new OrderedArray1d(typeof(DataType), false); 24 | 25 | Initialize(); 26 | } 27 | 28 | private CsvVariable1d(CsvDataSet dataSet, int id, MetadataDictionary metadata, ArrayWrapper data, string[] dims) 29 | : base(dataSet, id, metadata, dims) 30 | { 31 | this.data = data; 32 | Initialize(); 33 | } 34 | 35 | public override Variable CloneAndRenameDims(string[] newDims) 36 | { 37 | if (newDims == null || Rank != newDims.Length) 38 | throw new Exception("New dimensions are wrong"); 39 | Variable var = new CsvVariable1d((CsvDataSet)DataSet, ID, Metadata, data, newDims); 40 | return var; 41 | } 42 | 43 | /// 44 | /// Builds and returns inner data to be presented as a single column. 45 | /// 46 | protected override Array GetInnerData() 47 | { 48 | return data.Data; 49 | } 50 | 51 | protected override void InnerInitialize(Array data, int[] shape) 52 | { 53 | this.data.PutData(null, data); 54 | ChangesUpdateShape(this.changes, ReadShape()); 55 | } 56 | } 57 | } 58 | 59 | -------------------------------------------------------------------------------- /ScientificDataSet.DataSetReplicator/DataSetCommittedEventManager.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Windows; 7 | 8 | namespace Microsoft.Research.Science.Data.Utilities 9 | { 10 | public class DataSetCommittedEventManager : WeakEventManager 11 | { 12 | private DataSetCommittedEventManager() 13 | { 14 | } 15 | 16 | public static void AddListener(DataSet dataSet, IWeakEventListener listener) 17 | { 18 | DataSetCommittedEventManager.CurrentManager.ProtectedAddListener(dataSet, listener); 19 | } 20 | 21 | public static void RemoveListener(DataSet dataSet, IWeakEventListener listener) 22 | { 23 | DataSetCommittedEventManager.CurrentManager.ProtectedRemoveListener(dataSet, listener); 24 | } 25 | 26 | private void OnDataSetCommitted(object sender, DataSetCommittedEventArgs args) 27 | { 28 | base.DeliverEvent(sender, args); 29 | } 30 | 31 | protected override void StartListening(Object source) 32 | { 33 | DataSet dataSet = (DataSet)source; 34 | dataSet.Committed += this.OnDataSetCommitted; 35 | } 36 | 37 | protected override void StopListening(Object source) 38 | { 39 | DataSet dataSet = (DataSet)source; 40 | dataSet.Committed -= this.OnDataSetCommitted; 41 | } 42 | 43 | private static DataSetCommittedEventManager CurrentManager 44 | { 45 | get 46 | { 47 | Type managerType = typeof(DataSetCommittedEventManager); 48 | DataSetCommittedEventManager manager = (DataSetCommittedEventManager)WeakEventManager.GetCurrentManager(managerType); 49 | if (manager == null) 50 | { 51 | manager = new DataSetCommittedEventManager(); 52 | WeakEventManager.SetCurrentManager(managerType, manager); 53 | } 54 | return manager; 55 | } 56 | } 57 | } 58 | } 59 | 60 | -------------------------------------------------------------------------------- /.github/workflows/build-test.yml: -------------------------------------------------------------------------------- 1 | name: build-test 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | env: 9 | URL_NETCDF_WIN: https://downloads.unidata.ucar.edu/netcdf-c/4.9.2/netCDF4.9.2-NC4-64.exe 10 | jobs: 11 | build-test-windows: 12 | 13 | runs-on: windows-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: Setup .NET 18 | uses: actions/setup-dotnet@v1 19 | with: 20 | dotnet-version: 6.0.x 21 | - name: Restore dependencies 22 | run: dotnet restore 23 | - name: Build 24 | run: dotnet build --no-restore 25 | - name: Download NetCDF 26 | shell: pwsh 27 | run: | 28 | Invoke-WebRequest $env:URL_NETCDF_WIN -OutFile ${{ runner.temp }}\netcdf.exe 29 | 7z x ${{ runner.temp }}\netcdf.exe bin/*.dll -o${{ runner.temp }} -y 30 | Get-ChildItem ${{ runner.temp }}\bin 31 | - name: Test 32 | run: dotnet test --no-build --verbosity normal 33 | env: 34 | LIBNETCDFPATH: ${{ runner.temp }}\bin\netcdf.dll 35 | 36 | build-test-linux: 37 | 38 | runs-on: ubuntu-latest 39 | 40 | steps: 41 | - uses: actions/checkout@v2 42 | - name: Setup .NET 43 | uses: actions/setup-dotnet@v1 44 | with: 45 | dotnet-version: 6.0.x 46 | - name: Restore dependencies 47 | run: dotnet restore 48 | - name: Build 49 | run: dotnet build --no-restore ScientificDataSet 50 | - name: Download NetCDF 51 | run: sudo apt-get install libnetcdf-dev 52 | - name: Test 53 | run: dotnet test -f net6.0 --verbosity normal SDSLiteTests 54 | 55 | build-test-macos: 56 | 57 | runs-on: macos-latest 58 | 59 | steps: 60 | - uses: actions/checkout@v2 61 | - name: Setup .NET 62 | uses: actions/setup-dotnet@v1 63 | with: 64 | dotnet-version: 6.0.x 65 | - name: Restore dependencies 66 | run: dotnet restore 67 | - name: Build 68 | run: dotnet build --no-restore ScientificDataSet 69 | - name: Download NetCDF 70 | env: 71 | HOMEBREW_NO_INSTALL_CLEANUP: 1 72 | run: brew install netcdf 73 | - name: Test 74 | run: dotnet test -f 6.0 --verbosity normal SDSLiteTests 75 | -------------------------------------------------------------------------------- /FetchClimate1/ClimateServiceClient/NonDisposingServicePort.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | 6 | namespace Microsoft.Research.Science.Data 7 | { 8 | #if FC_WCFCLIENT 9 | public class NonDisposingServicePort : IServicePort 10 | { 11 | private IServicePort innerServicePort; 12 | 13 | public NonDisposingServicePort(IServicePort servicePort) 14 | { 15 | if (servicePort == null) 16 | throw new ArgumentNullException("servicePort"); 17 | this.innerServicePort = servicePort; 18 | } 19 | 20 | public IServicePort ServicePort { get { return innerServicePort; } } 21 | 22 | #region IServicePort Members 23 | 24 | public void PostSubscribeRequest(INotifyPort message, IResponsePort responsePort) 25 | { 26 | innerServicePort.PostSubscribeRequest(message, responsePort); 27 | } 28 | 29 | public void PostUnsubscribeRequest(int subscriptionId, IResponsePort responsePort) 30 | { 31 | innerServicePort.PostUnsubscribeRequest(subscriptionId, responsePort); 32 | } 33 | 34 | public void PostGetSchemaRequest(int dataSetId, ISchemaResponsePort responsePort) 35 | { 36 | innerServicePort.PostGetSchemaRequest(dataSetId, responsePort); 37 | } 38 | 39 | public void PostGetMultipleDataRequest(Pipeline.GetDataRequest[] message, IMultipleDataResponsePort responsePort) 40 | { 41 | innerServicePort.PostGetMultipleDataRequest(message, responsePort); 42 | } 43 | 44 | public void PostGetDataRequest(Pipeline.GetDataRequest message, IDataResponsePort responsePort) 45 | { 46 | innerServicePort.PostGetDataRequest(message, responsePort); 47 | } 48 | 49 | public void PostUpdateRequest(Pipeline.UpdateRequest message, IUpdateResponsePort responsePort) 50 | { 51 | innerServicePort.PostUpdateRequest(message, responsePort); 52 | } 53 | 54 | public void PostAttachRequest(string constructionString, IResponsePort responsePort) 55 | { 56 | innerServicePort.PostAttachRequest(constructionString, responsePort); 57 | } 58 | 59 | public void PostDetachRequest(int dataSetId, IResponsePort responsePort) 60 | { 61 | innerServicePort.PostDetachRequest(dataSetId, responsePort); 62 | } 63 | 64 | #endregion 65 | } 66 | #endif 67 | } 68 | -------------------------------------------------------------------------------- /FetchClimate1/ClimateService.Common/Extensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using Microsoft.Research.Science.Data.Climate.Common; 6 | using Microsoft.Research.Science.Data.Climate.Conventions; 7 | 8 | namespace Microsoft.Research.Science.Data.Climate 9 | { 10 | public static class Extensions 11 | { 12 | public static SpatialCell CorrectLonsTo0_360(this SpatialCell cq) 13 | { 14 | if (Math.Abs(cq.LonMin - cq.LonMax) == 360) 15 | { 16 | cq.LonMin = 0; 17 | cq.LonMax = 360; 18 | return cq; 19 | } 20 | if ((cq.LonMax == 0 && cq.LonMin == 0)) 21 | return cq; 22 | if ((cq.LonMax == 360 && cq.LonMin == 360)) 23 | { 24 | cq.LonMin = cq.LonMax = 0; 25 | return cq; 26 | } 27 | if (cq.LonMax <= 0) 28 | cq.LonMax += 360; 29 | if (cq.LonMin < 0) 30 | cq.LonMin += 360; 31 | if (cq.LonMax >= 0 && cq.LonMax <= 360 && cq.LonMin == 360) 32 | cq.LonMin = 0; 33 | return cq; 34 | } 35 | 36 | public static SpatialCell CorrectLonsTo180_180(this SpatialCell cq) 37 | { 38 | if (cq.LonMax - cq.LonMin == 360) 39 | { 40 | cq.LonMin = -180; 41 | cq.LonMax = 180; 42 | return cq; 43 | } 44 | if (cq.LonMin == -180 && cq.LonMax == -180) 45 | return cq; 46 | if (cq.LonMax == 180 && cq.LonMin == 180) 47 | { 48 | cq.LonMin = cq.LonMax = -180; 49 | return cq; 50 | } 51 | if (cq.LonMax > 180) 52 | cq.LonMax -= 360; 53 | if (cq.LonMin >= 180) 54 | cq.LonMin -= 360; 55 | 56 | 57 | 58 | 59 | 60 | return cq; 61 | } 62 | 63 | public static CellQuery TransformDefaultToExact(this CellQuery cq) 64 | { 65 | if (cq.HourMin == GlobalConsts.DefaultValue) 66 | cq.HourMin = 0; 67 | if (cq.HourMax == GlobalConsts.DefaultValue) 68 | cq.HourMax = 24; 69 | if (cq.DayMin == GlobalConsts.DefaultValue) 70 | cq.DayMin = 1; 71 | if (cq.DayMax == GlobalConsts.DefaultValue) 72 | cq.DayMax = 360; 73 | return cq; 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/IndexTransformVariable.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Diagnostics; 5 | 6 | namespace Microsoft.Research.Science.Data 7 | { 8 | internal class IndexTransformVariable : PureComputationalVariable 9 | { 10 | private readonly Func indexLambda; 11 | 12 | 13 | internal IndexTransformVariable(DataSet sds, string name, string[] dims, 14 | Func indexTransform) 15 | : base(sds, name, dims) 16 | { 17 | if (name == null) throw new ArgumentNullException("name"); 18 | if (sds == null) throw new ArgumentNullException("sds"); 19 | if (dims == null) throw new ArgumentNullException("dims"); 20 | if (indexTransform == null) throw new ArgumentNullException("indexTransform"); 21 | this.indexLambda = indexTransform; 22 | IsReadOnly = true; 23 | 24 | Initialize(); 25 | } 26 | 27 | public override Array GetData(int[] origin, int[] shape) 28 | { 29 | int[] lShape = this.GetShape(); 30 | if (shape == null) shape = this.GetShape(); 31 | if (origin == null) 32 | { 33 | origin = new int[this.Rank]; 34 | for (int i = 0; i < this.Rank; i++) origin[i] = 0; 35 | } 36 | for (int i = 0; i < this.Rank; i++) 37 | if (shape[i] <= 0) throw new Exception( 38 | "Shape can't be nonpositive"); 39 | for (int i = 0; i < this.Rank; i++) 40 | if (origin[i] + shape[i] > lShape[i]) 41 | throw new Exception("Index of requested data is out of range"); 42 | 43 | 44 | Array A = Array.CreateInstance(typeof(DataType), shape); 45 | 46 | int[] ind = new int[this.Rank]; 47 | for (int i = 0; i < this.Rank; i++) ind[i] = 0; 48 | 49 | int[] argI = new int[this.Rank]; 50 | 51 | while (ind[0] < shape[0]) 52 | { 53 | for (int k = 0; k < this.Rank; k++) argI[k] = origin[k] + ind[k]; 54 | 55 | A.SetValue(this.indexLambda(argI), ind); 56 | 57 | for (int j = this.Rank - 1; j >= 0; j--) 58 | { 59 | ind[j]++; 60 | if (ind[j] < shape[j]) break; 61 | else 62 | if (j > 0) ind[j] = 0; 63 | } 64 | } 65 | 66 | return A; 67 | } 68 | } 69 | 70 | internal class RangeVariable : IndexTransformVariable 71 | { 72 | internal RangeVariable(DataSet sds, string name, string[] dims, int origin) 73 | : base(sds, name, dims, (int[] i) => ((int)(i[0] + origin))) 74 | { 75 | if ((dims != null) && (dims.Length != 1)) 76 | throw new Exception("Must be one dimensions in RangeVariable"); 77 | } 78 | } 79 | 80 | } 81 | 82 | -------------------------------------------------------------------------------- /SDSLiteTests/Repro.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Research.Science.Data; 2 | using NUnit.Framework; 3 | using System; 4 | using System.Collections.Generic; 5 | using System.IO; 6 | using System.Linq; 7 | using System.Text; 8 | using System.Threading.Tasks; 9 | 10 | namespace SDSLiteTests 11 | { 12 | public class Repro_28 13 | { 14 | // Repro issue #28 15 | [Test] 16 | public void CanReadNetcdfNcChar() 17 | { 18 | var ncpath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".nc"); 19 | File.WriteAllBytes(ncpath, Convert.FromBase64String(netcdf_content)); 20 | try 21 | { 22 | using var ds = DataSet.Open(ncpath, ResourceOpenMode.Open); 23 | Assert.AreEqual(1, ds.Variables.Count); 24 | Assert.AreEqual(2, ds.Dimensions.Count); 25 | Assert.AreEqual("test", ds.Variables[0].Name); 26 | Assert.AreEqual("count", ds.Dimensions[0].Name); 27 | Assert.AreEqual(1, ds.Dimensions[0].Length); 28 | Assert.AreEqual("size", ds.Dimensions[1].Name); 29 | Assert.AreEqual(8, ds.Dimensions[1].Length); 30 | var data = ds[0].GetData(); 31 | Assert.AreEqual(2, data.Rank); 32 | Assert.AreEqual("value ", Encoding.UTF8.GetString((data.Cast().ToArray()))); 33 | ds[0].Append(Encoding.UTF8.GetBytes("addition")); 34 | Assert.AreEqual(2, ds.Dimensions[0].Length); 35 | data = ds[0].GetData(); 36 | Assert.AreEqual("value addition", Encoding.UTF8.GetString((data.Cast().ToArray()))); 37 | } 38 | finally 39 | { 40 | File.Delete(ncpath); 41 | } 42 | } 43 | // [1] test of type SByte (count:1) (size:8) 44 | const string netcdf_content = @" 45 | Q0RGAQAAAAEAAAAKAAAAAgAAAARzaXplAAAACAAAAAVjb3VudAAAAAAAAAAAAAAAAAAAAAAAAAsA 46 | AAABAAAABHRlc3QAAAACAAAAAQAAAAAAAAAAAAAAAAAAAAIAAAAIAAAAZHZhbHVlICAg"; 47 | } 48 | 49 | public class Repro_38 50 | { 51 | // Repro issue #38 52 | [Test] 53 | public void CsvAddVariableNoMissingValue() 54 | { 55 | var csvPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".csv"); 56 | try 57 | { 58 | using (var sdsout = DataSet.Open(csvPath, ResourceOpenMode.Create)) 59 | { 60 | sdsout.AddVariable("a", "a"); 61 | } 62 | 63 | } 64 | finally 65 | { 66 | File.Delete(csvPath); 67 | } 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /ScientificDataSet/Providers/NetCDF/ChunkSizeSelector.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | 6 | namespace Microsoft.Research.Science.Data.NetCDF4 7 | { 8 | internal static class ChunkSizeSelector 9 | { 10 | public const int suggestedChunkPower = 13; // 8 Kb 11 | public const int maxAllowedChunkPower = 17; // 128 Kb 12 | 13 | public static int GetChunkSize(Type type, int rank) 14 | { 15 | int sizeBytes = GetSizeOfType(type); 16 | int s = Log2m((uint)sizeBytes); 17 | for (int n = suggestedChunkPower; n <= maxAllowedChunkPower; n++) 18 | { 19 | if (n <= s) continue; 20 | int p = (n - s) / rank; 21 | if (p * rank == n - s) return Pow(p); 22 | } 23 | for (int n = suggestedChunkPower - 1; n > s; n--) 24 | { 25 | int p = (n - s) / rank; 26 | if (p * rank == n - s) return Pow(p); 27 | } 28 | return 1; 29 | } 30 | 31 | 32 | static int Pow(int p) 33 | { 34 | return 1 << p; 35 | } 36 | 37 | static int Log2m(uint n) 38 | { 39 | if (n < 0) throw new ArgumentException("n must be positive"); 40 | if (n == 1) return 0; 41 | int log = 32; // max 42 | while ((n & 0x80000000) == 0) 43 | { 44 | n <<= 1; 45 | log--; 46 | } 47 | if ((n ^ 0x80000000) == 0) log--; 48 | return log; 49 | } 50 | 51 | private static int GetSizeOfType(Type type) 52 | { 53 | if (type == typeof(Double)) 54 | return sizeof(Double); 55 | else if (type == typeof(Single)) 56 | return sizeof(Single); 57 | else if (type == typeof(Int16)) 58 | return sizeof(Int16); 59 | else if (type == typeof(Int32)) 60 | return sizeof(Int32); 61 | else if (type == typeof(Int64)) 62 | return sizeof(Int64); 63 | else if (type == typeof(UInt64)) 64 | return sizeof(UInt64); 65 | else if (type == typeof(UInt32)) 66 | return sizeof(UInt32); 67 | else if (type == typeof(UInt16)) 68 | return sizeof(UInt16); 69 | else if (type == typeof(Byte)) 70 | return sizeof(Byte); 71 | else if (type == typeof(SByte)) 72 | return sizeof(SByte); 73 | else if (type == typeof(String)) 74 | return 32; 75 | else if (type == typeof(DateTime)) 76 | return sizeof(double); 77 | else if (type == typeof(Boolean)) 78 | return sizeof(byte); 79 | return 1; 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /SDSLiteTests/CsvTests.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Research.Science.Data; 2 | using Microsoft.Research.Science.Data.CSV; 3 | using NUnit.Framework; 4 | using SDSLiteTests; 5 | using System.IO; 6 | 7 | namespace SdsLiteTests 8 | { 9 | public class CsvTests : GenericFileTests 10 | { 11 | protected override DataSetUri dsUri(string fileName, params object[] attributes) 12 | { 13 | var dsUri = new CsvUri() 14 | { 15 | FileName = fileName, 16 | }; 17 | return dsUri; 18 | } 19 | [Test] 20 | public void NoMissingValueAttribute() 21 | { 22 | var fn = Path.GetTempFileName(); 23 | File.WriteAllText(fn, "a\n0"); // minimum viable csv 24 | var dsuri = new CsvUri(); 25 | dsuri.FileName = fn; 26 | dsuri.OpenMode = ResourceOpenMode.ReadOnly; 27 | var ds = DataSet.Open(dsuri); 28 | Assert.AreEqual(1, ds.Variables.Count); 29 | Assert.IsTrue(ds.Variables[0].Metadata.ContainsKey("Name")); 30 | Assert.IsTrue(ds.Variables[0].Metadata.ContainsKey("DisplayName")); 31 | Assert.IsTrue(ds.Variables[0].Metadata.ContainsKey("csv_column")); 32 | Assert.AreEqual(3, ds.Variables[0].Metadata.Count); 33 | } 34 | 35 | [Test] 36 | public void InferMissingValueAttribute() 37 | { 38 | var fn = Path.GetTempFileName(); 39 | File.WriteAllText(fn, "t,u\n0,first\n,second\n1,\n2,last"); 40 | var dsuri = new CsvUri(); 41 | dsuri.FileName = fn; 42 | dsuri.OpenMode = ResourceOpenMode.ReadOnly; 43 | dsuri.FillUpMissingValues = true; // this generates MissingValue attributes 44 | var ds = DataSet.Open(dsuri); 45 | Assert.AreEqual(2, ds.Variables.Count); 46 | { 47 | var v = (Variable)ds.Variables["t"]; 48 | Assert.IsTrue(v.Metadata.ContainsKey("Name")); 49 | Assert.IsTrue(v.Metadata.ContainsKey("DisplayName")); 50 | Assert.IsTrue(v.Metadata.ContainsKey("csv_column")); 51 | Assert.IsTrue(v.Metadata.ContainsKey(v.Metadata.KeyForMissingValue)); 52 | Assert.AreEqual(4, v.Metadata.Count); 53 | Assert.IsTrue(double.IsNaN((double)v.MissingValue)); 54 | Assert.IsTrue(double.IsNaN(v[1])); 55 | } 56 | { 57 | var v = (Variable)ds.Variables["u"]; 58 | Assert.IsTrue(v.Metadata.ContainsKey("Name")); 59 | Assert.IsTrue(v.Metadata.ContainsKey("DisplayName")); 60 | Assert.IsTrue(v.Metadata.ContainsKey("csv_column")); 61 | Assert.IsTrue(v.Metadata.ContainsKey(v.Metadata.KeyForMissingValue)); 62 | Assert.AreEqual(4, v.Metadata.Count); 63 | Assert.IsTrue(v.MissingValue == null); 64 | Assert.IsNull(v[2]); 65 | } 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/IndexResolver.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | 7 | namespace Microsoft.Research.Science.Data 8 | { 9 | internal sealed class IndexResolver 10 | { 11 | private LinkedList currentSet; 12 | private Dictionary dimIndexes; 13 | 14 | public IndexResolver(string[] dims) 15 | { 16 | currentSet = new LinkedList(); 17 | dimIndexes = new Dictionary(dims.Length); 18 | 19 | for (int i = 0; i < dims.Length; i++) 20 | { 21 | dimIndexes[dims[i]] = i; 22 | } 23 | } 24 | 25 | public int[][] GetResolvedIndices() 26 | { 27 | return currentSet.ToArray(); 28 | } 29 | 30 | public void Resolve(int[][] indexSet, string[] dims) 31 | { 32 | if (currentSet.Count == 0) 33 | { 34 | for (int i = 0; i < indexSet.Length; i++) 35 | { 36 | int[] set = new int[dimIndexes.Count]; 37 | for (int j = 0; j < dimIndexes.Count; j++) 38 | set[j] = -1; 39 | for (int j = 0; j < dims.Length; j++) 40 | { 41 | set[dimIndexes[dims[j]]] = indexSet[i][j]; 42 | } 43 | currentSet.AddLast(set); 44 | } 45 | 46 | return; 47 | } 48 | 49 | 50 | LinkedListNode current = currentSet.First; 51 | 52 | while(current != null) 53 | { 54 | for (int i = 0; i < indexSet.Length; i++) 55 | { 56 | int[] resultRes = IndexSetEquals(current.Value, indexSet[i], dims); 57 | if (resultRes != null) 58 | { 59 | currentSet.AddBefore(current, resultRes); 60 | } 61 | } 62 | LinkedListNode toRemove = current; 63 | current = current.Next; 64 | currentSet.Remove(toRemove); 65 | } 66 | } 67 | 68 | private int[] IndexSetEquals(int[] nativeSet, int[] alienSet, string[] dims) 69 | { 70 | // Intersection of two sets will be here: 71 | int[] resultSet = new int[nativeSet.Length]; 72 | for (int j = 0; j < resultSet.Length; j++) 73 | resultSet[j] = -1; 74 | 75 | for (int i = 0; i < alienSet.Length; i++) 76 | { 77 | int j = dimIndexes[dims[i]]; 78 | 79 | if (nativeSet[j] == -1 || nativeSet[j] == alienSet[i]) 80 | resultSet[j] = alienSet[i]; 81 | else 82 | return null; 83 | } 84 | 85 | return resultSet; 86 | } 87 | } 88 | } 89 | 90 | -------------------------------------------------------------------------------- /FetchClimate1/ClimateService.Common/ClimateService.Common.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Debug 5 | AnyCPU 6 | {87558098-AEAB-46A0-8BEF-D838630AC90E} 7 | Library 8 | Properties 9 | Microsoft.Research.Science.Data.Climate.Common 10 | ClimateService.Common 11 | v4.7.1 12 | 512 13 | 14 | 15 | true 16 | full 17 | false 18 | bin\Debug\ 19 | DEBUG;TRACE 20 | prompt 21 | 4 22 | 23 | 24 | pdbonly 25 | true 26 | bin\Release\ 27 | TRACE 28 | prompt 29 | 4 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | {24d8613c-e1e9-4d7b-abaa-051eed4e5dbc} 54 | ScientificDataSet 55 | 56 | 57 | 58 | 65 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/Exceptions/Exceptions.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | 4 | namespace Microsoft.Research.Science.Data 5 | { 6 | /// 7 | /// Represents an error that occurs when an action cannot be performed. 8 | /// 9 | public class CannotPerformActionException : DataSetException 10 | { 11 | /// 12 | /// 13 | /// 14 | public CannotPerformActionException() 15 | : base("Value not found") 16 | { 17 | } 18 | /// 19 | /// 20 | /// 21 | /// 22 | public CannotPerformActionException(string message) 23 | : base(message) 24 | { 25 | } 26 | /// 27 | /// 28 | /// 29 | /// 30 | /// 31 | public CannotPerformActionException(string message, Exception innerException) 32 | : base(message, innerException) 33 | { 34 | 35 | } 36 | } 37 | 38 | /// 39 | /// Represents an error that occurs when there is no corresponded value or it cannot be calculated. 40 | /// 41 | public class ValueNotFoundException : ApplicationException 42 | { 43 | /// 44 | /// 45 | /// 46 | public ValueNotFoundException() 47 | : base("Value not found") 48 | { 49 | } 50 | /// 51 | /// 52 | /// 53 | /// 54 | public ValueNotFoundException(string message) 55 | : base(message) 56 | { 57 | } 58 | /// 59 | /// 60 | /// 61 | /// 62 | /// 63 | public ValueNotFoundException(string message, Exception innerException) 64 | : base(message, innerException) 65 | { 66 | 67 | } 68 | } 69 | 70 | /// 71 | /// Represents an error when someone tries to modify a read only instance. 72 | /// 73 | [Serializable] 74 | public class ReadOnlyException : DataSetException 75 | { 76 | /// 77 | /// 78 | /// 79 | public ReadOnlyException() { } 80 | /// 81 | /// 82 | /// 83 | /// 84 | public ReadOnlyException(string message) : base(message) { } 85 | /// 86 | /// 87 | /// 88 | /// 89 | /// 90 | public ReadOnlyException(string message, Exception inner) : base(message, inner) { } 91 | /// 92 | /// 93 | /// 94 | /// 95 | /// 96 | protected ReadOnlyException( 97 | System.Runtime.Serialization.SerializationInfo info, 98 | System.Runtime.Serialization.StreamingContext context) 99 | : base(info, context) { } 100 | } 101 | } 102 | 103 | -------------------------------------------------------------------------------- /FetchClimate1/ClimateService.Common/Hash.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Security.Cryptography; 6 | using System.IO; 7 | using Microsoft.Research.Science.Data.Climate.Conventions; 8 | 9 | namespace Microsoft.Research.Science.Data.Climate 10 | { 11 | 12 | public class ShaHash 13 | { 14 | static HashAlgorithm sha = new SHA1CryptoServiceProvider(); 15 | 16 | 17 | public static string HashStreamToHexString(Stream stream) 18 | { 19 | lock ("ShaHash") 20 | { 21 | return ByteArrayToHexStr(sha.ComputeHash(stream)); 22 | } 23 | } 24 | 25 | public static byte[] HexStrToByteArray(string str) 26 | { 27 | byte[] res = new byte[str.Length / 2]; 28 | for (int i = 0; i < str.Length / 2; i++) 29 | { 30 | string b = str.Substring(i * 2, 2); 31 | res[i] = Convert.ToByte(b, 16); 32 | } 33 | return res; 34 | } 35 | 36 | public static string ByteArrayToHexStr(byte[] data) 37 | { 38 | string str = string.Empty; 39 | for (int i = 0; i < data.Length; i++) 40 | str += Convert.ToString(data[i], 16); 41 | return str; 42 | } 43 | 44 | public static string CalculateShaHash(DataSet ds) 45 | { 46 | int[] yearsMin = (int[])ds.Variables[Namings.VarNameYearMin].GetData(); 47 | int[] yearsMax = (int[])ds.Variables[Namings.VarNameYearMax].GetData(); 48 | 49 | int[] daysMin = (int[])ds.Variables[Namings.VarNameDayMin].GetData(); 50 | int[] daysMax = (int[])ds.Variables[Namings.VarNameDayMax].GetData(); 51 | 52 | int[] hoursMin = (int[])ds.Variables[Namings.VarNameHourMin].GetData(); 53 | int[] hoursMax = (int[])ds.Variables[Namings.VarNameHourMax].GetData(); 54 | 55 | double[] latsMin = (double[])ds.Variables[Namings.VarNameLatMin].GetData(); 56 | double[] latsMax = (double[])ds.Variables[Namings.VarNameLatMax].GetData(); 57 | 58 | double[] lonsMin = (double[])ds.Variables[Namings.VarNameLonMin].GetData(); 59 | double[] lonsMax = (double[])ds.Variables[Namings.VarNameLonMax].GetData(); 60 | 61 | int cellsCount = ds.Dimensions[Namings.dimNameCells].Length; 62 | 63 | string metadataNameProvenanceHint = (string)ds.Metadata[Namings.metadataNameProvenanceHint]; 64 | string metadataNameParameter = (string)ds.Metadata[Namings.metadataNameParameter]; 65 | string metadataNameCoverage = (string)ds.Metadata[Namings.metadataNameCoverage]; 66 | 67 | MemoryStream memStm = new MemoryStream(); 68 | using (BinaryWriter writer = new BinaryWriter(memStm)) 69 | { 70 | writer.Write(metadataNameProvenanceHint); 71 | writer.Write(metadataNameParameter); 72 | writer.Write(metadataNameCoverage); 73 | for (int i = 0; i < cellsCount; i++) 74 | { 75 | writer.Write(yearsMin[i]); 76 | writer.Write(yearsMax[i]); 77 | writer.Write(daysMin[i]); 78 | writer.Write(daysMax[i]); 79 | writer.Write(hoursMin[i]); 80 | writer.Write(hoursMax[i]); 81 | writer.Write(latsMin[i]); 82 | writer.Write(latsMax[i]); 83 | writer.Write(lonsMin[i]); 84 | writer.Write(lonsMax[i]); 85 | } 86 | writer.Flush(); 87 | writer.BaseStream.Seek(0, SeekOrigin.Begin); 88 | return ShaHash.HashStreamToHexString(memStm); 89 | } 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/DataSetLink.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | 7 | namespace Microsoft.Research.Science.Data 8 | { 9 | internal class DataSetLink 10 | { 11 | public bool Committed = false; 12 | private Variable reference; 13 | private Variable target; 14 | 15 | public DataSetLink(Variable reference, Variable target) 16 | { 17 | if (reference == null) 18 | throw new ArgumentNullException("reference"); 19 | if (target == null) 20 | throw new ArgumentNullException("target"); 21 | this.reference = reference; 22 | this.target = target; 23 | } 24 | 25 | public Variable Reference { get { return reference; } } 26 | 27 | public Variable Target { get { return target; } } 28 | 29 | public DataSet SourceDataSet { get { return Reference.DataSet; } } 30 | 31 | public DataSet TargetDataSet { get { return Target.DataSet; } } 32 | 33 | /// 34 | /// Gets the value indicating that this reference has changes. 35 | /// 36 | public bool IsActive { get { return Reference.HasChanges || Target.HasChanges; } } 37 | 38 | public override string ToString() 39 | { 40 | return String.Format("{0} refers {1}", Reference, Target); 41 | } 42 | 43 | public override bool Equals(object obj) 44 | { 45 | DataSetLink l = obj as DataSetLink; 46 | if (object.Equals(l, null)) return false; 47 | 48 | return Reference == l.Reference && Target == l.Target; 49 | } 50 | 51 | public static bool operator==(DataSetLink l1, DataSetLink l2) 52 | { 53 | if (object.Equals(l1, null)) return object.Equals(l2, null); 54 | return l1.Equals(l2); 55 | } 56 | 57 | public static bool operator !=(DataSetLink l1, DataSetLink l2) 58 | { 59 | if (object.Equals(l1, null)) return !object.Equals(l2, null); 60 | return !l1.Equals(l2); 61 | } 62 | 63 | public override int GetHashCode() 64 | { 65 | return Reference.GetHashCode() ^ Target.GetHashCode(); 66 | } 67 | } 68 | 69 | internal class DataSetLinkCollection : IEnumerable 70 | { 71 | private List links = new List(); 72 | 73 | public DataSetLinkCollection() 74 | { 75 | } 76 | 77 | public DataSetLink AddLink(Variable reference, Variable target) 78 | { 79 | foreach (var item in links) 80 | { 81 | if (item.Reference == reference && item.Target == target) return item; 82 | } 83 | var link = new DataSetLink(reference, target); 84 | links.Add(link); 85 | return link; 86 | } 87 | 88 | public void RemoveLinks(DataSet srcDataSet) 89 | { 90 | for (int i = links.Count; --i >= 0; ) 91 | { 92 | if (links[i].SourceDataSet == srcDataSet) 93 | links.RemoveAt(i); 94 | } 95 | } 96 | 97 | public void RemoveLink(DataSetLink link) 98 | { 99 | links.Remove(link); 100 | } 101 | 102 | public int Count { get { return links.Count; } } 103 | 104 | public void CommitLinks() 105 | { 106 | foreach (var item in links) 107 | { 108 | item.Committed = true; 109 | } 110 | } 111 | 112 | public void Rollback() 113 | { 114 | for (int i = links.Count; --i >= 0; ) 115 | { 116 | if (!links[i].Committed) 117 | links.RemoveAt(i); 118 | } 119 | } 120 | 121 | public override string ToString() 122 | { 123 | return links.Count.ToString() + " links"; 124 | } 125 | 126 | #region IEnumerable Members 127 | 128 | public IEnumerator GetEnumerator() 129 | { 130 | return links.GetEnumerator(); 131 | } 132 | 133 | #endregion 134 | 135 | #region IEnumerable Members 136 | 137 | System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() 138 | { 139 | return links.GetEnumerator(); 140 | } 141 | 142 | #endregion 143 | 144 | } 145 | } 146 | 147 | -------------------------------------------------------------------------------- /FetchClimate1/ClimateServiceClient/RestProcessingClient.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading; 7 | using Microsoft.Research.Science.Data.Climate; 8 | using Microsoft.Research.Science.Data.Climate.Conventions; 9 | using System.IO; 10 | using Microsoft.Research.Science.Data.Climate.Common; 11 | using Microsoft.Research.Science.Data; 12 | 13 | namespace Microsoft.Research.Science.Data.Processing 14 | { 15 | public class RestProcessingClient : ProcessingClientBase 16 | { 17 | private static TimeSpan timeout = TimeSpan.FromHours(40); 18 | 19 | /// 20 | /// Gets or sets request timeout. 21 | /// 22 | public static TimeSpan Timeout 23 | { 24 | get { return RestProcessingClient.timeout; } 25 | set { RestProcessingClient.timeout = value; } 26 | } 27 | 28 | #if DEBUG 29 | static RestProcessingClient() 30 | { 31 | 32 | //Microsoft.Research.Science.Data.Factory.DataSetFactory.Register( 33 | // typeof(Microsoft.Research.Science.Data.CSV.CsvDataSet)); 34 | //Microsoft.Research.Science.Data.Factory.DataSetFactory.Register( 35 | // typeof(Microsoft.Research.Science.Data.Memory.MemoryDataSet)); 36 | } 37 | #endif 38 | 39 | protected override DataSet ServerProcessInternal(DataSet ds) 40 | { 41 | DateTime start = DateTime.Now; 42 | 43 | bool resultGot = false; 44 | 45 | DataSet inputDs = ds; 46 | 47 | DataSet resultDs = null; 48 | while (!resultGot) 49 | { 50 | resultDs = RestApiWrapper.Instance.Process(ds); 51 | if (FetchClimateRequestBuilder.IsResultDataSet(resultDs)) 52 | { 53 | resultGot = true; 54 | } 55 | else 56 | { 57 | if (FetchClimateRequestBuilder.ResendRequest(resultDs)) 58 | ds = inputDs; 59 | else 60 | ds = resultDs; 61 | 62 | int expectedCalculationTime = 0; 63 | string hash = string.Empty; 64 | FetchClimateRequestBuilder.GetStatusCheckParams(resultDs, out expectedCalculationTime, out hash); 65 | 66 | Thread.Sleep(expectedCalculationTime); 67 | } 68 | 69 | if ((!resultGot) && (DateTime.Now - start) > timeout) 70 | { 71 | throw new TimeoutException("Request to fetch climate has timed out. Increase timeout value or try again later."); 72 | } 73 | } 74 | 75 | return resultDs; 76 | } 77 | 78 | //#if !RELEASE_ASSEMBLY 79 | // protected override DataSet LocalProcess(DataSet ds) 80 | // { 81 | // DataSet resultDs = null; 82 | // try 83 | // { 84 | // resultDs = DataSet.Open("msds:memory2"); 85 | // resultDs.IsAutocommitEnabled = false; 86 | // FetchClimateRequestBuilder.CopyRequestedDataSet(ds, resultDs, false); 87 | // if (resultDs.HasChanges) resultDs.Commit(); 88 | // Microsoft.Research.Science.Data.Climate.Processing.ClimateRequestProcessor.Process(resultDs, 0); 89 | 90 | // if (FetchClimateRequestBuilder.IsProcessingSuccessful(resultDs)) 91 | // ;//cache.Add(ds,ComputeHash(request)); 92 | // else if (!FetchClimateRequestBuilder.IsProcessingFailed(resultDs)) 93 | // throw new Exception("Processor hasn't finished the work."); 94 | 95 | // resultDs.IsAutocommitEnabled = true; 96 | // return resultDs; 97 | // } 98 | // catch 99 | // { 100 | // if (resultDs != null && !resultDs.IsDisposed) resultDs.Dispose(); 101 | // throw; 102 | // } 103 | // } 104 | //#endif 105 | } 106 | } -------------------------------------------------------------------------------- /ScientificDataSet/Providers/CSV/CsvVariables2d.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | 5 | namespace Microsoft.Research.Science.Data.CSV 6 | { 7 | /// Class that represents 2D variable in CSV files 8 | /// Type of variable. For list of supported types see DataSet specification. 9 | /// Instances of this call cannot be constructed directly. See 10 | /// for details about creation 11 | /// of new variables 12 | internal sealed class CsvVariable2d : CsvVariable, ICsvVariableMd 13 | { 14 | internal CsvVariable2d(CsvDataSet dataSet, CsvColumn column) 15 | : base(dataSet, column) 16 | { 17 | if (column.Rank != 2) 18 | throw new Exception("This is 2d-variable and is being created with different rank."); 19 | 20 | data = new ArrayWrapper(2, typeof(DataType)); 21 | 22 | Initialize(); 23 | } 24 | 25 | private CsvVariable2d(CsvDataSet dataSet, int id, MetadataDictionary metadata, ArrayWrapper data, string[] dims) 26 | : base(dataSet, id, metadata, dims) 27 | { 28 | this.data = data; 29 | Initialize(); 30 | } 31 | 32 | public override Variable CloneAndRenameDims(string[] newDims) 33 | { 34 | if (newDims == null || Rank != newDims.Length) 35 | throw new Exception("New dimensions are wrong"); 36 | Variable var = new CsvVariable2d((CsvDataSet)DataSet, ID, Metadata, data, newDims); 37 | return var; 38 | } 39 | 40 | protected override Array GetInnerData() 41 | { 42 | DataType[,] array = (DataType[,])data.Data; 43 | if (array == null) 44 | return new DataType[0] { }; 45 | 46 | int W = array.GetLength(0); 47 | int H = array.GetLength(1); 48 | 49 | int i = 0; 50 | DataType[] innerData = new DataType[W * H]; 51 | for (int col = 0; col < W; col++) 52 | for (int row = 0; row < H; row++) 53 | { 54 | innerData[i++] = array[col, row]; 55 | } 56 | 57 | return innerData; 58 | } 59 | 60 | protected override void InnerInitialize(Array data, int[] shape) 61 | { 62 | int i = 0; 63 | DataType[] typedData = (DataType[])data; 64 | DataType[,] array = new DataType[shape[0], shape[1]]; 65 | for (int col = 0; col < shape[0]; col++) 66 | for (int row = 0; row < shape[1]; row++) 67 | { 68 | array[col, row] = typedData[i++]; 69 | } 70 | base.data.PutData(null, array); 71 | ChangesUpdateShape(this.changes, ReadShape()); 72 | } 73 | 74 | Array ICsvVariableMd.GetColumnData(int col) 75 | { 76 | DataType[,] array = (DataType[,])data.Data; 77 | if (array == null) 78 | return new DataType[0] { }; 79 | 80 | int W = array.GetLength(0); 81 | 82 | DataType[] colData = new DataType[W]; 83 | for (int i = 0; i < W; i++) 84 | { 85 | colData[i] = array[i, col]; 86 | } 87 | return colData; 88 | } 89 | 90 | void ICsvVariableMd.Initialize(Array data) 91 | { 92 | base.data.PutData(null, data); 93 | ChangesUpdateShape(this.changes, ReadShape()); 94 | } 95 | 96 | void ICsvVariableMd.FastCopyColumn(Array entireArray, Array column, int index) 97 | { 98 | DataType[] typedData = (DataType[])column; 99 | DataType[,] array = (DataType[,])entireArray; 100 | int n = column.Length; 101 | for (int i = 0; i < n; i++) 102 | { 103 | array[i, index] = typedData[i]; 104 | } 105 | } 106 | } 107 | } 108 | 109 | -------------------------------------------------------------------------------- /FetchClimate1/ClimateService.Common/RestApiUtilities.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using Microsoft.Research.Science.Data; 6 | using System.IO; 7 | using System.Globalization; 8 | 9 | namespace Microsoft.Research.Science.Data.Climate 10 | { 11 | /// 12 | /// Provides static methods for REST API implementation. 13 | /// 14 | public static class RestApiUtilities 15 | { 16 | private static readonly string tempFolder = System.IO.Path.Combine(Path.GetTempPath(), "FetchClimateTemp\\"); 17 | 18 | static RestApiUtilities() 19 | { 20 | if (!Directory.Exists(RestApiUtilities.tempFolder)) 21 | Directory.CreateDirectory(RestApiUtilities.tempFolder); 22 | } 23 | 24 | /// 25 | /// Returns new name for temporary csv file. 26 | /// 27 | /// New name for temporary csv file. 28 | public static string GetTempCsvFileName() 29 | { 30 | return Path.Combine(RestApiUtilities.tempFolder, Path.GetRandomFileName().Replace(".", "2") + ".csv"); 31 | } 32 | 33 | /// 34 | /// Returns bytes of specified instance after converting it into . 35 | /// 36 | /// Input instance. 37 | /// Resulting bytes array. 38 | public static byte[] GetCsvBytes(DataSet ds) 39 | { 40 | if (ds == null) 41 | { 42 | throw new ArgumentNullException("ds"); 43 | } 44 | string resultTempFileName = GetTempCsvFileName(); 45 | byte[] bytes = null; 46 | 47 | try 48 | { 49 | ds.Clone(String.Format(CultureInfo.InvariantCulture, "msds:csv?file={0}&openMode=create&appendMetadata=true", resultTempFileName)); 50 | bytes = File.ReadAllBytes(resultTempFileName); 51 | } 52 | finally 53 | { 54 | if (resultTempFileName != null) 55 | File.Delete(resultTempFileName); 56 | } 57 | 58 | return bytes; 59 | } 60 | 61 | /// 62 | /// Returns new instance from bytes of instance. 63 | /// 64 | /// Input bytes array. 65 | /// Resulting instance. 66 | public static DataSet GetDataSet(byte[] bytes) 67 | { 68 | string tempFile = GetTempCsvFileName(); 69 | 70 | try 71 | { 72 | File.WriteAllBytes(tempFile, bytes); 73 | 74 | using (DataSet csvDs = DataSet.Open(String.Format(CultureInfo.InvariantCulture, "msds:csv?file={0}&openMode=open&appendMetadata=true", tempFile))) 75 | { 76 | DataSet memoryDs = csvDs.Clone("msds:memory"); 77 | File.Delete(tempFile); 78 | tempFile = null; 79 | return memoryDs; 80 | } 81 | } 82 | finally 83 | { 84 | if (tempFile != null) 85 | { 86 | File.Delete(tempFile); 87 | tempFile = null; 88 | } 89 | } 90 | } 91 | 92 | /// 93 | /// Reads specified number of bytes from stream. 94 | /// 95 | /// , to read from. 96 | /// Number of bytes to read. 97 | /// Readen bytes array. 98 | public static byte[] ReadBytes(Stream stream, int length) 99 | { 100 | byte[] buffer = new byte[length]; 101 | int totalBytesRead = 0; 102 | while (totalBytesRead < length) 103 | { 104 | int bytesRead = stream.Read(buffer, totalBytesRead, length - totalBytesRead); 105 | totalBytesRead += bytesRead; 106 | } 107 | 108 | return buffer; 109 | } 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/Exceptions/DataSetCreatingException.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | 7 | namespace Microsoft.Research.Science.Data 8 | { 9 | /// 10 | /// The exception that is thrown when the fails to create a DataSet. 11 | /// 12 | [Serializable] 13 | public class DataSetCreateException : DataSetException 14 | { 15 | private string uri = ""; 16 | 17 | private static string FormatMessage(string uri, string outerMessage) 18 | { 19 | if (String.IsNullOrEmpty(uri)) 20 | if (String.IsNullOrEmpty(outerMessage)) 21 | return "Failed to create DataSet instance"; 22 | else 23 | return String.Format("Failed to create DataSet instance: {0}", outerMessage); 24 | if (String.IsNullOrEmpty(outerMessage)) 25 | return String.Format("Failed to create DataSet instance from uri {0}", uri); 26 | else 27 | return String.Format("Failed to create DataSet instance from uri {0}: {1}", uri, outerMessage); 28 | } 29 | 30 | /// 31 | /// 32 | /// 33 | /// Uri that caused the exception. 34 | public DataSetCreateException(string uri) : base(FormatMessage(uri, null)) { this.uri = uri; } 35 | /// 36 | /// 37 | /// 38 | /// Uri that caused the exception. 39 | /// 40 | public DataSetCreateException(string uri, string message) : base(FormatMessage(uri, message)) { this.uri = uri; } 41 | /// 42 | /// 43 | /// 44 | /// Uri that caused the exception. 45 | /// 46 | /// 47 | public DataSetCreateException(string uri, string message, Exception inner) : base(FormatMessage(uri, message), inner) { this.uri = uri; } 48 | /// 49 | /// 50 | /// 51 | /// 52 | /// 53 | protected DataSetCreateException( 54 | System.Runtime.Serialization.SerializationInfo info, 55 | System.Runtime.Serialization.StreamingContext context) 56 | : base(info, context) { } 57 | 58 | /// 59 | /// Gets the costruction URI that caused the exception. 60 | /// 61 | public string FailedUri 62 | { 63 | get { return uri; } 64 | } 65 | } 66 | 67 | /// 68 | /// The exception that is thrown when the fails to create a DataSet 69 | /// because it is not registered. 70 | /// 71 | [Serializable] 72 | public class ProviderNotRegisteredException : DataSetCreateException 73 | { 74 | /// 75 | /// 76 | /// 77 | /// Uri that caused the exception. 78 | public ProviderNotRegisteredException(string uri) : base(uri) { } 79 | /// 80 | /// 81 | /// 82 | /// Uri that caused the exception. 83 | /// 84 | public ProviderNotRegisteredException(string uri, string message) : base(uri, message) { } 85 | /// 86 | /// 87 | /// 88 | /// Uri that caused the exception. 89 | /// 90 | /// 91 | public ProviderNotRegisteredException(string uri, string message, Exception inner) : base(uri, message, inner) { } 92 | /// 93 | /// 94 | /// 95 | /// 96 | /// 97 | protected ProviderNotRegisteredException( 98 | System.Runtime.Serialization.SerializationInfo info, 99 | System.Runtime.Serialization.StreamingContext context) 100 | : base(info, context) { } 101 | } 102 | } 103 | 104 | -------------------------------------------------------------------------------- /SDSlite.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 17 4 | VisualStudioVersion = 17.5.33627.172 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "FetchClimate1", "FetchClimate1", "{540EC0ED-4A07-4AC9-A68E-E06BDD4FF89A}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ScientificDataSet", "ScientificDataSet\ScientificDataSet.csproj", "{24D8613C-E1E9-4D7B-ABAA-051EED4E5DBC}" 9 | EndProject 10 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "sdsutil", "sdsutil\sdsutil.csproj", "{0EAC3EBD-DC4D-4854-8C70-E096C49F8D47}" 11 | EndProject 12 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SDSLiteTests", "SDSLiteTests\SDSLiteTests.csproj", "{C6919B51-ED1A-455E-89E0-F295ECFC722B}" 13 | EndProject 14 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ClimateService.Common", "FetchClimate1\ClimateService.Common\ClimateService.Common.csproj", "{87558098-AEAB-46A0-8BEF-D838630AC90E}" 15 | EndProject 16 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ClimateService.Client", "FetchClimate1\ClimateServiceClient\ClimateService.Client.csproj", "{B02F58BB-072E-4DC7-AAAB-674C14E01E9D}" 17 | EndProject 18 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{1FC31A69-438C-47C0-A833-73226ADBAA5E}" 19 | ProjectSection(SolutionItems) = preProject 20 | .github\workflows\build-test.yml = .github\workflows\build-test.yml 21 | CHANGELOG.md = CHANGELOG.md 22 | Licence.txt = Licence.txt 23 | Common\Version.proj = Common\Version.proj 24 | EndProjectSection 25 | EndProject 26 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ScientificDataSet.DataSetReplicator", "ScientificDataSet.DataSetReplicator\ScientificDataSet.DataSetReplicator.csproj", "{CC740DCD-9607-45C6-812C-DE797DF4A50B}" 27 | EndProject 28 | Global 29 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 30 | Debug|Any CPU = Debug|Any CPU 31 | Release|Any CPU = Release|Any CPU 32 | EndGlobalSection 33 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 34 | {24D8613C-E1E9-4D7B-ABAA-051EED4E5DBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 35 | {24D8613C-E1E9-4D7B-ABAA-051EED4E5DBC}.Debug|Any CPU.Build.0 = Debug|Any CPU 36 | {24D8613C-E1E9-4D7B-ABAA-051EED4E5DBC}.Release|Any CPU.ActiveCfg = Release|Any CPU 37 | {24D8613C-E1E9-4D7B-ABAA-051EED4E5DBC}.Release|Any CPU.Build.0 = Release|Any CPU 38 | {0EAC3EBD-DC4D-4854-8C70-E096C49F8D47}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 39 | {0EAC3EBD-DC4D-4854-8C70-E096C49F8D47}.Debug|Any CPU.Build.0 = Debug|Any CPU 40 | {0EAC3EBD-DC4D-4854-8C70-E096C49F8D47}.Release|Any CPU.ActiveCfg = Release|Any CPU 41 | {0EAC3EBD-DC4D-4854-8C70-E096C49F8D47}.Release|Any CPU.Build.0 = Release|Any CPU 42 | {C6919B51-ED1A-455E-89E0-F295ECFC722B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 43 | {C6919B51-ED1A-455E-89E0-F295ECFC722B}.Debug|Any CPU.Build.0 = Debug|Any CPU 44 | {C6919B51-ED1A-455E-89E0-F295ECFC722B}.Release|Any CPU.ActiveCfg = Release|Any CPU 45 | {C6919B51-ED1A-455E-89E0-F295ECFC722B}.Release|Any CPU.Build.0 = Release|Any CPU 46 | {87558098-AEAB-46A0-8BEF-D838630AC90E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 47 | {87558098-AEAB-46A0-8BEF-D838630AC90E}.Debug|Any CPU.Build.0 = Debug|Any CPU 48 | {87558098-AEAB-46A0-8BEF-D838630AC90E}.Release|Any CPU.ActiveCfg = Release|Any CPU 49 | {87558098-AEAB-46A0-8BEF-D838630AC90E}.Release|Any CPU.Build.0 = Release|Any CPU 50 | {B02F58BB-072E-4DC7-AAAB-674C14E01E9D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 51 | {B02F58BB-072E-4DC7-AAAB-674C14E01E9D}.Debug|Any CPU.Build.0 = Debug|Any CPU 52 | {B02F58BB-072E-4DC7-AAAB-674C14E01E9D}.Release|Any CPU.ActiveCfg = Release|Any CPU 53 | {B02F58BB-072E-4DC7-AAAB-674C14E01E9D}.Release|Any CPU.Build.0 = Release|Any CPU 54 | {CC740DCD-9607-45C6-812C-DE797DF4A50B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 55 | {CC740DCD-9607-45C6-812C-DE797DF4A50B}.Debug|Any CPU.Build.0 = Debug|Any CPU 56 | {CC740DCD-9607-45C6-812C-DE797DF4A50B}.Release|Any CPU.ActiveCfg = Release|Any CPU 57 | {CC740DCD-9607-45C6-812C-DE797DF4A50B}.Release|Any CPU.Build.0 = Release|Any CPU 58 | EndGlobalSection 59 | GlobalSection(SolutionProperties) = preSolution 60 | HideSolutionNode = FALSE 61 | EndGlobalSection 62 | GlobalSection(NestedProjects) = preSolution 63 | {87558098-AEAB-46A0-8BEF-D838630AC90E} = {540EC0ED-4A07-4AC9-A68E-E06BDD4FF89A} 64 | {B02F58BB-072E-4DC7-AAAB-674C14E01E9D} = {540EC0ED-4A07-4AC9-A68E-E06BDD4FF89A} 65 | EndGlobalSection 66 | GlobalSection(ExtensibilityGlobals) = postSolution 67 | SolutionGuid = {D2DD130A-C9B4-444E-B1BB-5A37CA4AC782} 68 | EndGlobalSection 69 | EndGlobal 70 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![NuGet](https://img.shields.io/nuget/v/SDSlite.svg?style=flat)](https://www.nuget.org/packages/SDSlite/) 2 | ![build-test](https://github.com/predictionmachines/SDSlite/workflows/build-test/badge.svg) 3 | 4 | Scientific DataSet Lite 5 | ======================= 6 | 7 | This is a cross platform [.NET](https://dotnet.microsoft.com) library for manipulating netCDF, CSV and TSV files. 8 | This is a subset of **Scientific DataSet** [https://www.microsoft.com/en-us/research/project/scientific-dataset/](https://www.microsoft.com/en-us/research/project/scientific-dataset/). 9 | 10 | External Libraries 11 | ------------------ 12 | 13 | SDSLite requires a platform dependent library available from [Unidata](https://www.unidata.ucar.edu/software/netcdf/). 14 | 15 | ### Windows 16 | 17 | For Windows go to https://docs.unidata.ucar.edu/netcdf-c/current/winbin.html and download the version of netCDF4 (without DAP) corresponding to your machine, either 32 or 64 bit. 18 | As of May 2023 these are: https://downloads.unidata.ucar.edu/netcdf-c/4.9.2/netCDF4.9.2-NC4-64.exe or https://downloads.unidata.ucar.edu/netcdf-c/4.9.2/netCDF4.9.2-NC4-64.exe. 19 | 20 | The Scientific DataSet library looks for `netcdf.dll` file in the following locations: 21 | - `LIBNETCDFPATH` environment variable if it contains full path of the `netcdf.dll` file; 22 | - Current directory; 23 | - In the same directory as the `ScientificDataSet.dll` assembly; 24 | - PATH environment variable; 25 | - Default installation directory of netCDF4. 26 | 27 | ### Linux 28 | 29 | For Linux install pre-built netCDF-C libraries. For example on Ubuntu: 30 | 31 | `sudo apt-get install libnetcdf-dev` 32 | 33 | 34 | ### MacOS 35 | 36 | Use homebrew [http://brew.sh/](http://brew.sh/) to install netcdf: 37 | 38 | `brew install netcdf` 39 | 40 | Sample 41 | ------ 42 | 43 | C# example: 44 | 45 | ```csharp 46 | using Microsoft.Research.Science.Data; 47 | using Microsoft.Research.Science.Data.Imperative; 48 | 49 | // download a netCDF file from unidata web site 50 | var client = new System.Net.Http.HttpClient(); 51 | var response = await client.GetAsync("https://www.unidata.ucar.edu/software/netcdf/examples/sresa1b_ncar_ccsm3-example.nc"); 52 | using (var stream = System.IO.File.OpenWrite("sresa1b_ncar_ccsm3-example.nc")){ 53 | await response.Content.CopyToAsync(stream); 54 | } 55 | 56 | // open the file and print some info 57 | using (DataSet ds = DataSet.Open("msds:nc?file=sresa1b_ncar_ccsm3-example.nc&openMode=readOnly")){ 58 | Console.WriteLine(ds); 59 | Console.WriteLine(ds.Metadata["comment"]); 60 | var lat = ds.GetData("lat"); 61 | Console.WriteLine($"latitude: len={lat.Length}, min={lat.Min()}, max={lat.Max()}"); 62 | } 63 | ``` 64 | 65 | Output: 66 | 67 | ```text 68 | msds:nc?openMode=readOnly&file=c:\Users\***\sresa1b_ncar_ccsm3-example.nc 69 | [1] 70 | DSID: ca6c1f06-f743-4190-9b94-20cf0cbc18f7 71 | [12] ua of type Single (time:1) (plev:17) (lat:128) (lon:256) 72 | [11] time_bnds of type Double (time:1) (bnds:2) 73 | [10] time of type Double (time:1) 74 | [9] tas of type Single (time:1) (lat:128) (lon:256) 75 | [8] pr of type Single (time:1) (lat:128) (lon:256) 76 | [7] plev of type Double (plev:17) 77 | [6] msk_rgn of type Int32 (lat:128) (lon:256) 78 | [5] lon_bnds of type Double (lon:256) (bnds:2) 79 | [4] lon of type Single (lon:256) 80 | [3] lat_bnds of type Double (lat:128) (bnds:2) 81 | [2] lat of type Single (lat:128) 82 | [1] area of type Single (lat:128) (lon:256) 83 | 84 | This simulation was initiated from year 2000 of 85 | CCSM3 model run b30.030a and executed on 86 | hardware cheetah.ccs.ornl.gov. The input external forcings are 87 | ozone forcing : A1B.ozone.128x64_L18_1991-2100_c040528.nc 88 | aerosol optics : AerosolOptics_c040105.nc 89 | aerosol MMR : AerosolMass_V_128x256_clim_c031022.nc 90 | carbon scaling : carbonscaling_A1B_1990-2100_c040609.nc 91 | solar forcing : Fixed at 1366.5 W m-2 92 | GHGs : ghg_ipcc_A1B_1870-2100_c040521.nc 93 | GHG loss rates : noaamisc.r8.nc 94 | volcanic forcing : none 95 | DMS emissions : DMS_emissions_128x256_clim_c040122.nc 96 | oxidants : oxid_128x256_L26_clim_c040112.nc 97 | SOx emissions : SOx_emissions_A1B_128x256_L2_1990-2100_c040608.nc 98 | Physical constants used for derived data: 99 | Lv (latent heat of evaporation): 2.501e6 J kg-1 100 | Lf (latent heat of fusion ): 3.337e5 J kg-1 101 | r[h2o] (density of water ): 1000 kg m-3 102 | g2kg (grams to kilograms ): 1000 g kg-1 103 | 104 | Integrations were performed by NCAR and CRIEPI with support 105 | and facilities provided by NSF, DOE, MEXT and ESC/JAMSTEC. 106 | latitude: len=128, min=-88.927734, max=88.927734 107 | ``` 108 | LICENCE 109 | ------- 110 | 111 | This project is licensed under MIT. -------------------------------------------------------------------------------- /ScientificDataSet/Core/Factory/Attributes.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | 7 | namespace Microsoft.Research.Science.Data 8 | { 9 | /// 10 | /// Specifies the name of the DataSet provider. 11 | /// 12 | /// 13 | /// 14 | /// The name of a provider is referred in the DataSet URI 15 | /// (see remarks for the class). 16 | /// 17 | /// See remarks for the class. 18 | /// 19 | /// 20 | /// 21 | /// 22 | /// 23 | /// 24 | [AttributeUsage(AttributeTargets.Class, Inherited = false, AllowMultiple = false)] 25 | public sealed class DataSetProviderNameAttribute : Attribute 26 | { 27 | private string name; 28 | 29 | /// 30 | /// Initializes the attribute. 31 | /// 32 | /// Name of the provider. 33 | public DataSetProviderNameAttribute(string name) 34 | { 35 | this.name = name; 36 | } 37 | 38 | /// 39 | /// Gets the name of the provider. 40 | /// 41 | public string Name { get { return name; } } 42 | } 43 | 44 | /// 45 | /// Specifies the types of files (extensions) acceptable by the DataSet provider. 46 | /// 47 | /// 48 | /// See remarks for the class. 49 | /// 50 | /// 51 | /// 52 | /// 53 | [AttributeUsage(AttributeTargets.Class, Inherited = true, AllowMultiple = true)] 54 | public sealed class DataSetProviderFileExtensionAttribute : Attribute 55 | { 56 | private string ext; 57 | 58 | /// 59 | /// Initializes the attribute. 60 | /// 61 | /// File extension acceptable by the provider (including ".", e.g. ".dat"). 62 | public DataSetProviderFileExtensionAttribute(string extension) 63 | { 64 | this.ext = extension; 65 | } 66 | 67 | /// 68 | /// Gets the file extension acceptable by the provider (including ".", e.g. ".dat"). 69 | /// 70 | public string FileExtension { get { return ext; } } 71 | } 72 | 73 | /// 74 | /// Associates a class derived from the with 75 | /// a provider type. 76 | /// 77 | /// 78 | /// 79 | /// 80 | /// 81 | /// 82 | /// 83 | [AttributeUsage(AttributeTargets.Class, Inherited = false, AllowMultiple = false)] 84 | public sealed class DataSetProviderUriTypeAttribute : Attribute 85 | { 86 | private Type type; 87 | 88 | /// 89 | /// Initializes the attribute. 90 | /// 91 | /// Type of the provider. 92 | /// 93 | /// Type must be type or 94 | /// be derived from the class. 95 | /// 96 | public DataSetProviderUriTypeAttribute(Type uriType) 97 | { 98 | if (uriType == null) throw new ArgumentNullException("uriType"); 99 | if (uriType != typeof(DataSetUri) && !uriType.IsSubclassOf(typeof(DataSetUri))) 100 | throw new ArgumentException("uriType must be DataSetUri or derived from it"); 101 | 102 | this.type = uriType; 103 | } 104 | 105 | /// 106 | /// Gets the type of the provider. 107 | /// 108 | public Type UriType { get { return type; } } 109 | } 110 | 111 | /// 112 | /// Indicates that the target property contains file name. 113 | /// 114 | [AttributeUsage(AttributeTargets.Property, AllowMultiple=false)] 115 | public sealed class FileNamePropertyAttribute : Attribute { } 116 | 117 | /// 118 | /// Indicates that the target property contains directory name. 119 | /// 120 | [AttributeUsage(AttributeTargets.Property, AllowMultiple = false)] 121 | public sealed class DirectoryPropertyAttribute : Attribute { } 122 | 123 | /// 124 | /// Indicates that the target property contains URI 125 | /// 126 | [AttributeUsage(AttributeTargets.Property, AllowMultiple=false)] 127 | public sealed class UriPropertyAttribute : Attribute { } 128 | } 129 | 130 | -------------------------------------------------------------------------------- /FetchClimate1/ClimateService.Common/PrioritySemaphore.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading; 6 | 7 | namespace Microsoft.Research.Science.Data.Climate.Service 8 | { 9 | /// 10 | /// Semaphore with prioritization. Provides thread safe functionality for resource managing. 11 | /// 12 | public class PrioritySemaphore : IDisposable 13 | { 14 | private volatile int currentTicket; 15 | private volatile int processingCount; 16 | private Dictionary> ticketsDictionary; 17 | private object monitor; 18 | 19 | private bool disposed = false; 20 | 21 | Dictionary ticketsLocks; 22 | 23 | private int resourcesCount; 24 | 25 | /// 26 | /// Gets total amount of resources. 27 | /// 28 | public int ResourcesCount 29 | { 30 | get { return this.resourcesCount; } 31 | } 32 | 33 | /// 34 | /// Gets amount of processing resources. 35 | /// 36 | public int ProcessingCount 37 | { 38 | get { return this.processingCount; } 39 | } 40 | 41 | /// 42 | /// Creates new instance of . 43 | /// 44 | /// Amount of available resources. 45 | public PrioritySemaphore(int resourcesCount) 46 | { 47 | this.ticketsDictionary = new Dictionary>(); 48 | this.ticketsLocks = new Dictionary(); 49 | this.monitor = new object(); 50 | 51 | this.resourcesCount = resourcesCount; 52 | this.processingCount = 0; 53 | this.currentTicket = 0; 54 | } 55 | 56 | /// 57 | /// Waits for available resource. 58 | /// 59 | /// 60 | /// Priority of request. 61 | /// The higher is priority, the sooner resource will be achieved. 62 | /// 63 | public void WaitOne(int priority) 64 | { 65 | AutoResetEvent ticketLock = null; 66 | 67 | lock (monitor) 68 | { 69 | if (processingCount < resourcesCount) 70 | { 71 | processingCount++; 72 | return; 73 | } 74 | else 75 | { 76 | int ticket = currentTicket++; 77 | 78 | if (!this.ticketsDictionary.ContainsKey(priority)) 79 | this.ticketsDictionary[priority] = new Queue(); 80 | this.ticketsDictionary[priority].Enqueue(ticket); 81 | 82 | ticketLock = new AutoResetEvent(false); 83 | this.ticketsLocks[ticket] = ticketLock; 84 | } 85 | } 86 | 87 | if (ticketLock != null) 88 | { 89 | ticketLock.WaitOne(); 90 | ticketLock.Dispose(); 91 | } 92 | } 93 | 94 | /// 95 | /// Releases resource. 96 | /// 97 | public void Set() 98 | { 99 | int ticket = -1; 100 | 101 | lock (monitor) 102 | { 103 | foreach (var queue in ticketsDictionary.OrderByDescending(x => x.Key).Select(x => x.Value)) 104 | { 105 | if (queue.Count != 0) 106 | { 107 | ticket = queue.Dequeue(); 108 | break; 109 | } 110 | } 111 | if (ticket < 0) 112 | { 113 | processingCount--; 114 | } 115 | } 116 | 117 | if (ticket >= 0) 118 | { 119 | var ticketLock = ticketsLocks[ticket]; 120 | ticketLock.Set(); 121 | ticketsLocks.Remove(ticket); 122 | } 123 | } 124 | 125 | /// 126 | /// Disposes this instance. 127 | /// 128 | public void Dispose() 129 | { 130 | Dispose(true); 131 | GC.SuppressFinalize(this); 132 | } 133 | 134 | protected virtual void Dispose(bool disposing) 135 | { 136 | // Check to see if Dispose has already been called. 137 | if(!this.disposed) 138 | { 139 | // If disposing equals true, dispose all managed 140 | // and unmanaged resources. 141 | if(disposing) 142 | { 143 | foreach (var tickLock in this.ticketsLocks.Values) 144 | { 145 | tickLock.Dispose(); 146 | } 147 | } 148 | // Note disposing has been done. 149 | disposed = true; 150 | } 151 | } 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /ScientificDataSet/Providers/NetCDF/NetCDFUri.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.ComponentModel; 7 | 8 | namespace Microsoft.Research.Science.Data.NetCDF4 9 | { 10 | /// 11 | /// Allows to customize a URI with parameters specific for the provider. 12 | /// 13 | /// 14 | public class NetCDFUri : DataSetUri 15 | { 16 | /// 17 | /// Instantiates an instance of the class. 18 | /// 19 | /// The DataSet uri. 20 | public NetCDFUri(string uri) 21 | : base(uri, typeof(NetCDFDataSet)) 22 | { 23 | if (GetParameterOccurences("file") > 1) 24 | throw new ArgumentException("The given uri must define single file path"); 25 | } 26 | /// 27 | /// Instantiates an instance of the class with default parameters. 28 | /// 29 | public NetCDFUri() 30 | : base(typeof(NetCDFDataSet)) 31 | { 32 | } 33 | 34 | /// 35 | /// Indicates whether the provider should trim trailing zero (\x00 symbol) when reads attribute values or not. 36 | /// Default is false. 37 | /// 38 | [Description("Indicates whether the provider must trim trailing zero (\\x00 symbol) when reads attribute values or not.\nDefault is false.")] 39 | public bool TrimTrailingZero 40 | { 41 | get 42 | { 43 | if (GetParameterOccurences("trimZero") == 0) 44 | return false; 45 | string trim = this["trimZero"]; 46 | 47 | return bool.Parse(trim); 48 | } 49 | set 50 | { 51 | this["trimZero"] = value.ToString().ToLower(); 52 | } 53 | } 54 | 55 | /// 56 | /// Specifies how the data set should open a file. 57 | /// 58 | [Description("Specifies how the data set should open a file.")] 59 | public new ResourceOpenMode OpenMode 60 | { 61 | get { return base.OpenMode; } 62 | set { base.OpenMode = value; } 63 | } 64 | 65 | /// 66 | /// Defines the compression level. 67 | /// 68 | [Description("Defines the data compression level. Affects only DataSet modification.")] 69 | public DeflateLevel Deflate 70 | { 71 | get 72 | { 73 | if (GetParameterOccurences("deflate") == 0) 74 | return DeflateLevel.Normal; 75 | string defl = this["deflate"]; 76 | 77 | return (DeflateLevel)Enum.Parse(typeof(DeflateLevel), defl, true); 78 | } 79 | set 80 | { 81 | this["deflate"] = value.ToString().ToLower(); 82 | } 83 | } 84 | 85 | /// 86 | /// Specifies the file to open or create. 87 | /// 88 | [FileNameProperty] 89 | [Description("Specifies the file to open or create.")] 90 | public string FileName 91 | { 92 | get 93 | { 94 | if (GetParameterOccurences("file") == 0) 95 | return ""; 96 | return this["file"]; 97 | } 98 | set { this["file"] = value; } 99 | } 100 | 101 | /// 102 | /// Indicates whether the rollback is enabled or not. 103 | /// 104 | /// 105 | /// Indicates whether the rollback is enabled or not. Rollback can happen if commit fails. If it is enabled, each transaction copies source file for further possible restoration. Otherwise, rollback will throw an exception in case of its invoke. Set it for data sets having not only NetCDF variables and consider set it false for very large files. Default is false. 106 | /// 107 | [Description("Indicates whether the rollback is enabled or not. Rollback can happen if commit fails.\nIf it is enabled, each transaction copies source file for further possible restoration. Otherwise, rollback will throw an exception in case of its invoke.\nSet it for data sets having not only NetCDF variables and consider set it false for very large files.\nDefault is false.")] 108 | public bool EnableRollback 109 | { 110 | get 111 | { 112 | string val = GetParameterValue("enableRollback", "false").ToLower(); 113 | return bool.Parse(val); 114 | } 115 | set 116 | { 117 | SetParameterValue("enableRollback", value.ToString().ToLower()); 118 | } 119 | } 120 | } 121 | 122 | #pragma warning disable 1591 123 | /// 124 | /// Represents data deflate level for NetCDF variables. 125 | /// 126 | /// 127 | public enum DeflateLevel 128 | { 129 | Off = -1, 130 | Store = 0, 131 | Fastest = 1, 132 | Fast = 3, 133 | Normal = 5, 134 | Good = 7, 135 | Best = 9, 136 | BestWithShuffle = 10 137 | } 138 | } 139 | 140 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/DataSetChangeset.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Diagnostics; 7 | 8 | namespace Microsoft.Research.Science.Data 9 | { 10 | /// 11 | /// Represents changes in a DataSet. 12 | /// 13 | public class DataSetChangeset 14 | { 15 | private int changeSetId; 16 | 17 | private Variable.Changes[] varAdded; 18 | private Variable.Changes[] varUpdated; 19 | private Variable.Changes[] varAffected; 20 | 21 | private bool initialized = false; 22 | private DataSet.Changes changes; 23 | private DataSet dataSet; 24 | 25 | private ChangesetSource source; 26 | 27 | internal DataSetChangeset(DataSet sds, DataSet.Changes changes) 28 | : this(sds, changes, false) 29 | { 30 | } 31 | 32 | internal DataSetChangeset(DataSet sds, DataSet.Changes changes, bool initializeAtOnce) 33 | { 34 | if (changes == null) 35 | throw new ArgumentNullException("changes"); 36 | lock (sds) 37 | { 38 | this.changes = changes; 39 | this.dataSet = sds; 40 | this.changeSetId = sds.Version;// +1; 41 | this.source = changes.ChangesetSource; 42 | 43 | if (initializeAtOnce) 44 | Initialize(); 45 | } 46 | } 47 | 48 | /// 49 | /// Gets the changed DataSet instance. 50 | /// 51 | public DataSet DataSet 52 | { 53 | get { return dataSet; } 54 | } 55 | 56 | /// 57 | /// Gets the source of the changeset: it can be either local or remote or both. 58 | /// 59 | public ChangesetSource ChangesetSource 60 | { 61 | get { return source; } 62 | } 63 | 64 | /// 65 | /// Get the version number proposed for this changeset. 66 | /// 67 | public int ProposedVersion { get { return changeSetId; } } 68 | 69 | /// 70 | /// Returns changes for the variable with given id. 71 | /// If this variable hasn't been updated, returns null. 72 | /// 73 | /// 74 | /// 75 | internal Variable.Changes GetChanges(int varId) 76 | { 77 | Variable.Changes vc = null; 78 | if (varAdded != null) 79 | { 80 | vc = Array.Find(varAdded, p => p.ID == varId); 81 | if (vc != null) 82 | return vc; 83 | } 84 | if (varUpdated != null) 85 | { 86 | vc = Array.Find(varUpdated, p => p.ID == varId); 87 | } 88 | return vc; 89 | } 90 | 91 | private void Initialize() 92 | { 93 | if (initialized) return; 94 | 95 | /* Variables */ 96 | List added = new List(); 97 | List updated = new List(); 98 | 99 | VariableSchema[] initialVars = 100 | (changes.InitialSchema == null || changes.InitialSchema.Variables == null || changes.InitialSchema.Variables.Length == 0) 101 | ? null : changes.InitialSchema.Variables; 102 | Variable.Changes vch = null; 103 | foreach (Variable v in changes.Variables) 104 | { 105 | if (initialVars == null || !Array.Exists(initialVars, iv => iv.ID == v.ID)) 106 | { 107 | // New variable 108 | added.Add(changes.GetVariableChanges(v.ID).Clone()); 109 | } 110 | else if ((vch = changes.GetVariableChanges(v.ID)) != null) 111 | { 112 | // Updated 113 | updated.Add(vch.Clone()); 114 | } 115 | } 116 | varAdded = added.ToArray(); 117 | varUpdated = updated.ToArray(); 118 | 119 | 120 | initialized = true; 121 | } 122 | 123 | /// 124 | /// Gets an array of added variables. 125 | /// 126 | /// 127 | public Variable.Changes[] AddedVariables 128 | { 129 | get 130 | { 131 | Initialize(); 132 | return varAdded; 133 | } 134 | } 135 | 136 | /// 137 | /// Gets an array of updated variables. 138 | /// 139 | /// 140 | /// The array returned by this property doesn't include added variables 141 | /// even if they are updated just after they have been added. 142 | /// To get an array of added variables, see property . 143 | /// 144 | /// 145 | public Variable.Changes[] UpdatedVariables 146 | { 147 | get 148 | { 149 | Initialize(); 150 | return varUpdated; 151 | } 152 | } 153 | 154 | /// 155 | /// Gets an array of variables affected by the DataSet changes. 156 | /// 157 | /// 158 | /// The array returned by this property includes both added variables (see 159 | /// ) and updated variables (). 160 | /// 161 | /// 162 | /// 163 | public Variable.Changes[] AllAffectedVariables 164 | { 165 | get{ 166 | if (varAffected == null) 167 | { 168 | Initialize(); 169 | varAffected = new Variable.Changes[varAdded.Length + varUpdated.Length]; 170 | varAdded.CopyTo(varAffected, 0); 171 | varUpdated.CopyTo(varAffected, varAdded.Length); 172 | } 173 | return varAffected; 174 | } 175 | } 176 | 177 | /// 178 | /// Returns brief desrcription of the changeset. 179 | /// 180 | /// 181 | public override string ToString() 182 | { 183 | return String.Format("Changes: variables: {0} added, {1} updated", 184 | AddedVariables.Length, UpdatedVariables.Length); 185 | } 186 | } 187 | 188 | } 189 | 190 | -------------------------------------------------------------------------------- /FetchClimate1/ClimateService.Common/RequestDescriptors.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Globalization; 6 | using Microsoft.Research.Science.Data.Climate.Conventions; 7 | 8 | namespace Microsoft.Research.Science.Data.Climate.Common 9 | { 10 | public class MultipleRequestDescriptor 11 | { 12 | private int timeStep; 13 | 14 | public int TimeStep 15 | { 16 | get { return timeStep; } 17 | set { timeStep = value; } 18 | } 19 | protected ClimateParameter parameter; 20 | 21 | public ClimateParameter Parameter 22 | { 23 | get { return parameter; } 24 | set { parameter = value; } 25 | } 26 | protected TimeBounds timeBounds; 27 | 28 | public TimeBounds TimeBounds 29 | { 30 | get { return timeBounds; } 31 | set { timeBounds = value; } 32 | } 33 | protected FetchingTimeModes timeMode; 34 | 35 | public FetchingTimeModes TimeMode 36 | { 37 | get { return timeMode; } 38 | set { timeMode = value; } 39 | } 40 | protected List regions = new List(); 41 | 42 | public List Regions 43 | { 44 | get { return regions; } 45 | set { regions = value; } 46 | } 47 | 48 | public MultipleRequestDescriptor(ClimateParameter p, TimeBounds tb, FetchingTimeModes mode) 49 | { 50 | parameter = p; 51 | timeBounds = tb; 52 | timeMode = mode; 53 | } 54 | 55 | public void AddRegion(SpatialGrid sp) 56 | { 57 | regions.Add(sp); 58 | } 59 | 60 | 61 | public string ToString(char separator, char regionSeparator = ':') 62 | { 63 | StringBuilder sb = new StringBuilder(); 64 | sb.Append(Conventions.Namings.ParameterShortNames[Parameter]); 65 | if (TimeMode != FetchingTimeModes.Single) 66 | { 67 | sb.Append(separator); 68 | switch (TimeMode) 69 | { 70 | case FetchingTimeModes.TsHourly: 71 | sb.Append("/th"); 72 | break; 73 | case FetchingTimeModes.TsSeasonly: 74 | sb.Append("/ts"); 75 | break; 76 | case FetchingTimeModes.TsYearly: 77 | sb.Append("/ty"); 78 | break; 79 | } 80 | sb.Append(separator); 81 | sb.Append(TimeStep); 82 | } 83 | 84 | if (TimeBounds.MinYear != GlobalConsts.StartYear || TimeBounds.MaxYear != GlobalConsts.EndYear) 85 | { 86 | sb.Append(separator); 87 | sb.Append("/years"); 88 | sb.Append(separator); 89 | sb.Append(TimeBounds.MinYear); 90 | sb.Append(separator); 91 | sb.Append(TimeBounds.MaxYear); 92 | } 93 | if (!(TimeBounds.MinDay == GlobalConsts.DefaultValue && TimeBounds.MaxDay == GlobalConsts.DefaultValue) && 94 | !(TimeBounds.MinDay == 1 && TimeBounds.MaxDay == 365 && TimeBounds.MinYear != TimeBounds.MaxYear) && 95 | !(TimeBounds.MinDay == 1 && TimeBounds.MaxDay == 365 && TimeBounds.MinYear == TimeBounds.MaxYear && !DateTime.IsLeapYear(TimeBounds.MaxYear)) && 96 | !(TimeBounds.MinDay == 1 && TimeBounds.MaxDay == 366 && TimeBounds.MinYear == TimeBounds.MaxYear && DateTime.IsLeapYear(TimeBounds.MaxYear))) 97 | { 98 | sb.Append(separator); 99 | sb.Append("/days"); 100 | sb.Append(separator); 101 | sb.Append(TimeBounds.MinDay); 102 | sb.Append(separator); 103 | sb.Append(TimeBounds.MaxDay); 104 | } 105 | if (!(TimeBounds.MinHour == GlobalConsts.DefaultValue && TimeBounds.MaxHour == GlobalConsts.DefaultValue) && 106 | !(TimeBounds.MinHour == 0 && TimeBounds.MaxHour == 24)) 107 | { 108 | sb.Append(separator); 109 | sb.Append("/hours"); 110 | sb.Append(separator); 111 | sb.Append(TimeBounds.MinHour); 112 | sb.Append(separator); 113 | sb.Append(TimeBounds.MaxHour); 114 | } 115 | foreach (var reg in Regions) 116 | { 117 | sb.Append(separator); 118 | sb.Append(reg.LatMin.ToString(CultureInfo.InvariantCulture)); 119 | if (!double.IsNaN(reg.StepLat)) 120 | { 121 | sb.Append(regionSeparator); 122 | sb.Append(reg.StepLat.ToString(CultureInfo.InvariantCulture)); 123 | } 124 | if (reg.LatMax != reg.LatMin) 125 | { 126 | sb.Append(regionSeparator); 127 | sb.Append(reg.LatMax.ToString(CultureInfo.InvariantCulture)); 128 | } 129 | 130 | sb.Append(separator); 131 | sb.Append(reg.LonMin.ToString(CultureInfo.InvariantCulture)); 132 | if (!double.IsNaN(reg.StepLon)) 133 | { 134 | sb.Append(regionSeparator); 135 | sb.Append(reg.StepLon.ToString(CultureInfo.InvariantCulture)); 136 | } 137 | if (reg.LonMax != reg.LonMin) 138 | { 139 | sb.Append(regionSeparator); 140 | sb.Append(reg.LonMax.ToString(CultureInfo.InvariantCulture)); 141 | } 142 | } 143 | return sb.ToString(); 144 | } 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /FetchClimate1/ClimateServiceClient/ClimateService.Client.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Debug 5 | AnyCPU 6 | {B02F58BB-072E-4DC7-AAAB-674C14E01E9D} 7 | Library 8 | Properties 9 | Microsoft.Research.Science.Data 10 | ClimateServiceClient 11 | v4.7.1 12 | 512 13 | 3.5 14 | false 15 | publish\ 16 | true 17 | Disk 18 | false 19 | Foreground 20 | 7 21 | Days 22 | false 23 | false 24 | true 25 | 0 26 | 1.0.0.%2a 27 | false 28 | true 29 | 30 | 31 | bin\Release\ 32 | TRACE 33 | true 34 | pdbonly 35 | AnyCPU 36 | true 37 | GlobalSuppressions.cs 38 | prompt 39 | AllRules.ruleset 40 | 4 41 | 42 | 43 | true 44 | bin\Debug\ 45 | TRACE;DEBUG 46 | full 47 | AnyCPU 48 | C:\Scs\ScientificDataSet\WorldClimGenericProvider-dgrechka\sln\ClimateService\Debug\bin\ClimateServiceClient.dll.CodeAnalysisLog.xml 49 | true 50 | GlobalSuppressions.cs 51 | prompt 52 | AllRules.ruleset 53 | ;C:\Program Files (x86)\Microsoft Visual Studio 10.0\Team Tools\Static Analysis Tools\\Rule Sets 54 | true 55 | ;C:\Program Files (x86)\Microsoft Visual Studio 10.0\Team Tools\Static Analysis Tools\FxCop\\Rules 56 | true 57 | 4 58 | false 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | False 83 | .NET Framework 3.5 SP1 Client Profile 84 | false 85 | 86 | 87 | False 88 | .NET Framework 3.5 SP1 89 | true 90 | 91 | 92 | False 93 | Windows Installer 3.1 94 | true 95 | 96 | 97 | 98 | 99 | {87558098-AEAB-46A0-8BEF-D838630AC90E} 100 | ClimateService.Common 101 | 102 | 103 | 104 | 105 | {24d8613c-e1e9-4d7b-abaa-051eed4e5dbc} 106 | ScientificDataSet 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 121 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/Schemas.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | 6 | namespace Microsoft.Research.Science.Data 7 | { 8 | /// Provides structure information for a variable. 9 | /// This class is intended to be immutable, but metadata field 10 | /// and corresponding property allow modify operations. 11 | public class VariableSchema 12 | { 13 | private int id; 14 | private int changeSetId; 15 | private Type dataType; 16 | private ReadOnlyDimensionList dimensions; 17 | private MetadataDictionary metadata; 18 | 19 | internal VariableSchema(int changeSetId, int id, Type dataType, ReadOnlyDimensionList dimensions, MetadataDictionary metadata) 20 | { 21 | if (dimensions == null) 22 | throw new ArgumentNullException("dimensions"); 23 | if (metadata == null) 24 | throw new ArgumentNullException("metadata"); 25 | this.changeSetId = changeSetId; 26 | this.id = id; 27 | this.dimensions = dimensions; 28 | this.metadata = metadata; 29 | this.dataType = dataType; 30 | } 31 | 32 | /// 33 | /// Gets the ID of the variable. 34 | /// 35 | public int ID 36 | { 37 | get { return id; } 38 | } 39 | /// 40 | /// Gets the changeset of the variable. 41 | /// 42 | public int ChangeSet 43 | { 44 | get { return changeSetId; } 45 | } 46 | /// 47 | /// Gets the name of the variable. 48 | /// 49 | public string Name 50 | { 51 | get { return metadata.GetComittedOtherwiseProposedValue(metadata.KeyForName, null); } 52 | } 53 | /// 54 | /// Gets the data type of the variable. 55 | /// 56 | public Type TypeOfData 57 | { 58 | get { return dataType; } 59 | } 60 | /// 61 | /// Gets the read onlt dimension lists the variable depends on. 62 | /// 63 | public ReadOnlyDimensionList Dimensions 64 | { 65 | get { return dimensions; } 66 | } 67 | /// 68 | /// Gets the metadata collection. 69 | /// 70 | public MetadataDictionary Metadata 71 | { 72 | get { return metadata; } 73 | } 74 | /// 75 | /// Gets the rank of the variable. 76 | /// 77 | public int Rank 78 | { 79 | get { return dimensions.Count; } 80 | } 81 | /// 82 | /// Represents the schema as string in short. 83 | /// 84 | /// 85 | public override string ToString() 86 | { 87 | StringBuilder sb = new StringBuilder(); 88 | sb.AppendFormat("<[{0}]{1} of type {2}", ID, 89 | ID == DataSet.GlobalMetadataVariableID ? "" : Name, TypeOfData.Name); 90 | for (int i = 0; i < dimensions.Count; i++) 91 | { 92 | sb.Append(' '); 93 | sb.Append(dimensions[i]); 94 | } 95 | 96 | sb.Append('>'); 97 | return sb.ToString(); 98 | } 99 | } 100 | 101 | /// Provides structure information for a . 102 | /// This class is immutable. 103 | public class DataSetSchema 104 | { 105 | private readonly Guid guid; 106 | private readonly int version; 107 | private readonly VariableSchema[] vars; 108 | private readonly string uri; 109 | 110 | internal DataSetSchema(Guid guid, string uri, int version, VariableSchema[] vars) 111 | { 112 | this.guid = guid; 113 | this.vars = vars; 114 | this.version = version; 115 | this.uri = uri; 116 | } 117 | 118 | /// 119 | /// Gets the unique identifier of the DataSet. 120 | /// 121 | /// 122 | public Guid DataSetGuid 123 | { 124 | get { return guid; } 125 | } 126 | 127 | /// 128 | /// Gets the changeset number of the DataSet. 129 | /// 130 | /// 131 | public int Version 132 | { 133 | get { return version; } 134 | } 135 | 136 | /// 137 | /// Gets the DataSet URI. 138 | /// 139 | /// 140 | public string URI 141 | { 142 | get { return uri; } 143 | } 144 | 145 | /// 146 | /// Gets the DataSet URI. 147 | /// 148 | /// 149 | /// The property is obsolete. Use instead. 150 | [Obsolete("This property is obsolete and will be removed in the next version. Use the property \"URI\" instead.")] 151 | public string ConstructionString 152 | { 153 | get { return uri; } 154 | } 155 | 156 | /// 157 | /// Gets an array of the variables contained in the DataSet. 158 | /// 159 | public VariableSchema[] Variables 160 | { 161 | get { return vars; } 162 | } 163 | 164 | /// 165 | /// Gets an array of dimensions of the DataSet. 166 | /// 167 | /// An array of . 168 | /// 169 | /// If the schema corresponds to the proposed version of the DataSet and 170 | /// some dimension differs for different variables, in the returning array the dimension 171 | /// has length equal to -1. 172 | /// 173 | public Dimension[] GetDimensions() 174 | { 175 | if (vars == null || vars.Length == 0) return new Dimension[0]; 176 | 177 | Dictionary dims = new Dictionary(); 178 | foreach (var v in vars) 179 | foreach (var vd in v.Dimensions) 180 | { 181 | Dimension dim; 182 | if (dims.TryGetValue(vd.Name, out dim)) 183 | dim.Length = -1; 184 | else 185 | dim = vd; 186 | dims[vd.Name] = vd; 187 | } 188 | Dimension[] dimsArr = new Dimension[dims.Count]; 189 | dims.Values.CopyTo(dimsArr, 0); 190 | return dimsArr; 191 | } 192 | } 193 | 194 | /// 195 | /// Specifies the schema version. 196 | /// 197 | public enum SchemaVersion 198 | { 199 | /// 200 | /// Represents committed unmodified elements. 201 | /// 202 | Committed, 203 | /// 204 | /// Represents modified or added elements. 205 | /// 206 | Proposed, 207 | /// If the Proposed version is available, it is used; otherwise, the Committed version is used. 208 | Recent, 209 | } 210 | } 211 | 212 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/LambdaTransformVariable.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Diagnostics; 5 | 6 | namespace Microsoft.Research.Science.Data 7 | { 8 | internal class LambdaTransformVariable : TransformationVariable 9 | { 10 | private readonly Func forwardLambda; 11 | private readonly Func backwardLambda; 12 | 13 | internal LambdaTransformVariable( 14 | string name, 15 | Variable rawVariable, 16 | Func forwardTransform, 17 | Func backwardTransform, 18 | IList hiddenEntries, 19 | IList readonlyEntries) 20 | : base(rawVariable.DataSet, name, rawVariable, rawVariable.Dimensions.AsNamesArray(), hiddenEntries, readonlyEntries) 21 | { 22 | if (name == null) throw new ArgumentNullException("name"); 23 | if (rawVariable == null) throw new ArgumentNullException("rawVariable"); 24 | if (forwardTransform == null) throw new ArgumentNullException("forwardTransform"); 25 | if (rawVariable.Rank > 6) 26 | throw new NotSupportedException("LambdaTransformVariable doesn't support ranks more than 6"); 27 | 28 | forwardLambda = forwardTransform; 29 | backwardLambda = backwardTransform; 30 | 31 | Initialize(); 32 | if (backwardTransform == null) 33 | IsReadOnly = true; 34 | } 35 | protected override Array Transform(int[] origin, Array rawData) 36 | { 37 | int[] shape = new int[rawData.Rank]; 38 | for (int i = 0; i < shape.Length; i++) 39 | shape[i] = rawData.GetLength(i); 40 | 41 | Array data = Array.CreateInstance(typeof(DataType), shape); 42 | System.Diagnostics.Debug.Assert(rawData.LongLength == data.LongLength); 43 | switch (rawData.Rank) 44 | { 45 | case 1: 46 | loop(forwardLambda, (RawType[])rawData, (DataType[])data, shape[0]); 47 | break; 48 | case 2: 49 | loop(forwardLambda, (RawType[,])rawData, (DataType[,])data, shape[0], shape[1]); 50 | break; 51 | case 3: 52 | loop(forwardLambda, (RawType[, ,])rawData, (DataType[, ,])data, shape[0], shape[1], shape[2]); 53 | break; 54 | case 4: 55 | loop(forwardLambda, (RawType[, , ,])rawData, (DataType[, , ,])data, shape[0], shape[1], shape[2], shape[3]); 56 | break; 57 | case 5: 58 | loop(forwardLambda, (RawType[, , , ,])rawData, (DataType[, , , ,])data, shape[0], shape[1], shape[2], shape[3], shape[4]); 59 | break; 60 | case 6: 61 | loop(forwardLambda, (RawType[, , , , ,])rawData, (DataType[, , , , ,])data, shape[0], shape[1], shape[2], shape[3], shape[4], shape[5]); 62 | break; 63 | default: 64 | throw new NotSupportedException("LambdaTransformVariable doesn't support ranks more than 6"); 65 | } 66 | return data; 67 | } 68 | 69 | protected override Array ReverseTransform(int[] origin, Array data) 70 | { 71 | if (backwardLambda == null) 72 | throw new InvalidOperationException("Backward transformation wasn't supplied for LambdaTransformationVariable"); 73 | int rank = data.Rank; 74 | int[] shape = new int[rank]; 75 | for (int i = 0; i < shape.Length; i++) 76 | shape[i] = data.GetLength(i); 77 | 78 | Array rawData = Array.CreateInstance(typeof(RawType), shape); 79 | System.Diagnostics.Debug.Assert(rawData.LongLength == data.LongLength); 80 | switch (rank) 81 | { 82 | case 1: 83 | loop(backwardLambda, (DataType[])data, (RawType[])rawData, shape[0]); 84 | break; 85 | case 2: 86 | loop(backwardLambda, (DataType[,])data, (RawType[,])rawData, shape[0], shape[1]); 87 | break; 88 | case 3: 89 | loop(backwardLambda, (DataType[, ,])data, (RawType[, ,])rawData, shape[0], shape[1], shape[2]); 90 | break; 91 | case 4: 92 | loop(backwardLambda, (DataType[, , ,])data, (RawType[, , ,])rawData, shape[0], shape[1], shape[2], shape[3]); 93 | break; 94 | case 5: 95 | loop(backwardLambda, (DataType[, , , ,])data, (RawType[, , , ,])rawData, shape[0], shape[1], shape[2], shape[3], shape[4]); 96 | break; 97 | case 6: 98 | loop(backwardLambda, (DataType[, , , , ,])data, (RawType[, , , , ,])rawData, shape[0], shape[1], shape[2], shape[3], shape[4], shape[5]); 99 | break; 100 | default: 101 | throw new NotSupportedException("LambdaTransformVariable doesn't support ranks more than 6"); 102 | } 103 | return rawData; 104 | } 105 | 106 | #region loop routines 107 | private void loop(Func lambda, FROM[] from, TO[] to, int l0) 108 | { 109 | for (int i0 = 0; i0 < l0; i0++) 110 | to[i0] = lambda(from[i0]); 111 | } 112 | private void loop(Func lambda, FROM[,] from, TO[,] to, int l0, int l1) 113 | { 114 | for (int i0 = 0; i0 < l0; i0++) 115 | for (int i1 = 0; i1 < l1; i1++) 116 | to[i0, i1] = lambda(from[i0, i1]); 117 | } 118 | private void loop(Func lambda, FROM[, ,] from, TO[, ,] to, int l0, int l1, int l2) 119 | { 120 | for (int i0 = 0; i0 < l0; i0++) 121 | for (int i1 = 0; i1 < l1; i1++) 122 | for (int i2 = 0; i2 < l2; i2++) 123 | to[i0, i1, i2] = lambda(from[i0, i1, i2]); 124 | } 125 | private void loop(Func lambda, FROM[, , ,] from, TO[, , ,] to, int l0, int l1, int l2, int l3) 126 | { 127 | for (int i0 = 0; i0 < l0; i0++) 128 | for (int i1 = 0; i1 < l1; i1++) 129 | for (int i2 = 0; i2 < l2; i2++) 130 | for (int i3 = 0; i3 < l3; i3++) 131 | to[i0, i1, i2, i3] = lambda(from[i0, i1, i2, i3]); 132 | } 133 | private void loop(Func lambda, FROM[, , , ,] from, TO[, , , ,] to, int l0, int l1, int l2, int l3, int l4) 134 | { 135 | for (int i0 = 0; i0 < l0; i0++) 136 | for (int i1 = 0; i1 < l1; i1++) 137 | for (int i2 = 0; i2 < l2; i2++) 138 | for (int i3 = 0; i3 < l3; i3++) 139 | for (int i4 = 0; i4 < l4; i4++) 140 | to[i0, i1, i2, i3, i4] = lambda(from[i0, i1, i2, i3, i4]); 141 | } 142 | private void loop(Func lambda, FROM[, , , , ,] from, TO[, , , , ,] to, int l0, int l1, int l2, int l3, int l4, int l5) 143 | { 144 | for (int i0 = 0; i0 < l0; i0++) 145 | for (int i1 = 0; i1 < l1; i1++) 146 | for (int i2 = 0; i2 < l2; i2++) 147 | for (int i3 = 0; i3 < l3; i3++) 148 | for (int i4 = 0; i4 < l4; i4++) 149 | for (int i5 = 0; i5 < l5; i5++) 150 | to[i0, i1, i2, i3, i4, i5] = lambda(from[i0, i2, i3, i4, i4, i5]); 151 | } 152 | #endregion 153 | 154 | } 155 | } 156 | 157 | -------------------------------------------------------------------------------- /ScientificDataSet/Providers/Memory/MemoryDataSet.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections; 4 | using System.Collections.Generic; 5 | using System.Diagnostics; 6 | using System.IO; 7 | using System.Linq; 8 | using System.Reflection; 9 | using System.Text; 10 | using System.Xml; 11 | using System.Xml.Serialization; 12 | 13 | namespace Microsoft.Research.Science.Data.Memory 14 | { 15 | /// 16 | /// A provider that keeps all data in memory. 17 | /// 18 | /// 19 | /// 20 | /// Supports variables of any non-negative rank. 21 | /// 22 | /// 23 | /// accepts following parameters in the URI 24 | /// (read about DataSet URIs here: ; 25 | /// about preliminary URI customization ): 26 | /// 27 | /// 28 | /// Parameter 29 | /// Description 30 | /// 31 | /// 32 | /// name=DataSetName 33 | /// Name of the . 34 | /// 35 | /// 36 | /// include 37 | /// Allows including variables as references from another , defined as a URI, 38 | /// into this . 39 | /// Example: msds:memory?include=msds%3Acsv%3Ffile%example.csv%23lat%2Clon 40 | /// (escaped version of "msds:memory?include=escape[msds:csv?file=example.csv#lat,lon]") 41 | /// includes variables lat and lon from msds:csv?file=example.csv. If variables names are not specified, 42 | /// all variables are included. 43 | /// 44 | /// 45 | /// 46 | /// 47 | /// Creates the and adds a depending on dimension "x", initialized with an array: 48 | /// 49 | /// using(DataSet ds = DataSet.Open("msds:memory")) 50 | /// { 51 | /// Variable<int> var = ds.AddVariable<int>("var", "x"); 52 | /// var.PutData(new int[] { 1, 2, 3 }); 53 | /// } 54 | /// 55 | /// 56 | /// 57 | [DataSetProviderName("memory")] 58 | [DataSetProviderUriTypeAttribute(typeof(MemoryUri))] 59 | public class MemoryDataSet : DataSet 60 | { 61 | internal const string XmlNamespace = "http://research.microsoft.com/science/dataset"; 62 | 63 | /// 64 | /// Filters trace messages related to the DataSet. 65 | /// 66 | internal static TraceSwitch TraceMemoryDataSet = DataSet.TraceDataSet; 67 | 68 | /// 69 | /// Initializes an instance. 70 | /// 71 | public MemoryDataSet() : this(null) 72 | { 73 | } 74 | /// 75 | /// Initializes an instance. 76 | /// 77 | /// 78 | public MemoryDataSet(string uri) 79 | { 80 | if (DataSetUri.IsDataSetUri(uri)) 81 | { 82 | this.uri = new MemoryUri(uri); 83 | } 84 | else 85 | { 86 | this.uri = MemoryUri.FromName(uri); 87 | } 88 | 89 | bool autocommit = IsAutocommitEnabled; 90 | IsAutocommitEnabled = false; 91 | 92 | Name = ((MemoryUri)this.uri).Name; 93 | Commit(); 94 | 95 | IsAutocommitEnabled = autocommit; 96 | } 97 | /// 98 | /// Derived class must use the constructor to instantiate the data set. 99 | /// 100 | protected MemoryDataSet(bool createGlobalMetadataVariable) 101 | : base(createGlobalMetadataVariable, false) 102 | { 103 | } 104 | /// 105 | /// 106 | /// 107 | /// 108 | /// 109 | /// 110 | /// 111 | protected override Variable CreateVariable(string varName, string[] dims) 112 | { 113 | if (dims.Length == 1) 114 | return new MemoryVariable1d(this, varName, dims); 115 | return new MemoryVariable(this, varName, dims); 116 | } 117 | 118 | /// 119 | /// Gets the recent data from given memory variable. 120 | /// 121 | /// 122 | /// 123 | protected static Array GetRecentData(Variable variable) 124 | { 125 | if (variable == null) 126 | throw new ArgumentNullException("variable"); 127 | IInternalMemoryVariable mv = variable as IInternalMemoryVariable; 128 | if (mv == null) 129 | throw new ArgumentException("Given variable is not a MemoryVariable", "variable"); 130 | return mv.GetRecentData(); 131 | } 132 | } 133 | 134 | /// 135 | /// Allows to customize a URI with parameters specific for the provider. 136 | /// 137 | /// 138 | public class MemoryUri : DataSetUri 139 | { 140 | /// 141 | /// Instantiates an instance of the class. 142 | /// 143 | /// The DataSet uri. 144 | public MemoryUri(string uri) 145 | : base(uri, typeof(MemoryDataSet)) 146 | { 147 | } 148 | /// 149 | /// Instantiates an instance of the class with default parameters. 150 | /// 151 | public MemoryUri() 152 | : base(typeof(MemoryDataSet)) 153 | { 154 | } 155 | 156 | /// 157 | /// Name of the DataSet. 158 | /// 159 | public string Name 160 | { 161 | get { return GetParameterValue("name", ""); } 162 | set { SetParameterValue("name", value); } 163 | } 164 | 165 | internal static MemoryUri FromName(string name) 166 | { 167 | string providerName = Microsoft.Research.Science.Data.Factory.DataSetFactory.GetProviderNameByType(typeof(MemoryDataSet)) ?? 168 | ((DataSetProviderNameAttribute)typeof(MemoryDataSet).GetCustomAttributes(typeof(DataSetProviderNameAttribute), false)[0]).Name; 169 | if (String.IsNullOrEmpty(name)) 170 | return new MemoryUri(DataSetUri.DataSetUriScheme + ":" + providerName); 171 | return new MemoryUri(DataSetUri.DataSetUriScheme + ":" + providerName + "?name=" + name); 172 | } 173 | } 174 | } 175 | 176 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/Range.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | 7 | namespace Microsoft.Research.Science.Data 8 | { 9 | /// 10 | /// Represents non-negative integer range with stride. 11 | /// 12 | /// 13 | /// An instance of the struct can be instantiated 14 | /// using static methods of the class: 15 | /// 16 | /// 17 | /// Single index to choose. 18 | /// 19 | /// A range of indices from "from" up to "to". 20 | /// 21 | /// A range of indices with specified step. 22 | /// 23 | /// A range of indices from "from" up to the maximum index for a given dimension. 24 | /// 25 | /// A range of indices from "from" up to the maximum index for a given dimension with the specified step. 26 | /// 27 | /// Single index to choose and reduce this dimension so that the rank of the resulting array is less than the rank of the variable is read from. 28 | /// 29 | /// 30 | /// 31 | /// Ranges are used in the procedural API available as extensions methods for the 32 | /// class. See . 33 | /// 34 | /// 35 | /// 36 | public struct Range 37 | { 38 | private int origin; 39 | private int stride; 40 | private int count; 41 | private bool reduce; 42 | 43 | /// 44 | /// 45 | /// 46 | /// 47 | /// 48 | /// 49 | /// 50 | /// 51 | /// Negative count means that it is unlimited. 52 | /// 53 | /// 54 | internal Range(int origin, int stride, int count) 55 | { 56 | if (origin < 0) throw new ArgumentException("origin is negative"); 57 | if (stride <= 0) throw new ArgumentException("stride is not positive"); 58 | this.origin = origin; 59 | this.stride = stride; 60 | if (count < 0) count = -1; // unlimited 61 | this.count = count; 62 | this.reduce = false; 63 | } 64 | 65 | internal Range(int index, bool reduce) 66 | { 67 | if (index < 0) throw new ArgumentException("index is negative"); 68 | this.origin = index; 69 | this.stride = 1; 70 | this.count = 1; 71 | this.reduce = reduce; 72 | } 73 | 74 | /// 75 | /// Gets the value indicating that the related dimension is to be reduced. 76 | /// 77 | public bool IsReduced { get { return reduce; } } 78 | /// 79 | /// Gets the starting value of the range. 80 | /// 81 | public int Origin { get { return origin; } } 82 | /// 83 | /// Gets the stride value. 84 | /// 85 | public int Stride { get { return stride; } } 86 | /// 87 | /// Gets the number of values in the range. 88 | /// 89 | /// 90 | /// If the range , gets -1. 91 | /// 92 | public int Count { get { return count; } } 93 | /// 94 | /// Gets the last value of the range. 95 | /// 96 | /// 97 | /// If the range , the property throws an exception. 98 | /// 99 | public int Last 100 | { 101 | get 102 | { 103 | if (count < 0) throw new NotSupportedException("Range is unlimited"); 104 | return origin + stride * (count - 1); 105 | } 106 | } 107 | 108 | /// 109 | /// Gets the value indicating whether the range is unlimited. 110 | /// 111 | public bool IsUnlimited 112 | { 113 | get { return count < 0; } 114 | } 115 | 116 | /// 117 | /// Gets the value indicating that the range contains no values. 118 | /// 119 | public bool IsEmpty 120 | { 121 | get { return count == 0; } 122 | } 123 | 124 | /// 125 | /// Gets the string representation of the range. 126 | /// 127 | /// 128 | /// 129 | /// It is either "(empty)" or "([start]:[stride]:[final])" 130 | /// 131 | public override string ToString() 132 | { 133 | if (IsEmpty) return "(empty)"; 134 | if (count == 1) 135 | if (reduce) return origin.ToString() + " (reduce)"; 136 | else return origin.ToString(); 137 | return String.Format("({0}:{1}:{2})", origin, stride, IsUnlimited ? String.Empty : Last.ToString()); 138 | } 139 | /// 140 | /// 141 | /// 142 | /// 143 | public override bool Equals(object obj) 144 | { 145 | if (obj == null || !(obj is Range)) return false; 146 | 147 | Range r = (Range)obj; 148 | if (count != r.count) return false; 149 | if (count == 0) return true; 150 | if (count == 1) return origin == r.origin && reduce == r.reduce; 151 | return origin == r.origin && stride == r.stride && count == r.count; 152 | } 153 | /// 154 | /// 155 | /// 156 | public override int GetHashCode() 157 | { 158 | if (count == 0) return 0; 159 | if (count == 1) return (origin + 1); 160 | return (origin + 1) ^ stride ^ count; 161 | } 162 | } 163 | } 164 | 165 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/OrderedArray1d.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Collections; 7 | 8 | namespace Microsoft.Research.Science.Data 9 | { 10 | internal sealed class OrderedArray1d : ArrayWrapper 11 | { 12 | private ArrayOrder order; 13 | 14 | public OrderedArray1d(Type type, bool ordered) 15 | : base(1, type) 16 | { 17 | if (ordered && TypeUtils.IsNumeric(type)) 18 | order = ArrayOrder.Unknown; 19 | else 20 | order = ArrayOrder.None; 21 | } 22 | 23 | public OrderedArray1d(Type type) 24 | : this(type, true) 25 | { 26 | } 27 | 28 | public void EnforceOrder(bool enforce) 29 | { 30 | if (!TypeUtils.IsNumeric(type) && !TypeUtils.IsDateTime(type) && !(type == typeof(string))) return; 31 | 32 | if (enforce) 33 | order = ArrayOrder.Unknown; 34 | else 35 | order = ArrayOrder.None; 36 | } 37 | 38 | public ArrayOrder Order { get { return order; } } 39 | 40 | /// 41 | /// Works as Array.BinarySearch() 42 | /// 43 | public int IndexOf(object value) 44 | { 45 | if (order == ArrayOrder.None) 46 | throw new Exception("Axis is not ordered."); 47 | 48 | if (order == ArrayOrder.Unknown) 49 | { 50 | SetUpOrder(array); 51 | } 52 | 53 | if (array == null || array.Length == 0) 54 | throw new Exception("Array is empty."); 55 | if (array.Length == 1) 56 | return 0; 57 | 58 | if (!TypeUtils.IsNumeric(type) && !TypeUtils.IsDateTime(type)) 59 | { 60 | for (int i = 0; i < array.Length; i++) 61 | { 62 | if (value == array.GetValue(i)) 63 | return i; 64 | } 65 | throw new Exception("Value not found"); 66 | } 67 | 68 | if (order == ArrayOrder.Ascendant) 69 | return Array.BinarySearch(array, value); 70 | 71 | return Array.BinarySearch(array, value, new DescComparer()); 72 | } 73 | 74 | public override void PutData(int[] origin, Array a) 75 | { 76 | base.PutData(origin, a); 77 | 78 | if (order == ArrayOrder.None) return; 79 | if (!TypeUtils.IsNumeric(type)) return; 80 | 81 | int iStart = (origin == null) ? 0 : origin[0]; 82 | int iEnd = Math.Min(iStart + a.Length, array.Length); 83 | 84 | if (iStart > 0) iStart--; 85 | if (iStart > 0) iStart--; 86 | if (iEnd < array.Length) iEnd++; 87 | if (iEnd < array.Length) iEnd++; 88 | 89 | 90 | if (order == ArrayOrder.Unknown) 91 | { 92 | SetUpOrder(array); 93 | iStart = 0; 94 | iEnd = array.Length; 95 | } 96 | 97 | Comparer cmp = Comparer.DefaultInvariant; 98 | 99 | if (order == ArrayOrder.Ascendant) 100 | { 101 | for (int i = iStart; i < iEnd - 1; i++) 102 | if (cmp.Compare(array.GetValue(i + 1), array.GetValue(i)) <= 0) 103 | throw new Exception("Axis is not sorted."); 104 | } 105 | else if (order == ArrayOrder.Descendant) 106 | { 107 | for (int i = iStart; i < iEnd - 1; i++) 108 | if (cmp.Compare(array.GetValue(i + 1), array.GetValue(i)) >= 0) 109 | throw new Exception("Axis is not sorted."); 110 | } 111 | } 112 | 113 | private void SetUpOrder(Array a) 114 | { 115 | if (a.Length <= 1) return; 116 | int res = GetOrder(a); 117 | if (res == 0) throw new Exception("Axis must be strictly monotonic."); 118 | order = (res > 0) ? ArrayOrder.Ascendant : ArrayOrder.Descendant; 119 | } 120 | 121 | public override ArrayWrapper Copy() 122 | { 123 | ArrayWrapper aw = new OrderedArray1d(type); 124 | if (array != null) 125 | aw.PutData(null, array); 126 | 127 | return aw; 128 | } 129 | 130 | private class DescComparer : IComparer 131 | { 132 | #region IComparer Members 133 | 134 | public int Compare(object x, object y) 135 | { 136 | return Comparer.Default.Compare(y, x); 137 | } 138 | 139 | #endregion 140 | } 141 | 142 | /// 143 | /// Determines the order of the array. 144 | /// 145 | public static ArrayOrder IsOrdered(Array a) 146 | { 147 | if (a == null || a.Length <= 1) 148 | return ArrayOrder.Unknown; 149 | 150 | int diff = GetOrder(a); 151 | 152 | if (diff == 0) return ArrayOrder.None; 153 | ArrayOrder order = (diff > 0) ? ArrayOrder.Ascendant : ArrayOrder.Descendant; 154 | 155 | Comparer cmp = Comparer.DefaultInvariant; 156 | if (order == ArrayOrder.Ascendant) 157 | { 158 | for (int i = 0; i < a.Length - 1; i++) 159 | if (cmp.Compare(a.GetValue(i + 1), a.GetValue(i)) <= 0) 160 | return ArrayOrder.None; 161 | } 162 | else if (order == ArrayOrder.Descendant) 163 | { 164 | for (int i = 0; i < a.Length - 1; i++) 165 | if (cmp.Compare(a.GetValue(i + 1), a.GetValue(i)) >= 0) 166 | return ArrayOrder.None; 167 | } 168 | return order; 169 | } 170 | 171 | /// 172 | /// Determines the order of the array. 173 | /// 174 | public static ArrayOrder IsOrdered(T[] a) 175 | { 176 | if (a == null || a.Length <= 1) 177 | return ArrayOrder.Unknown; 178 | 179 | Comparer c = Comparer.DefaultInvariant; 180 | int diff = c.Compare(a[1], a[0]); 181 | 182 | if (diff == 0) return ArrayOrder.None; 183 | ArrayOrder order = (diff > 0) ? ArrayOrder.Ascendant : ArrayOrder.Descendant; 184 | 185 | 186 | if (order == ArrayOrder.Ascendant) 187 | { 188 | for (int i = 1; i < a.Length - 1; i++) 189 | if (c.Compare(a[i + 1], a[i]) <= 0) 190 | return ArrayOrder.None; 191 | } 192 | else if (order == ArrayOrder.Descendant) 193 | { 194 | for (int i = 1; i < a.Length - 1; i++) 195 | if (c.Compare(a[i + 1], a[i]) >= 0) 196 | return ArrayOrder.None; 197 | } 198 | return order; 199 | } 200 | 201 | 202 | /// 203 | /// Positive: ascend 204 | /// Negative: descend 205 | /// 206 | private static int GetOrder(Array a) 207 | { 208 | return Comparer.DefaultInvariant.Compare(a.GetValue(1), a.GetValue(0)); 209 | } 210 | 211 | /// 212 | /// Positive: ascend 213 | /// Negative: descend 214 | /// 215 | private static int GetOrder(T[] a) 216 | { 217 | return Comparer.DefaultInvariant.Compare(a[1], a[0]); 218 | } 219 | } 220 | 221 | /// 222 | /// Describes an order of an array. 223 | /// 224 | public enum ArrayOrder 225 | { 226 | /// 227 | /// An array is not ordered. 228 | /// 229 | None, 230 | /// 231 | /// An array is not checked for an order. 232 | /// 233 | NotChecked, 234 | /// 235 | /// An array is ordered, but its order is still unknown. 236 | /// 237 | Unknown, 238 | /// 239 | /// An array has an ascendant order. 240 | /// 241 | Ascendant, 242 | /// 243 | /// An array has a descendant order. 244 | /// 245 | Descendant 246 | } 247 | 248 | } 249 | 250 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/RefVariable.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Diagnostics; 5 | using System.Linq; 6 | using System.Text; 7 | using System.Threading; 8 | 9 | namespace Microsoft.Research.Science.Data 10 | { 11 | /// 12 | /// RefVariable is a special kind of variables whose purpose is to refer another variable. 13 | /// 14 | /// It enables adding of a variable to not native data set. 15 | /// Data operation applied to the reference are translated to the target variable. 16 | /// Both target and reference variables share the data changing as the change of its inner state. 17 | /// Both target and reference variables share the metadata collection. 18 | /// Names of dimensions might change independently and don't cause another variable to be changed. 19 | /// RefVariable allows to extend a collection of coordinate systems of the target variable with its own coordinate system. 20 | /// 21 | /// 22 | internal sealed class RefVariable : Variable, IRefVariable, IDataRequestable 23 | { 24 | private Variable refVariable; 25 | 26 | public RefVariable(DataSet sds, Variable variable, string[] dimensions) 27 | : base(sds, dimensions) 28 | { 29 | refVariable = variable; 30 | refVariable.Changed += new VariableChangedEventHandler(RefVariableChanged); 31 | refVariable.RolledBack += new VariableRolledBackEventHandler(RefVariableRolledBack); 32 | //refVariable.CoordinateSystemAdded += new CoordinateSystemAddedEventHandler(RefVariableCoordinateSystemAdded); 33 | 34 | this.ID = ~this.ID; 35 | 36 | Initialize(); 37 | } 38 | 39 | public RefVariable(DataSet sds, Variable variable) 40 | : this(sds, variable, variable.Dimensions.AsNamesArray()) 41 | { 42 | } 43 | 44 | /// 45 | /// Gets the refenced variable instance. 46 | /// 47 | public Variable ReferencedVariable 48 | { 49 | get { return refVariable; } 50 | } 51 | 52 | /// 53 | /// Gets the shared metadata collection. 54 | /// 55 | public override MetadataDictionary Metadata 56 | { 57 | get { return refVariable.Metadata; } 58 | } 59 | 60 | private void RefVariableRolledBack(object sender, VariableRolledBackEventArgs e) 61 | { 62 | Rollback(); 63 | } 64 | 65 | private void RefVariableChanged(object sender, VariableChangedEventArgs e) 66 | { 67 | if (e.Action == VariableChangeAction.PutData) 68 | { 69 | StartChanges(); 70 | changes.Shape = e.Changes.Shape; 71 | changes.AffectedRectangle = e.Changes.AffectedRectangle; 72 | 73 | FireEventVariableChanged(VariableChangeAction.PutData); 74 | } 75 | } 76 | 77 | internal protected override void OnRollback(Variable.Changes proposedChanges) 78 | { 79 | try 80 | { 81 | refVariable.RolledBack -= RefVariableRolledBack; 82 | refVariable.Rollback(); 83 | } 84 | finally 85 | { 86 | refVariable.RolledBack += RefVariableRolledBack; 87 | } 88 | 89 | if (Version == 0) // the variable is added and rolled back 90 | { 91 | refVariable.Changed -= RefVariableChanged; 92 | refVariable.RolledBack -= RefVariableRolledBack; 93 | } 94 | } 95 | 96 | /// 97 | /// 98 | /// 99 | /// 100 | /// 101 | public override void PutData(int[] origin, Array a) 102 | { 103 | refVariable.PutData(origin, a); 104 | } 105 | 106 | public override void Append(Array a, int dimToAppend) 107 | { 108 | refVariable.Append(a, dimToAppend); 109 | } 110 | 111 | /// 112 | /// 113 | /// 114 | /// 115 | protected override int[] ReadShape() 116 | { 117 | return refVariable.GetShape(); 118 | } 119 | 120 | /// 121 | /// 122 | /// 123 | /// 124 | /// 125 | /// 126 | public override Array GetData(int[] origin, int[] shape) 127 | { 128 | return refVariable.GetData(origin, shape); 129 | } 130 | 131 | /// 132 | /// 133 | /// 134 | /// 135 | public override string ToString() 136 | { 137 | return ToString("@"); 138 | } 139 | 140 | #region Explicit implementation of IRefVariable 141 | 142 | Variable IRefVariable.ReferencedVariable 143 | { 144 | get { return refVariable; } 145 | } 146 | 147 | 148 | /// 149 | /// Updates the reference variable on the basis of custom changes of 150 | /// its referenced variable. 151 | /// 152 | /// 153 | void IRefVariable.UpdateChanges(Variable.Changes customChanges) 154 | { 155 | StartChanges(); 156 | changes.Shape = customChanges.Shape; 157 | changes.AffectedRectangle = customChanges.AffectedRectangle; 158 | changes.MetadataChanges = customChanges.MetadataChanges; 159 | 160 | FireEventVariableChanged(VariableChangeAction.PutData); 161 | } 162 | 163 | #endregion 164 | 165 | #region IDataRequestable Members 166 | 167 | void IDataRequestable.RequestData(int[] origin, int[] shape, VariableResponseHandler responseHandler) 168 | { 169 | ((IDataRequestable)this).RequestData(origin, null, shape, responseHandler); 170 | } 171 | 172 | void IDataRequestable.RequestData(int[] origin, int[] stride, int[] shape, VariableResponseHandler responseHandler) 173 | { 174 | if (this.refVariable is IDataRequestable) 175 | { 176 | VariableResponseHandler myResponseHandler = new VariableResponseHandler( 177 | delegate(VariableResponse resp) 178 | { 179 | VariableResponse response = new VariableResponse(this, origin, stride, resp.Data, Version); 180 | responseHandler(response); 181 | }); 182 | 183 | ((IDataRequestable)refVariable).RequestData(origin, stride, shape, myResponseHandler); 184 | return; 185 | } 186 | 187 | ThreadPool.QueueUserWorkItem(new WaitCallback((state) => 188 | { 189 | try 190 | { 191 | Array a = GetData(origin, stride, shape); 192 | responseHandler(new VariableResponse(this, origin, stride, a, Version)); 193 | } 194 | catch (Exception ex) 195 | { 196 | responseHandler(new VariableResponse(this, origin, stride, ex)); 197 | } 198 | })); 199 | } 200 | 201 | #endregion 202 | } 203 | 204 | /// 205 | /// Provides non-generic access to the reference variable. 206 | /// 207 | public interface IRefVariable 208 | { 209 | /// 210 | /// Gets the referenced variable. 211 | /// 212 | Variable ReferencedVariable { get; } 213 | 214 | /// 215 | /// Updates the reference variable on the basis of custom changes of 216 | /// its referenced variable. 217 | /// 218 | /// 219 | void UpdateChanges(Variable.Changes changes); 220 | } 221 | } 222 | 223 | 224 | -------------------------------------------------------------------------------- /FetchClimate1/ClimateServiceClient/WcfProcessingClient.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading; 7 | using System.ServiceModel.Channels; 8 | using System.ServiceModel; 9 | using Microsoft.Research.Science.Data.Climate; 10 | using Microsoft.Research.Science.Data.Climate.Conventions; 11 | using System.IO; 12 | using Microsoft.Research.Science.Data.Climate.Common; 13 | using Microsoft.Research.Science.Data.Imperative; 14 | using Microsoft.Research.Science.Data; 15 | 16 | #if FC_WCFCLIENT 17 | using Microsoft.Research.Science.Data.Proxy; 18 | using Microsoft.Research.Science.Data.Proxy.WCF; 19 | using Microsoft.Research.Science.Data.Utils; 20 | 21 | namespace Microsoft.Research.Science.Data.Processing 22 | { 23 | public class WcfProcessingClient : ProcessingClientBase 24 | { 25 | private static DispatcherQueue taskQueue; 26 | private string serviceUri; 27 | 28 | private NonDisposingServicePort _servicePort = null; 29 | 30 | protected NonDisposingServicePort ServicePort 31 | { 32 | get 33 | { 34 | lock (taskQueue) 35 | { 36 | if (_servicePort == null) 37 | _servicePort = new NonDisposingServicePort(WcfDataSetFactory.GetRemoteServicePort(this.serviceUri, "sdsControlEP", "sdsDataEP")); 38 | } 39 | return _servicePort; 40 | } 41 | } 42 | 43 | static WcfProcessingClient() 44 | { 45 | #if DEBUG 46 | // Microsoft.Research.Science.Data.Factory.DataSetFactory.Register(typeof(Microsoft.Research.Science.Data.Memory.MemoryDataSet)); 47 | //Microsoft.Research.Science.Data.Factory.DataSetFactory.Register( 48 | // typeof(Microsoft.Research.Science.Data.CSV.CsvDataSet)); 49 | //Microsoft.Research.Science.Data.Factory.DataSetFactory.Register( 50 | // typeof(Microsoft.Research.Science.Data.Proxy.WCF.WcfDataSetFactory)); 51 | //Console.WriteLine(Microsoft.Research.Science.Data.Factory.DataSetFactory.RegisteredToString()); 52 | #endif 53 | taskQueue = new DispatcherQueue("ClimateServiceClient", dispatcher); 54 | } 55 | 56 | static Dispatcher dispatcher = new Dispatcher(2, ThreadPriority.Normal, true, "ClimateClient"); 57 | 58 | public WcfProcessingClient(string serviceUri) 59 | : base() 60 | { 61 | if (serviceUri == null) 62 | throw new ArgumentNullException("serviceUri"); 63 | if (string.IsNullOrEmpty(serviceUri)) 64 | throw new ArgumentException("Service URI is not specified", "serviceUri"); 65 | 66 | this.serviceUri = serviceUri; 67 | 68 | } 69 | 70 | protected override DataSet ServerProcessInternal(DataSet ds) 71 | { 72 | if (serviceUri == "(local)") 73 | return LocalProcess(ds); 74 | 75 | string hash = DataSetDiskCache.ComputeHash(ds); 76 | 77 | DataSet proxyDataSet = null; 78 | 79 | // Creating new DataSet at the service. 80 | // TODO: fix following: 81 | try 82 | { 83 | try 84 | { 85 | proxyDataSet = ProxyDataSet.CreateProxySync(taskQueue, ServicePort, "msds:memory", false, 10 * 60 * 1000); 86 | } 87 | catch (CommunicationObjectFaultedException) 88 | { 89 | //Connection to server closed. 90 | //Recreate service port and try again. 91 | if (proxyDataSet != null && !proxyDataSet.IsDisposed) proxyDataSet.Dispose(); 92 | this._servicePort = null; 93 | proxyDataSet = ProxyDataSet.CreateProxySync(taskQueue, ServicePort, "msds:memory", false, 10 * 60 * 1000); 94 | } 95 | AutoResetEvent completed = new AutoResetEvent(false); 96 | OnCommittedHandler onCommitted = new OnCommittedHandler(completed, OnDataSetCommitted); 97 | proxyDataSet.Committed += onCommitted.Handler; 98 | 99 | proxyDataSet.IsAutocommitEnabled = false; 100 | FetchClimateRequestBuilder.CopyRequestedDataSet(ds, proxyDataSet, false); 101 | proxyDataSet.Metadata[Namings.metadataNameHash] = hash; 102 | proxyDataSet.Commit(); 103 | 104 | if (proxyDataSet.HasChanges) proxyDataSet.Commit(); 105 | 106 | completed.WaitOne(); 107 | proxyDataSet.IsAutocommitEnabled = true; 108 | return proxyDataSet; 109 | } 110 | catch 111 | { 112 | if (proxyDataSet != null && !proxyDataSet.IsDisposed) proxyDataSet.Dispose(); 113 | throw; 114 | } 115 | } 116 | 117 | #if !RELEASE_ASSEMBLY 118 | protected override DataSet LocalProcess(DataSet ds) 119 | { 120 | DataSet resultDs = null; 121 | try 122 | { 123 | //Microsoft.Research.Science.Data.Factory.DataSetFactory.Register(typeof(Microsoft.Research.Science.Data.Memory2.ChunkedMemoryDataSet)); 124 | resultDs = DataSet.Open("msds:memory"); 125 | resultDs.IsAutocommitEnabled = false; 126 | FetchClimateRequestBuilder.CopyRequestedDataSet(ds, resultDs, false); 127 | if (resultDs.HasChanges) resultDs.Commit(); 128 | Microsoft.Research.Science.Data.Climate.Processing.ClimateRequestProcessor.Process(resultDs, 0); 129 | 130 | if (FetchClimateRequestBuilder.IsProcessingSuccessful(resultDs)) 131 | ;//cache.Add(ds,ComputeHash(request)); 132 | else if (!FetchClimateRequestBuilder.IsProcessingFailed(resultDs)) 133 | throw new Exception("Processor hasn't finished the work."); 134 | 135 | resultDs.IsAutocommitEnabled = true; 136 | return resultDs; 137 | } 138 | catch 139 | { 140 | if (resultDs != null && !resultDs.IsDisposed) resultDs.Dispose(); 141 | throw; 142 | } 143 | } 144 | #endif 145 | 146 | private void OnDataSetCommitted(DataSetCommittedEventArgs e, OnCommittedHandler handler) 147 | { 148 | if ((e.Changes.ChangesetSource & ChangesetSource.Remote) == 0) 149 | { 150 | return; // we're waiting for remote changes 151 | } 152 | var ds = e.DataSet; 153 | if (FetchClimateRequestBuilder.IsProcessingSuccessful(ds) || FetchClimateRequestBuilder.IsProcessingFailed(ds)) 154 | { 155 | ds.Committed -= handler.Handler; 156 | handler.Completed.Set(); 157 | } 158 | } 159 | } 160 | } 161 | 162 | #endif 163 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/Rectangle.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | 7 | namespace Microsoft.Research.Science.Data 8 | { 9 | /// 10 | /// Represents multidimensional integer rectangle. 11 | /// 12 | public struct Rectangle 13 | { 14 | private int[] origin; 15 | private int[] shape; 16 | 17 | /// 18 | /// Initialies an instance of the class. 19 | /// 20 | /// 21 | /// 22 | /// 23 | /// and cannot be null. 24 | /// 25 | public Rectangle(int[] origin, int[] shape) 26 | { 27 | if (origin == null && shape != null) 28 | throw new ArgumentNullException("origin"); 29 | if (shape == null && origin != null) 30 | throw new ArgumentNullException("shape"); 31 | if (origin != null && shape.Length != origin.Length) 32 | throw new ArgumentException("Shape length is not equal to origin's length."); 33 | 34 | this.origin = origin; 35 | this.shape = shape; 36 | } 37 | 38 | /// 39 | /// 40 | /// 41 | /// 42 | /// 43 | public Rectangle(int origin, int shape) 44 | { 45 | this.origin = new int[] { origin }; 46 | this.shape = new int[] { shape }; 47 | } 48 | 49 | /// 50 | /// 51 | /// 52 | /// 53 | public Rectangle(int rank) 54 | { 55 | this.origin = new int[rank]; 56 | this.shape = new int[rank]; 57 | } 58 | /// 59 | /// Gets the value indicating whether the rectangle is empty or not. 60 | /// 61 | /// 62 | /// 63 | /// Rectangle is empty if its shape is null or contains only zeros. 64 | /// 65 | /// 66 | public bool IsEmpty 67 | { 68 | get 69 | { 70 | return (shape == null || Array.TrueForAll(shape, s => s == 0)); 71 | } 72 | } 73 | /// 74 | /// 75 | /// 76 | public int[] Origin 77 | { 78 | get { return origin; } 79 | } 80 | /// 81 | /// 82 | /// 83 | public int[] Shape 84 | { 85 | get { return shape; } 86 | } 87 | /// 88 | /// 89 | /// 90 | public int Rank 91 | { 92 | get { return origin.Length; } 93 | } 94 | /// 95 | /// Combines two rectangle returning the mimimal rectangle containg both rectangles. 96 | /// 97 | /// 98 | /// Different ranks. 99 | public void CombineWith(Rectangle r) 100 | { 101 | if (Rank != r.Rank) 102 | throw new ArgumentException("Rectangles are defined in spaces with different ranks."); 103 | 104 | for (int i = 0; i < origin.Length; i++) 105 | { 106 | int max = shape[i] + origin[i]; 107 | 108 | origin[i] = Math.Min(origin[i], r.origin[i]); 109 | 110 | if (max < r.origin[i] + r.shape[i]) 111 | max = r.origin[i] + r.shape[i]; 112 | shape[i] = max - origin[i]; 113 | } 114 | } 115 | /// 116 | /// 117 | /// 118 | /// 119 | public override string ToString() 120 | { 121 | StringBuilder sb = new StringBuilder(); 122 | if (IsEmpty) 123 | { 124 | sb.Append("Empty"); 125 | } 126 | else 127 | { 128 | sb.Append("Origin:"); 129 | for (int i = 0; i < origin.Length; i++) 130 | sb.AppendFormat(" {0}", origin[i]); 131 | 132 | sb.Append(", shape:"); 133 | for (int i = 0; i < shape.Length; i++) 134 | sb.AppendFormat(" {0}", shape[i]); 135 | } 136 | return sb.ToString(); 137 | } 138 | /// 139 | /// 140 | /// 141 | /// 142 | /// 143 | public override bool Equals(object obj) 144 | { 145 | if (obj == null || GetType() != obj.GetType()) 146 | { 147 | return false; 148 | } 149 | 150 | Rectangle r = (Rectangle)obj; 151 | if (r.shape.Length != shape.Length) 152 | return false; 153 | for (int i = 0; i < shape.Length; i++) 154 | { 155 | if (shape[i] != r.shape[i]) return false; 156 | if (origin[i] != r.origin[i]) return false; 157 | } 158 | return true; 159 | } 160 | 161 | /// 162 | /// 163 | /// 164 | /// 165 | public override int GetHashCode() 166 | { 167 | int hash = 0; 168 | for (int i = 0; i < shape.Length; i++) 169 | { 170 | hash ^= origin[i] ^ shape[i]; 171 | } 172 | return hash; 173 | } 174 | 175 | /// 176 | /// Checks whether two rectangles intersect or not. 177 | /// 178 | /// 179 | /// 180 | /// 181 | public static bool HasIntersection(Rectangle r1, Rectangle r2) 182 | { 183 | if (r1.IsEmpty || r2.IsEmpty) 184 | return false; 185 | if (r1.Rank != r2.Rank) 186 | throw new ArithmeticException("r1 and r2 have different rank"); 187 | 188 | int r = r1.Rank; 189 | for (int i = 0; i < r; i++) 190 | { 191 | int a = Math.Max(r1.origin[i], r2.origin[i]); 192 | int b = Math.Min(r1.origin[i] + r1.shape[i] - 1, r2.origin[i] + r2.shape[i] - 1); 193 | if (a > b) 194 | return false; 195 | } 196 | 197 | return true; 198 | } 199 | 200 | /// 201 | /// Combines two rectangle returning the mimimal rectangle containg both rectangles. 202 | /// 203 | /// 204 | /// 205 | /// 206 | public static Rectangle Combine(Rectangle r1, Rectangle r2) 207 | { 208 | if (r1.Rank != r2.Rank) 209 | throw new ArgumentException("Rectangles are defined in spaces with different ranks."); 210 | 211 | Rectangle r = new Rectangle(r1.Rank); 212 | 213 | for (int i = 0; i < r1.origin.Length; i++) 214 | { 215 | int max = r1.shape[i] + r1.origin[i]; 216 | 217 | r.origin[i] = Math.Min(r1.origin[i], r2.origin[i]); 218 | 219 | if (max < r2.origin[i] + r2.shape[i]) 220 | max = r2.origin[i] + r2.shape[i]; 221 | 222 | r.shape[i] = max - r1.origin[i]; 223 | } 224 | return r; 225 | } 226 | 227 | /// 228 | /// Gets the rectangle with zero origin and given shape. 229 | /// 230 | /// Shape of the resulting rectangle. 231 | /// The rectangle. 232 | public static Rectangle EntireShape(int[] shape) 233 | { 234 | if (shape == null) 235 | throw new ArgumentNullException("shape"); 236 | return new Rectangle(new int[shape.Length], shape); 237 | } 238 | 239 | /// 240 | /// Gets an empty rectangle (i.e. its origin and shape are null). 241 | /// 242 | public static Rectangle EmptyRectangle 243 | { 244 | get { return new Rectangle(); } 245 | } 246 | } 247 | } 248 | 249 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/RefVariableMetadata.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | 7 | namespace Microsoft.Research.Science.Data 8 | { 9 | /// 10 | /// RefVariableMetadata is a wrapper of an existing variable's metadata with 11 | /// some additional capabilities. 12 | /// 13 | /// 14 | /// The and parameters 15 | /// allow to make some metadata entries indpendent from the metadata of the underlying variable. 16 | /// These parameters can be null and this will be considered as an empty collection. 17 | /// Two entries are always independent from the underlying metadata. These are 18 | /// the name and the provenance entries of the variable's metadata. 19 | /// 20 | /// 21 | internal class RefVariableMetadata : MetadataDictionary 22 | { 23 | #region Private fields 24 | 25 | /// The target variable that contains referred metadata. 26 | private Variable target; 27 | 28 | /// Collection of keys which are not to be inherited from the underlying metadata. 29 | /// These entries are changed independently. 30 | private List hiddenEntries; 31 | 32 | /// Collection of keys that cannot be changed through this collection. 33 | private List readonlyEntries; 34 | 35 | /// Collection of entries currently being handled. This collection should 36 | /// enable correct concurrent work on simultaneous changing of the target and this collections. 37 | private List> proposedMetadataEntries = new List>(); 38 | 39 | #endregion 40 | 41 | #region Constructors 42 | 43 | /// 44 | /// Creates an instance of the RefVariableMetadata class. 45 | /// 46 | /// The target variable that contains referred metadata. 47 | /// 48 | /// See remarks for the class. 49 | /// 50 | public RefVariableMetadata(Variable var) 51 | : this(var, null, null) 52 | { 53 | } 54 | 55 | /// 56 | /// Creates an instance of the RefVariableMetadata class. 57 | /// 58 | /// The target variable that contains referred metadata. 59 | /// Collection of keys which are not to be inherited from the underlying metadata. 60 | /// These entries are changed independently. 61 | /// 62 | /// See remarks for the class. 63 | /// 64 | public RefVariableMetadata(Variable var, IList hiddenEntries) 65 | : this(var, hiddenEntries, null) 66 | { 67 | } 68 | 69 | /// 70 | /// Creates an instance of the RefVariableMetadata class. 71 | /// 72 | /// The target variable that contains referred metadata. 73 | /// Collection of keys which are not to be inherited from the underlying metadata. 74 | /// These entries are changed independently. 75 | /// Collection of keys that cannot be changed through this collection. 76 | /// 77 | /// See remarks for the class. 78 | /// 79 | public RefVariableMetadata(Variable var, IList hiddenEntries, IList readonlyEntries) 80 | { 81 | if (var == null) 82 | throw new ArgumentNullException("var"); 83 | this.target = var; 84 | 85 | this.hiddenEntries = new List(); 86 | if (hiddenEntries != null) 87 | this.hiddenEntries.AddRange(hiddenEntries); 88 | this.hiddenEntries.Add(this.KeyForName); 89 | 90 | this.readonlyEntries = new List(); 91 | if (readonlyEntries != null) 92 | this.readonlyEntries.AddRange(readonlyEntries); 93 | 94 | // Initializing the instance 95 | target.Metadata.ForEach( 96 | delegate(KeyValuePair spair) 97 | { 98 | if (!IsHiddenEntry(spair.Key)) 99 | this[spair.Key] = spair.Value; 100 | }, SchemaVersion.Recent); 101 | } 102 | 103 | #endregion 104 | 105 | #region Source Variable Events Handling 106 | 107 | // TODO:srcM.Changing can fail after our Changing is done. 108 | // For example, if another subsciber is called after this and it cancels the update. 109 | // Then the target will have the entry, this one - doesn't. 110 | private void MetadataChanging(object sender, VariableMetadataChangingEventArgs e) 111 | { 112 | if (IsReadOnlyEntry(e.Key)) 113 | { 114 | e.Cancel = true; 115 | return; 116 | } 117 | 118 | if (!IsHiddenEntry(e.Key)) 119 | { 120 | if (proposedMetadataEntries.Exists(entry => 121 | entry.Key == e.Key && AreEquals(entry.Value, e.ProposedValue))) 122 | return; 123 | 124 | var pair = new KeyValuePair(e.Key, e.ProposedValue); 125 | try 126 | { 127 | proposedMetadataEntries.Add(pair); 128 | target.Metadata[e.Key] = e.ProposedValue; 129 | } 130 | finally 131 | { 132 | proposedMetadataEntries.Remove(pair); 133 | } 134 | } 135 | } 136 | 137 | private void TargetVariableChanging(object sender, VariableChangingEventArgs e) 138 | { 139 | if (e.Action == VariableChangeAction.UpdateMetadata) 140 | { 141 | foreach (KeyValuePair item in e.ProposedChanges.MetadataChanges) 142 | { 143 | if (IsHiddenEntry(item.Key)) continue; 144 | if (IsReadOnlyEntry(item.Key)) continue; 145 | 146 | if (proposedMetadataEntries.Exists(entry => 147 | entry.Key == item.Key && AreEquals(entry.Value, item.Value))) 148 | continue; 149 | 150 | var pair = new KeyValuePair(item.Key, item.Value); 151 | try 152 | { 153 | proposedMetadataEntries.Add(pair); 154 | this[item.Key] = item.Value; 155 | } 156 | finally 157 | { 158 | proposedMetadataEntries.Remove(pair); 159 | } 160 | } 161 | } 162 | } 163 | 164 | private bool IsHiddenEntry(string key) 165 | { 166 | return hiddenEntries.Contains(key); 167 | } 168 | 169 | private bool IsReadOnlyEntry(string key) 170 | { 171 | return readonlyEntries.Contains(key); 172 | } 173 | 174 | internal MetadataDictionary FilterChanges(MetadataDictionary metadata) 175 | { 176 | MetadataDictionary md = new MetadataDictionary(); 177 | if (metadata.Count == 0 && !metadata.HasChanges) 178 | return md; 179 | 180 | Dictionary dict = metadata.AsDictionary(SchemaVersion.Recent); 181 | foreach (var item in dict) 182 | { 183 | if (!IsHiddenEntry(item.Key)) 184 | { 185 | md[item.Key] = item.Value; 186 | } 187 | } 188 | return md; 189 | } 190 | 191 | #endregion 192 | 193 | internal void Subscribe() 194 | { 195 | this.target.Changing += new VariableChangingEventHandler(TargetVariableChanging); 196 | this.Changing += new VariableMetadataChangingEventHandler(MetadataChanging); 197 | } 198 | 199 | internal void Unsubscribe() 200 | { 201 | this.target.Changing -= new VariableChangingEventHandler(TargetVariableChanging); 202 | this.Changing -= MetadataChanging; 203 | } 204 | } 205 | } 206 | 207 | -------------------------------------------------------------------------------- /ScientificDataSet/Core/CollectionCombination.cs: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft Corporation, All Rights Reserved. 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | 7 | namespace Microsoft.Research.Science.Data 8 | { 9 | /// 10 | /// Enables smooth iteration through two collections as a single one. 11 | /// 12 | /// 13 | internal class ReadOnlyCollectionCombination : ICollection 14 | { 15 | private ICollection c1; 16 | private ICollection c2; 17 | 18 | public ReadOnlyCollectionCombination(ICollection c1, ICollection c2) 19 | { 20 | this.c1 = c1; 21 | this.c2 = c2; 22 | } 23 | 24 | #region ICollection Members 25 | 26 | public void Add(T item) 27 | { 28 | throw new NotSupportedException(); 29 | } 30 | 31 | public void Clear() 32 | { 33 | throw new NotSupportedException(); 34 | } 35 | 36 | public bool Contains(T item) 37 | { 38 | if (c1 != null && c1.Contains(item)) 39 | return true; 40 | if (c2 != null && c2.Contains(item)) 41 | return true; 42 | 43 | return false; 44 | } 45 | 46 | public void CopyTo(T[] array, int arrayIndex) 47 | { 48 | if (c1 != null) 49 | { 50 | c1.CopyTo(array, arrayIndex); 51 | arrayIndex += c1.Count; 52 | } 53 | if (c2 != null) 54 | { 55 | c2.CopyTo(array, arrayIndex); 56 | } 57 | } 58 | 59 | public int Count 60 | { 61 | get 62 | { 63 | int n = 0; 64 | if (c1 != null) n += c1.Count; 65 | if (c2 != null) n += c2.Count; 66 | return n; 67 | } 68 | } 69 | 70 | public bool IsReadOnly 71 | { 72 | get { return true; } 73 | } 74 | 75 | public bool Remove(T item) 76 | { 77 | throw new NotSupportedException(); 78 | } 79 | 80 | #endregion 81 | 82 | #region IEnumerable Members 83 | 84 | public IEnumerator GetEnumerator() 85 | { 86 | return new Enumerator(this); 87 | } 88 | 89 | #endregion 90 | 91 | #region IEnumerable Members 92 | 93 | System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() 94 | { 95 | return new Enumerator(this); 96 | } 97 | 98 | #endregion 99 | 100 | private class Enumerator : IEnumerator 101 | { 102 | private ReadOnlyCollectionCombination c; 103 | private ICollection currentCollection = null; 104 | private IEnumerator enumerator = null; 105 | private bool finish = false; 106 | 107 | internal Enumerator(ReadOnlyCollectionCombination c) 108 | { 109 | this.c = c; 110 | } 111 | 112 | #region IEnumerator Members 113 | 114 | public T Current 115 | { 116 | get 117 | { 118 | if (finish || enumerator == null) 119 | throw new Exception("Enumeration finished"); 120 | return enumerator.Current; 121 | } 122 | } 123 | 124 | #endregion 125 | 126 | #region IDisposable Members 127 | 128 | public void Dispose() 129 | { 130 | if (enumerator != null) 131 | enumerator.Dispose(); 132 | } 133 | 134 | #endregion 135 | 136 | #region IEnumerator Members 137 | 138 | object System.Collections.IEnumerator.Current 139 | { 140 | get { throw new NotImplementedException(); } 141 | } 142 | 143 | public bool MoveNext() 144 | { 145 | if (finish) return false; 146 | 147 | if (currentCollection == null) 148 | { 149 | currentCollection = c.c1; 150 | if (currentCollection == null) 151 | { 152 | currentCollection = c.c2; 153 | } 154 | if (currentCollection == null) 155 | { 156 | finish = true; 157 | return false; 158 | } 159 | 160 | enumerator = currentCollection.GetEnumerator(); 161 | } 162 | 163 | if (enumerator != null) 164 | { 165 | bool res = enumerator.MoveNext(); 166 | if (!res && currentCollection == c.c1) 167 | { 168 | currentCollection = c.c2; 169 | if (currentCollection == null) 170 | { 171 | finish = true; 172 | return false; 173 | } 174 | if (enumerator != null) 175 | enumerator.Dispose(); 176 | enumerator = currentCollection.GetEnumerator(); 177 | return MoveNext(); 178 | } 179 | if (res == false) finish = true; 180 | return res; 181 | } 182 | else throw new Exception("Enumerator is null"); 183 | } 184 | 185 | public void Reset() 186 | { 187 | throw new NotSupportedException(); 188 | } 189 | 190 | #endregion 191 | } 192 | } 193 | 194 | internal class ReadOnlyEnumerableCombination : IEnumerable 195 | { 196 | private IEnumerable c1; 197 | private IEnumerable c2; 198 | 199 | public ReadOnlyEnumerableCombination(IEnumerable c1, IEnumerable c2) 200 | { 201 | this.c1 = c1; 202 | this.c2 = c2; 203 | } 204 | 205 | #region IEnumerable Members 206 | 207 | public IEnumerator GetEnumerator() 208 | { 209 | return new Enumerator(this); 210 | } 211 | 212 | #endregion 213 | 214 | #region IEnumerable Members 215 | 216 | System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() 217 | { 218 | return new Enumerator(this); 219 | } 220 | 221 | #endregion 222 | 223 | private class Enumerator : IEnumerator 224 | { 225 | private ReadOnlyEnumerableCombination c; 226 | private IEnumerable currentCollection = null; 227 | private IEnumerator enumerator = null; 228 | private bool finish = false; 229 | 230 | internal Enumerator(ReadOnlyEnumerableCombination c) 231 | { 232 | this.c = c; 233 | } 234 | 235 | #region IEnumerator Members 236 | 237 | public T Current 238 | { 239 | get 240 | { 241 | if (finish || enumerator == null) 242 | throw new Exception("Enumeration finished"); 243 | return enumerator.Current; 244 | } 245 | } 246 | 247 | #endregion 248 | 249 | #region IDisposable Members 250 | 251 | public void Dispose() 252 | { 253 | if (enumerator != null) 254 | enumerator.Dispose(); 255 | } 256 | 257 | #endregion 258 | 259 | #region IEnumerator Members 260 | 261 | object System.Collections.IEnumerator.Current 262 | { 263 | get { throw new NotImplementedException(); } 264 | } 265 | 266 | public bool MoveNext() 267 | { 268 | if (finish) return false; 269 | 270 | if (currentCollection == null) 271 | { 272 | currentCollection = c.c1; 273 | if (currentCollection == null) 274 | { 275 | currentCollection = c.c2; 276 | } 277 | if (currentCollection == null) 278 | { 279 | finish = true; 280 | return false; 281 | } 282 | 283 | enumerator = currentCollection.GetEnumerator(); 284 | } 285 | 286 | if (enumerator != null) 287 | { 288 | bool res = enumerator.MoveNext(); 289 | if (!res && currentCollection == c.c1) 290 | { 291 | currentCollection = c.c2; 292 | if (currentCollection == null) 293 | { 294 | finish = true; 295 | return false; 296 | } 297 | if (enumerator != null) 298 | enumerator.Dispose(); 299 | enumerator = currentCollection.GetEnumerator(); 300 | return MoveNext(); 301 | } 302 | if (res == false) finish = true; 303 | return res; 304 | } 305 | else throw new Exception("Enumerator is null"); 306 | } 307 | 308 | public void Reset() 309 | { 310 | throw new NotSupportedException(); 311 | } 312 | 313 | #endregion 314 | } 315 | } 316 | } 317 | 318 | --------------------------------------------------------------------------------