├── storage
├── sql-database
│ ├── Stats.xlsx
│ ├── sample
│ │ ├── ConnectionStrings.config.SAMPLE
│ │ ├── App.config
│ │ ├── packages.config
│ │ ├── sql-database-sample.sln
│ │ ├── Properties
│ │ │ └── AssemblyInfo.cs
│ │ ├── DatasourceRecord.cs
│ │ ├── Program.cs
│ │ ├── .gitignore
│ │ ├── DatasourceRecordExtensions.cs
│ │ ├── sql-database-sample.csproj
│ │ ├── DbDataReaderSimulator.cs
│ │ └── .vs
│ │ │ └── config
│ │ │ └── applicationhost.config
│ ├── bulk-insert-performance-screenshot.jpg
│ └── sql-database.md
└── azure-data-lake
│ ├── Calculations.xlsx
│ ├── sample
│ └── CsvDataGenerator
│ │ ├── packages.config
│ │ ├── App.config
│ │ ├── Program.cs
│ │ ├── RandomDatasourceGenerator.cs
│ │ ├── CsvDataGenerator.sln
│ │ ├── DatasourceRecord.cs
│ │ ├── Properties
│ │ └── AssemblyInfo.cs
│ │ └── CsvDataGenerator.csproj
│ └── azure-data-lake.md
├── README.md
├── .gitignore
└── LICENSE
/storage/sql-database/Stats.xlsx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ytechie/time-series-data-azure-guidance/HEAD/storage/sql-database/Stats.xlsx
--------------------------------------------------------------------------------
/storage/azure-data-lake/Calculations.xlsx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ytechie/time-series-data-azure-guidance/HEAD/storage/azure-data-lake/Calculations.xlsx
--------------------------------------------------------------------------------
/storage/sql-database/sample/ConnectionStrings.config.SAMPLE:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
--------------------------------------------------------------------------------
/storage/sql-database/bulk-insert-performance-screenshot.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ytechie/time-series-data-azure-guidance/HEAD/storage/sql-database/bulk-insert-performance-screenshot.jpg
--------------------------------------------------------------------------------
/storage/azure-data-lake/sample/CsvDataGenerator/packages.config:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
--------------------------------------------------------------------------------
/storage/azure-data-lake/sample/CsvDataGenerator/App.config:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/storage/sql-database/sample/App.config:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
--------------------------------------------------------------------------------
/storage/sql-database/sample/packages.config:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/storage/azure-data-lake/sample/CsvDataGenerator/Program.cs:
--------------------------------------------------------------------------------
1 | using ServiceStack;
2 | using System.IO;
3 |
4 | namespace CsvDataGenerator
5 | {
6 | public class Program
7 | {
8 | static void Main(string[] args)
9 | {
10 | var records = RandomDatasourceRecordGenerator.GenerateDummyData(1000000);
11 | File.WriteAllText("out.csv", records.ToCsv());
12 | }
13 | }
14 | }
15 |
--------------------------------------------------------------------------------
/storage/azure-data-lake/azure-data-lake.md:
--------------------------------------------------------------------------------
1 | # Azure Data Lake for Time Series Data Storage
2 |
3 | The Azure Data Lake service consists of 2 parts:
4 |
5 | * Azure Data Lake *Store*
6 | * Azure Data Lake *Analytics*
7 |
8 | **Important Note:** Performance numbers are only to give a general idea about what performance is possible. They should not be considered official or unofficial benchmarks.
9 |
10 | ## Storage Efficiency
11 |
12 | 10,000 records in CSV is 535,430 bytes
13 |
14 | Applying Gzip compression results in a file that is 143,427 bytes. This is 26% compression ratio.
15 |
16 | A single record, after compression, is 14.3 bytes.
17 |
18 | ## Cost
19 |
20 | Note: Always check the [current Data Lake Pricing](https://azure.microsoft.com/en-us/pricing/details/data-lake-store/).
21 |
22 | The cost at the time of this writing is $.08/GB.
23 |
24 | Within 1GB, we can store ~75 million records. In other words, we can store around 75 million time series records for about 8 cents per month.
--------------------------------------------------------------------------------
/storage/azure-data-lake/sample/CsvDataGenerator/RandomDatasourceGenerator.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Linq;
4 | using System.Text;
5 | using System.Threading.Tasks;
6 |
7 | namespace CsvDataGenerator
8 | {
9 | public class RandomDatasourceRecordGenerator
10 | {
11 | private static int[] _datasources = { 1, 2, 3 };
12 | public static IEnumerable GenerateDummyData(int records)
13 | {
14 | var rand = new Random();
15 |
16 | for (var i = 0; i < records; i++)
17 | {
18 | var dr = new DatasourceRecord
19 | {
20 | DatasourceId = _datasources[rand.Next(0, 2)],
21 | IntervalSeconds = rand.Next(60, 3600),
22 | Timestamp = DateTime.UtcNow.AddMilliseconds(i * 10),
23 | Value = (rand.NextDouble() * 100000)
24 | };
25 |
26 | yield return dr;
27 | }
28 | }
29 | }
30 | }
--------------------------------------------------------------------------------
/storage/sql-database/sample/sql-database-sample.sln:
--------------------------------------------------------------------------------
1 |
2 | Microsoft Visual Studio Solution File, Format Version 12.00
3 | # Visual Studio 14
4 | VisualStudioVersion = 14.0.24720.0
5 | MinimumVisualStudioVersion = 10.0.40219.1
6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "sql-database-sample", "sql-database-sample.csproj", "{47404295-345F-43AE-9660-5DE653D2CEC9}"
7 | EndProject
8 | Global
9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution
10 | Debug|Any CPU = Debug|Any CPU
11 | Release|Any CPU = Release|Any CPU
12 | EndGlobalSection
13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution
14 | {47404295-345F-43AE-9660-5DE653D2CEC9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
15 | {47404295-345F-43AE-9660-5DE653D2CEC9}.Debug|Any CPU.Build.0 = Debug|Any CPU
16 | {47404295-345F-43AE-9660-5DE653D2CEC9}.Release|Any CPU.ActiveCfg = Release|Any CPU
17 | {47404295-345F-43AE-9660-5DE653D2CEC9}.Release|Any CPU.Build.0 = Release|Any CPU
18 | EndGlobalSection
19 | GlobalSection(SolutionProperties) = preSolution
20 | HideSolutionNode = FALSE
21 | EndGlobalSection
22 | EndGlobal
23 |
--------------------------------------------------------------------------------
/storage/azure-data-lake/sample/CsvDataGenerator/CsvDataGenerator.sln:
--------------------------------------------------------------------------------
1 |
2 | Microsoft Visual Studio Solution File, Format Version 12.00
3 | # Visual Studio 15
4 | VisualStudioVersion = 15.0.25807.0
5 | MinimumVisualStudioVersion = 10.0.40219.1
6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CsvDataGenerator", "CsvDataGenerator.csproj", "{FD7EA3BD-8FBA-4F2B-9327-DFAC5B0A059C}"
7 | EndProject
8 | Global
9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution
10 | Debug|Any CPU = Debug|Any CPU
11 | Release|Any CPU = Release|Any CPU
12 | EndGlobalSection
13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution
14 | {FD7EA3BD-8FBA-4F2B-9327-DFAC5B0A059C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
15 | {FD7EA3BD-8FBA-4F2B-9327-DFAC5B0A059C}.Debug|Any CPU.Build.0 = Debug|Any CPU
16 | {FD7EA3BD-8FBA-4F2B-9327-DFAC5B0A059C}.Release|Any CPU.ActiveCfg = Release|Any CPU
17 | {FD7EA3BD-8FBA-4F2B-9327-DFAC5B0A059C}.Release|Any CPU.Build.0 = Release|Any CPU
18 | EndGlobalSection
19 | GlobalSection(SolutionProperties) = preSolution
20 | HideSolutionNode = FALSE
21 | EndGlobalSection
22 | EndGlobal
23 |
--------------------------------------------------------------------------------
/storage/azure-data-lake/sample/CsvDataGenerator/DatasourceRecord.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Linq;
3 |
4 | namespace CsvDataGenerator
5 | {
6 | //copied from https://github.com/ytechie/Manufacturing.Framework
7 |
8 | public class DatasourceRecord
9 | {
10 | public int DatasourceId { get; set; }
11 | public DateTime Timestamp { get; set; }
12 | public int IntervalSeconds { get; set; } //TODO: make more granular/generic?
13 | public double Value { get; set; }
14 |
15 | public virtual bool Equivalent(DatasourceRecord compare)
16 | {
17 | if (compare == null)
18 | return false;
19 | if (this == compare)
20 | return true;
21 |
22 | if (DatasourceId != compare.DatasourceId)
23 | return false;
24 | if (Timestamp != compare.Timestamp)
25 | return false;
26 | if (IntervalSeconds != compare.IntervalSeconds)
27 | return false;
28 | if (Value == compare.Value)
29 | return false;
30 |
31 | return true;
32 | }
33 |
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/storage/sql-database/sample/Properties/AssemblyInfo.cs:
--------------------------------------------------------------------------------
1 | using System.Reflection;
2 | using System.Runtime.CompilerServices;
3 | using System.Runtime.InteropServices;
4 |
5 | // General Information about an assembly is controlled through the following
6 | // set of attributes. Change these attribute values to modify the information
7 | // associated with an assembly.
8 | [assembly: AssemblyTitle("sql-database-sample")]
9 | [assembly: AssemblyDescription("")]
10 | [assembly: AssemblyConfiguration("")]
11 | [assembly: AssemblyCompany("")]
12 | [assembly: AssemblyProduct("sql-database-sample")]
13 | [assembly: AssemblyCopyright("Copyright © 2016")]
14 | [assembly: AssemblyTrademark("")]
15 | [assembly: AssemblyCulture("")]
16 |
17 | // Setting ComVisible to false makes the types in this assembly not visible
18 | // to COM components. If you need to access a type in this assembly from
19 | // COM, set the ComVisible attribute to true on that type.
20 | [assembly: ComVisible(false)]
21 |
22 | // The following GUID is for the ID of the typelib if this project is exposed to COM
23 | [assembly: Guid("47404295-345f-43ae-9660-5de653d2cec9")]
24 |
25 | // Version information for an assembly consists of the following four values:
26 | //
27 | // Major Version
28 | // Minor Version
29 | // Build Number
30 | // Revision
31 | //
32 | // You can specify all the values or you can default the Build and Revision Numbers
33 | // by using the '*' as shown below:
34 | // [assembly: AssemblyVersion("1.0.*")]
35 | [assembly: AssemblyVersion("1.0.0.0")]
36 | [assembly: AssemblyFileVersion("1.0.0.0")]
37 |
--------------------------------------------------------------------------------
/storage/azure-data-lake/sample/CsvDataGenerator/Properties/AssemblyInfo.cs:
--------------------------------------------------------------------------------
1 | using System.Reflection;
2 | using System.Runtime.CompilerServices;
3 | using System.Runtime.InteropServices;
4 |
5 | // General Information about an assembly is controlled through the following
6 | // set of attributes. Change these attribute values to modify the information
7 | // associated with an assembly.
8 | [assembly: AssemblyTitle("CsvDataGenerator")]
9 | [assembly: AssemblyDescription("")]
10 | [assembly: AssemblyConfiguration("")]
11 | [assembly: AssemblyCompany("")]
12 | [assembly: AssemblyProduct("CsvDataGenerator")]
13 | [assembly: AssemblyCopyright("Copyright © 2016")]
14 | [assembly: AssemblyTrademark("")]
15 | [assembly: AssemblyCulture("")]
16 |
17 | // Setting ComVisible to false makes the types in this assembly not visible
18 | // to COM components. If you need to access a type in this assembly from
19 | // COM, set the ComVisible attribute to true on that type.
20 | [assembly: ComVisible(false)]
21 |
22 | // The following GUID is for the ID of the typelib if this project is exposed to COM
23 | [assembly: Guid("fd7ea3bd-8fba-4f2b-9327-dfac5b0a059c")]
24 |
25 | // Version information for an assembly consists of the following four values:
26 | //
27 | // Major Version
28 | // Minor Version
29 | // Build Number
30 | // Revision
31 | //
32 | // You can specify all the values or you can default the Build and Revision Numbers
33 | // by using the '*' as shown below:
34 | // [assembly: AssemblyVersion("1.0.*")]
35 | [assembly: AssemblyVersion("1.0.0.0")]
36 | [assembly: AssemblyFileVersion("1.0.0.0")]
37 |
--------------------------------------------------------------------------------
/storage/sql-database/sample/DatasourceRecord.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Linq;
3 |
4 | namespace sql_database_sample
5 | {
6 | //copied from https://github.com/ytechie/Manufacturing.Framework
7 |
8 | public class DatasourceRecord
9 | {
10 | public int DatasourceId { get; set; }
11 | public DateTime Timestamp { get; set; }
12 | public int IntervalSeconds { get; set; } //TODO: make more granular/generic?
13 | public byte[] Value { get; set; }
14 |
15 | public int EncodedDataType
16 | {
17 | get
18 | {
19 | return (int)DataType;
20 | }
21 | set
22 | {
23 | DataType = (DataTypeEnum)value;
24 | }
25 | }
26 |
27 | public DataTypeEnum DataType { get; set; }
28 |
29 | public enum DataTypeEnum
30 | {
31 | Undefined = 0,
32 | Integer = 1,
33 | UnsignedFloat = 2,
34 | Double = 3,
35 | DateTime = 4,
36 | String = 5,
37 | Boolean = 6,
38 | Decimal = 7
39 | }
40 |
41 | public virtual bool Equivalent(DatasourceRecord compare)
42 | {
43 | if (compare == null)
44 | return false;
45 | if (this == compare)
46 | return true;
47 |
48 | if (DatasourceId != compare.DatasourceId)
49 | return false;
50 | if (Timestamp != compare.Timestamp)
51 | return false;
52 | if (IntervalSeconds != compare.IntervalSeconds)
53 | return false;
54 | if (Value == null ^ compare.Value == null || (Value != null && !Value.SequenceEqual(compare.Value)))
55 | return false;
56 | if (EncodedDataType != compare.EncodedDataType)
57 | return false;
58 |
59 | return true;
60 | }
61 |
62 | }
63 | }
64 |
--------------------------------------------------------------------------------
/storage/sql-database/sample/Program.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Configuration;
3 | using System.Data.SqlClient;
4 | using System.Diagnostics;
5 |
6 | namespace sql_database_sample
7 | {
8 | class Program
9 | {
10 | static void Main()
11 | {
12 | string connectionString = GetConnectionString();
13 |
14 | using (var connection = new SqlConnection(connectionString))
15 | {
16 | connection.Open();
17 |
18 | var simulator = new DbDataReaderSimulator(1, DatasourceRecord.DataTypeEnum.Double);
19 |
20 | using (SqlBulkCopy bulkCopy = new SqlBulkCopy(connection))
21 | {
22 | var totalSw = new Stopwatch();
23 | var sw = new Stopwatch();
24 |
25 | var totalInserted = 0;
26 |
27 | bulkCopy.DestinationTableName = "dbo.Data";
28 | bulkCopy.BatchSize = 5000;
29 | bulkCopy.NotifyAfter = 10000;
30 | bulkCopy.EnableStreaming = false;
31 |
32 | bulkCopy.SqlRowsCopied += (s, e) => {
33 | totalInserted += bulkCopy.NotifyAfter;
34 |
35 | sw.Stop();
36 | Console.WriteLine("({0}) {1} Records Inserted in {2}ms ({3:N0} aggregate rps)", e.RowsCopied, bulkCopy.NotifyAfter, sw.ElapsedMilliseconds, totalInserted / totalSw.Elapsed.TotalSeconds);
37 | sw.Restart();
38 | };
39 |
40 | try
41 | {
42 | sw.Start();
43 | totalSw.Start();
44 | // Write from the source to the destination.
45 | bulkCopy.WriteToServer(simulator);
46 | }
47 | catch (Exception ex)
48 | {
49 | Console.WriteLine(ex.Message);
50 | }
51 | }
52 |
53 |
54 | Console.WriteLine("Press Enter to exit");
55 | Console.ReadLine();
56 | }
57 | }
58 |
59 | private static string GetConnectionString()
60 | {
61 | return ConfigurationManager.ConnectionStrings["TimeSeriesDatabase"].ConnectionString;
62 | }
63 | }
64 | }
65 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Guidance for storing time series data in Azure
2 |
3 | The goal of this project is to assess basic ways of storing time series data in Azure.
4 |
5 | ## Should I be using an existing solution?
6 |
7 | There are a number of companies that specialize in cloud-based time series data historians. For projects with extremely large storage requirements, high-performance needs, or advanced features such as model-based compression or interpolation, it's advisable to look at the existing products on the market.
8 |
9 | These third-party solutions may be much easier to implement versus creating an in-house solution.
10 |
11 | ## High-Level Comparison of Popular Storage Approaches
12 |
13 | Click on a storage solution header for details.
14 |
15 |
16 |
17 | |
18 | SQL Database |
19 | Blob Storage |
20 | HBase |
21 | Azure Data Lake |
22 | OpenTSDB |
23 |
24 |
25 | | Writes |
26 | Excellent |
27 | |
28 | |
29 | Excellent |
30 | |
31 |
32 |
33 | | Random Access Reads |
34 | Excellent |
35 | Poor |
36 | |
37 | Poor |
38 | |
39 |
40 |
41 | | Batch Reads |
42 | Excellent |
43 | Good |
44 | |
45 | Excellent |
46 | |
47 |
48 |
49 | | Aggregation |
50 | Good (TSQL) |
51 | Poor |
52 | |
53 | Excellent |
54 | |
55 |
56 |
57 | | Storage Efficiency |
58 | Excellent |
59 | Excellent |
60 | |
61 | Excellent |
62 | |
63 |
64 |
65 | | Cost |
66 | Configurable |
67 | |
68 | |
69 | Excellent |
70 | |
71 |
72 |
73 | | Interpolation |
74 | |
75 | |
76 | |
77 | Excellent |
78 | |
79 |
80 |
81 | | Interoperability |
82 | Excellent |
83 | |
84 | |
85 | Poor |
86 | |
87 |
88 |
89 |
90 | ## Storage Solutions Not Tested
91 |
92 | * DocumentDB
93 |
--------------------------------------------------------------------------------
/storage/azure-data-lake/sample/CsvDataGenerator/CsvDataGenerator.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Debug
6 | AnyCPU
7 | {FD7EA3BD-8FBA-4F2B-9327-DFAC5B0A059C}
8 | Exe
9 | Properties
10 | CsvDataGenerator
11 | CsvDataGenerator
12 | v4.5.2
13 | 512
14 | true
15 |
16 |
17 | AnyCPU
18 | true
19 | full
20 | false
21 | bin\Debug\
22 | DEBUG;TRACE
23 | prompt
24 | 4
25 |
26 |
27 | AnyCPU
28 | pdbonly
29 | true
30 | bin\Release\
31 | TRACE
32 | prompt
33 | 4
34 |
35 |
36 |
37 | packages\ServiceStack.Text.4.5.0\lib\net45\ServiceStack.Text.dll
38 | True
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
67 |
--------------------------------------------------------------------------------
/storage/sql-database/sample/.gitignore:
--------------------------------------------------------------------------------
1 | ## Ignore Visual Studio temporary files, build results, and
2 | ## files generated by popular Visual Studio add-ons.
3 |
4 | # User-specific files
5 | *.suo
6 | *.user
7 | *.sln.docstates
8 |
9 | # Build results
10 | [Dd]ebug/
11 | [Dd]ebugPublic/
12 | [Rr]elease/
13 | x64/
14 | build/
15 | bld/
16 | [Bb]in/
17 | [Oo]bj/
18 |
19 | # MSTest test Results
20 | [Tt]est[Rr]esult*/
21 | [Bb]uild[Ll]og.*
22 |
23 | #NUNIT
24 | *.VisualState.xml
25 | TestResult.xml
26 |
27 | # Build Results of an ATL Project
28 | [Dd]ebugPS/
29 | [Rr]eleasePS/
30 | dlldata.c
31 |
32 | *_i.c
33 | *_p.c
34 | *_i.h
35 | *.ilk
36 | *.meta
37 | *.obj
38 | *.pch
39 | *.pdb
40 | *.pgc
41 | *.pgd
42 | *.rsp
43 | *.sbr
44 | *.tlb
45 | *.tli
46 | *.tlh
47 | *.tmp
48 | *.tmp_proj
49 | *.log
50 | *.vspscc
51 | *.vssscc
52 | .builds
53 | *.pidb
54 | *.svclog
55 | *.scc
56 |
57 | # Chutzpah Test files
58 | _Chutzpah*
59 |
60 | # Visual C++ cache files
61 | ipch/
62 | *.aps
63 | *.ncb
64 | *.opensdf
65 | *.sdf
66 | *.cachefile
67 |
68 | # Visual Studio profiler
69 | *.psess
70 | *.vsp
71 | *.vspx
72 |
73 | # TFS 2012 Local Workspace
74 | $tf/
75 |
76 | # Guidance Automation Toolkit
77 | *.gpState
78 |
79 | # ReSharper is a .NET coding add-in
80 | _ReSharper*/
81 | *.[Rr]e[Ss]harper
82 | *.DotSettings.user
83 |
84 | # JustCode is a .NET coding addin-in
85 | .JustCode
86 |
87 | # TeamCity is a build add-in
88 | _TeamCity*
89 |
90 | # DotCover is a Code Coverage Tool
91 | *.dotCover
92 |
93 | # NCrunch
94 | *.ncrunch*
95 | _NCrunch_*
96 | .*crunch*.local.xml
97 |
98 | # MightyMoose
99 | *.mm.*
100 | AutoTest.Net/
101 |
102 | # Web workbench (sass)
103 | .sass-cache/
104 |
105 | # Installshield output folder
106 | [Ee]xpress/
107 |
108 | # DocProject is a documentation generator add-in
109 | DocProject/buildhelp/
110 | DocProject/Help/*.HxT
111 | DocProject/Help/*.HxC
112 | DocProject/Help/*.hhc
113 | DocProject/Help/*.hhk
114 | DocProject/Help/*.hhp
115 | DocProject/Help/Html2
116 | DocProject/Help/html
117 |
118 | # Click-Once directory
119 | publish/
120 |
121 | # Publish Web Output
122 | *.[Pp]ublish.xml
123 | *.azurePubxml
124 |
125 | # NuGet Packages Directory
126 | packages/
127 | ## TODO: If the tool you use requires repositories.config uncomment the next line
128 | #!packages/repositories.config
129 |
130 | # Enable "build/" folder in the NuGet Packages folder since NuGet packages use it for MSBuild targets
131 | # This line needs to be after the ignore of the build folder (and the packages folder if the line above has been uncommented)
132 | !packages/build/
133 |
134 | # Windows Azure Build Output
135 | csx/
136 | *.build.csdef
137 |
138 | # Windows Store app package directory
139 | AppPackages/
140 |
141 | # Others
142 | sql/
143 | *.Cache
144 | ClientBin/
145 | [Ss]tyle[Cc]op.*
146 | ~$*
147 | *~
148 | *.dbmdl
149 | *.dbproj.schemaview
150 | *.pfx
151 | *.publishsettings
152 | node_modules/
153 |
154 | # RIA/Silverlight projects
155 | Generated_Code/
156 |
157 | # Backup & report files from converting an old project file to a newer
158 | # Visual Studio version. Backup files are not needed, because we have git ;-)
159 | _UpgradeReport_Files/
160 | Backup*/
161 | UpgradeLog*.XML
162 | UpgradeLog*.htm
163 |
164 | # SQL Server files
165 | *.mdf
166 | *.ldf
167 |
168 | # Business Intelligence projects
169 | *.rdl.data
170 | *.bim.layout
171 | *.bim_*.settings
172 |
173 | # Microsoft Fakes
174 | FakesAssemblies/
175 |
176 | *.codegen.cs
177 |
178 | ConnectionStrings.config
--------------------------------------------------------------------------------
/storage/sql-database/sample/DatasourceRecordExtensions.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.IO;
3 | using System.Text;
4 |
5 | namespace sql_database_sample
6 | {
7 | public static class DatasourceRecordExtensions
8 | {
9 | public static decimal GetDecimalValue(this DatasourceRecord record)
10 | {
11 | ThrowExceptionOnInvalidConversion(record.DataType, DatasourceRecord.DataTypeEnum.Decimal);
12 |
13 | using (var memoryStream = new MemoryStream(record.Value))
14 | {
15 | using (var binaryReader = new BinaryReader(memoryStream))
16 | {
17 | return binaryReader.ReadDecimal();
18 | }
19 | }
20 | }
21 |
22 | public static void SetDecimalValue(this DatasourceRecord record, decimal value)
23 | {
24 | ThrowExceptionOnInvalidConversion(record.DataType, DatasourceRecord.DataTypeEnum.Decimal);
25 |
26 | using (var memoryStream = new MemoryStream())
27 | {
28 | using (var binaryWriter = new BinaryWriter(memoryStream))
29 | {
30 | binaryWriter.Write(value);
31 | record.Value = memoryStream.ToArray();
32 | }
33 | }
34 | record.DataType = DatasourceRecord.DataTypeEnum.Decimal;
35 | }
36 |
37 | public static int GetIntValue(this DatasourceRecord record)
38 | {
39 | ThrowExceptionOnInvalidConversion(record.DataType, DatasourceRecord.DataTypeEnum.Integer);
40 |
41 | return BitConverter.ToInt32(record.Value, 0);
42 | }
43 |
44 | public static void SetIntValue(this DatasourceRecord record, int value)
45 | {
46 | ThrowExceptionOnInvalidConversion(record.DataType, DatasourceRecord.DataTypeEnum.Integer);
47 |
48 | record.Value = BitConverter.GetBytes(value);
49 | record.DataType = DatasourceRecord.DataTypeEnum.Integer;
50 | }
51 |
52 | public static double GetDoubleValue(this DatasourceRecord record)
53 | {
54 | ThrowExceptionOnInvalidConversion(record.DataType, DatasourceRecord.DataTypeEnum.Double);
55 |
56 | return BitConverter.ToDouble(record.Value, 0);
57 | }
58 |
59 | public static void SetDoubleValue(this DatasourceRecord record, double value)
60 | {
61 | ThrowExceptionOnInvalidConversion(record.DataType, DatasourceRecord.DataTypeEnum.Double);
62 |
63 | record.Value = BitConverter.GetBytes(value);
64 | record.DataType = DatasourceRecord.DataTypeEnum.Double;
65 | }
66 |
67 | public static string GetStringValue(this DatasourceRecord record)
68 | {
69 | ThrowExceptionOnInvalidConversion(record.DataType, DatasourceRecord.DataTypeEnum.String);
70 |
71 | return Encoding.UTF8.GetString(record.Value);
72 | }
73 |
74 | public static void SetStringValue(this DatasourceRecord record, string value)
75 | {
76 | ThrowExceptionOnInvalidConversion(record.DataType, DatasourceRecord.DataTypeEnum.String);
77 |
78 | record.Value = Encoding.UTF8.GetBytes(value);
79 | record.DataType = DatasourceRecord.DataTypeEnum.String;
80 | }
81 |
82 | private static void ThrowExceptionOnInvalidConversion(DatasourceRecord.DataTypeEnum from,
83 | DatasourceRecord.DataTypeEnum to)
84 | {
85 | if (!(from == DatasourceRecord.DataTypeEnum.Undefined || from == to))
86 | throw new NotSupportedException();
87 | }
88 | }
89 | }
--------------------------------------------------------------------------------
/storage/sql-database/sample/sql-database-sample.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Debug
6 | AnyCPU
7 | {47404295-345F-43AE-9660-5DE653D2CEC9}
8 | Exe
9 | Properties
10 | sql_database_sample
11 | sql-database-sample
12 | v4.6.1
13 | 512
14 | true
15 |
16 |
17 | AnyCPU
18 | true
19 | full
20 | false
21 | bin\Debug\
22 | DEBUG;TRACE
23 | prompt
24 | 4
25 |
26 |
27 | AnyCPU
28 | pdbonly
29 | true
30 | bin\Release\
31 | TRACE
32 | prompt
33 | 4
34 |
35 |
36 |
37 | packages\log4net.2.0.3\lib\net40-full\log4net.dll
38 | True
39 |
40 |
41 | packages\Newtonsoft.Json.6.0.4\lib\net45\Newtonsoft.Json.dll
42 | True
43 |
44 |
45 | packages\structuremap.3.1.0.133\lib\net40\StructureMap.dll
46 | True
47 |
48 |
49 | packages\structuremap.3.1.0.133\lib\net40\StructureMap.Net4.dll
50 | True
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 | PreserveNewest
73 |
74 |
75 |
76 |
77 |
78 |
85 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | ## Ignore Visual Studio temporary files, build results, and
2 | ## files generated by popular Visual Studio add-ons.
3 |
4 | # User-specific files
5 | *.suo
6 | *.user
7 | *.userosscache
8 | *.sln.docstates
9 |
10 | # User-specific files (MonoDevelop/Xamarin Studio)
11 | *.userprefs
12 |
13 | # Build results
14 | [Dd]ebug/
15 | [Dd]ebugPublic/
16 | [Rr]elease/
17 | [Rr]eleases/
18 | x64/
19 | x86/
20 | bld/
21 | [Bb]in/
22 | [Oo]bj/
23 | [Ll]og/
24 |
25 | # Visual Studio 2015 cache/options directory
26 | .vs/
27 | # Uncomment if you have tasks that create the project's static files in wwwroot
28 | #wwwroot/
29 |
30 | # MSTest test Results
31 | [Tt]est[Rr]esult*/
32 | [Bb]uild[Ll]og.*
33 |
34 | # NUNIT
35 | *.VisualState.xml
36 | TestResult.xml
37 |
38 | # Build Results of an ATL Project
39 | [Dd]ebugPS/
40 | [Rr]eleasePS/
41 | dlldata.c
42 |
43 | # DNX
44 | project.lock.json
45 | artifacts/
46 |
47 | *_i.c
48 | *_p.c
49 | *_i.h
50 | *.ilk
51 | *.meta
52 | *.obj
53 | *.pch
54 | *.pdb
55 | *.pgc
56 | *.pgd
57 | *.rsp
58 | *.sbr
59 | *.tlb
60 | *.tli
61 | *.tlh
62 | *.tmp
63 | *.tmp_proj
64 | *.log
65 | *.vspscc
66 | *.vssscc
67 | .builds
68 | *.pidb
69 | *.svclog
70 | *.scc
71 |
72 | # Chutzpah Test files
73 | _Chutzpah*
74 |
75 | # Visual C++ cache files
76 | ipch/
77 | *.aps
78 | *.ncb
79 | *.opendb
80 | *.opensdf
81 | *.sdf
82 | *.cachefile
83 | *.VC.db
84 | *.VC.VC.opendb
85 |
86 | # Visual Studio profiler
87 | *.psess
88 | *.vsp
89 | *.vspx
90 | *.sap
91 |
92 | # TFS 2012 Local Workspace
93 | $tf/
94 |
95 | # Guidance Automation Toolkit
96 | *.gpState
97 |
98 | # ReSharper is a .NET coding add-in
99 | _ReSharper*/
100 | *.[Rr]e[Ss]harper
101 | *.DotSettings.user
102 |
103 | # JustCode is a .NET coding add-in
104 | .JustCode
105 |
106 | # TeamCity is a build add-in
107 | _TeamCity*
108 |
109 | # DotCover is a Code Coverage Tool
110 | *.dotCover
111 |
112 | # NCrunch
113 | _NCrunch_*
114 | .*crunch*.local.xml
115 | nCrunchTemp_*
116 |
117 | # MightyMoose
118 | *.mm.*
119 | AutoTest.Net/
120 |
121 | # Web workbench (sass)
122 | .sass-cache/
123 |
124 | # Installshield output folder
125 | [Ee]xpress/
126 |
127 | # DocProject is a documentation generator add-in
128 | DocProject/buildhelp/
129 | DocProject/Help/*.HxT
130 | DocProject/Help/*.HxC
131 | DocProject/Help/*.hhc
132 | DocProject/Help/*.hhk
133 | DocProject/Help/*.hhp
134 | DocProject/Help/Html2
135 | DocProject/Help/html
136 |
137 | # Click-Once directory
138 | publish/
139 |
140 | # Publish Web Output
141 | *.[Pp]ublish.xml
142 | *.azurePubxml
143 | # TODO: Comment the next line if you want to checkin your web deploy settings
144 | # but database connection strings (with potential passwords) will be unencrypted
145 | *.pubxml
146 | *.publishproj
147 |
148 | # Microsoft Azure Web App publish settings. Comment the next line if you want to
149 | # checkin your Azure Web App publish settings, but sensitive information contained
150 | # in these scripts will be unencrypted
151 | PublishScripts/
152 |
153 | # NuGet Packages
154 | *.nupkg
155 | # The packages folder can be ignored because of Package Restore
156 | **/packages/*
157 | # except build/, which is used as an MSBuild target.
158 | !**/packages/build/
159 | # Uncomment if necessary however generally it will be regenerated when needed
160 | #!**/packages/repositories.config
161 | # NuGet v3's project.json files produces more ignoreable files
162 | *.nuget.props
163 | *.nuget.targets
164 |
165 | # Microsoft Azure Build Output
166 | csx/
167 | *.build.csdef
168 |
169 | # Microsoft Azure Emulator
170 | ecf/
171 | rcf/
172 |
173 | # Windows Store app package directories and files
174 | AppPackages/
175 | BundleArtifacts/
176 | Package.StoreAssociation.xml
177 | _pkginfo.txt
178 |
179 | # Visual Studio cache files
180 | # files ending in .cache can be ignored
181 | *.[Cc]ache
182 | # but keep track of directories ending in .cache
183 | !*.[Cc]ache/
184 |
185 | # Others
186 | ClientBin/
187 | ~$*
188 | *~
189 | *.dbmdl
190 | *.dbproj.schemaview
191 | *.pfx
192 | *.publishsettings
193 | node_modules/
194 | orleans.codegen.cs
195 |
196 | # Since there are multiple workflows, uncomment next line to ignore bower_components
197 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
198 | #bower_components/
199 |
200 | # RIA/Silverlight projects
201 | Generated_Code/
202 |
203 | # Backup & report files from converting an old project file
204 | # to a newer Visual Studio version. Backup files are not needed,
205 | # because we have git ;-)
206 | _UpgradeReport_Files/
207 | Backup*/
208 | UpgradeLog*.XML
209 | UpgradeLog*.htm
210 |
211 | # SQL Server files
212 | *.mdf
213 | *.ldf
214 |
215 | # Business Intelligence projects
216 | *.rdl.data
217 | *.bim.layout
218 | *.bim_*.settings
219 |
220 | # Microsoft Fakes
221 | FakesAssemblies/
222 |
223 | # GhostDoc plugin setting file
224 | *.GhostDoc.xml
225 |
226 | # Node.js Tools for Visual Studio
227 | .ntvs_analysis.dat
228 |
229 | # Visual Studio 6 build log
230 | *.plg
231 |
232 | # Visual Studio 6 workspace options file
233 | *.opt
234 |
235 | # Visual Studio LightSwitch build output
236 | **/*.HTMLClient/GeneratedArtifacts
237 | **/*.DesktopClient/GeneratedArtifacts
238 | **/*.DesktopClient/ModelManifest.xml
239 | **/*.Server/GeneratedArtifacts
240 | **/*.Server/ModelManifest.xml
241 | _Pvt_Extensions
242 |
243 | # Paket dependency manager
244 | .paket/paket.exe
245 | paket-files/
246 |
247 | # FAKE - F# Make
248 | .fake/
249 |
250 | # JetBrains Rider
251 | .idea/
252 | *.sln.iml
253 |
--------------------------------------------------------------------------------
/storage/sql-database/sql-database.md:
--------------------------------------------------------------------------------
1 | # SQL Server for Time Series Data Storage
2 |
3 | Let's explore the possibility of using SQL Database for the storage of time series data. SQL Database is a relational database-as-a-service that is entirely managed for you. It offers features such as predictable performance that you can dial up or down, high availability, data protection options such as restore and geo-replication. It's also fully compatible with most existing applications that are using SQL Server, so it plays a vital role in applications being migrated to the cloud.
4 |
5 | Since SQL Server is already part of many architectures, it makes sense to evaluate its effectiveness for basic time series data storage. It's maturity and obiquity makes it an interesting option.
6 |
7 | **Important Note:** Performance numbers are only to give a general idea about what performance is possible. They should not be considered official or unofficial benchmarks.
8 |
9 | ## Running the Sample
10 |
11 | 1. Create a V12 SQL Database in Azure
12 | 1. Clone this repository
13 | 1. Open the sample `/storage/sql-database/sample` in Visual Studio 2015
14 | 1. Copy the `ConnectionStrings.config.SAMPLE` to `ConnectionStrings.config`
15 | 1. Change the connection string in `ConnectionStrings.config`
16 | 1. Run it - data will start being bulk inserted into your database
17 |
18 | ## Write Performance
19 |
20 | How do we get data quickly into SQL Database? In order to see what was possible, I created a sample console application in .NET to bulk insert records. This sample uses a custom [DbDataReader](https://msdn.microsoft.com/en-us/library/system.data.common.dbdatareader(v=vs.110).aspx) class to generate random time-series values.
21 |
22 | 
23 |
24 | With the lowest end SQL Database, I was able to insert about 4,000 records per second consistently. I repeated this test over the internet, and within the same resource group as the database. Locality didn't affect throughput, although it likely affected latency. Using a larger, P0 instance, I was able to insert about 10,000 records each second. Either of these numbers should be acceptable for the majority of scenarios.
25 |
26 |
27 | ## Random Access Performance
28 |
29 | *Coming Soon*
30 |
31 | ## Batch Reads
32 |
33 | *Coming Soon*
34 |
35 | ## Aggregation
36 |
37 | *Coming Soon*
38 |
39 | ## Storage Efficiency
40 |
41 | SQL Database and SQL Server both use a schema-based relational approach to storing data. When contrasted with schema-less approaches, this is far more efficient. There is no need to store column metadata with each record. Additionally, looking up metadata about the datasource itself is a fast operation, so it's easy to separate (normalize) the information about the data source.
42 |
43 | Here is one possible table configuration:
44 |
45 | CREATE TABLE [dbo].[Data](
46 | [Timestamp] [datetime] NOT NULL,
47 | [DatsourceId] [int] NOT NULL,
48 | [Value] [varbinary](max) NOT NULL
49 |
50 | We're using the `DatasourceId` as a lookup for the datasource details. We left the datasource metadata table out of the sample.
51 |
52 | The `Value` column is of type `varbinary`. This column type favors flexibility over efficiency. It allows the storage of any possible type of data. Later, we'll run tests with a more specific data type.
53 |
54 | SQL Database supports [page level compression](https://msdn.microsoft.com/en-us/library/cc280464.aspx). This is a form of compression that will compress data across multiple rows. The repetitiveness of rows provides a very high level of compression.
55 |
56 | Turning on compression is simple:
57 |
58 | ALTER TABLE Dbo.Data REBUILD PARTITION = ALL
59 | WITH (DATA_COMPRESSION = PAGE);
60 |
61 | To determine storage requirements, we use the simple formula of `table size / number of rows`.
62 |
63 | To calculate the space used by the table and the number of rows: `sp_spaceused Data`
64 |
65 | With compression on, we're able to store 10 million rows in approximately 375,000 KB. When we factor in the index size of 455,000 KB, our average record size is **81 bytes**.
66 |
67 | If a specific data type is used, for example using `float` instead of `varbinary(max)`, our storage efficiency doubles. Each record can be stored in **~40 bytes**.
68 |
69 |
70 |
71 | | Level |
72 | DTUs |
73 | Space (GB) |
74 | Records varbinary(max) |
75 | Records float |
76 | Approx. Insert Rate |
77 |
78 |
79 | | S0 |
80 | 15 |
81 | 250 |
82 | 3,318,702,167 |
83 | 6,590,172,489 |
84 | 4,000 |
85 |
86 |
87 | | P1 |
88 | 125 |
89 | 500 |
90 | 6,637,404,334 |
91 | 13,180,344,978 |
92 | 9,900 |
93 |
94 |
95 |
96 | In summary, we're able to store billions of **indexed** records with even the most inexpensive SQL Database option.
97 |
98 | ## In-Memory Storage
99 |
100 | The premium tiers of the V12 SQL Database in Azure support in-memory databases. Here is an example create statement for an in-memory time series table:
101 |
102 | CREATE TABLE [dbo].[Data_InMem]
103 | (
104 | [Timestamp] [datetime] NOT NULL,
105 | [DatsourceId] [int] NOT NULL,
106 | [Value] [varchar](1000) COLLATE SQL_Latin1_General_CP1_CI_AS NOT NULL,
107 | CONSTRAINT [Data_InMem_primaryKey] PRIMARY KEY NONCLUSTERED HASH
108 | (
109 | [Timestamp]
110 | ) WITH ( BUCKET_COUNT = 131072)
111 | ) WITH ( MEMORY_OPTIMIZED = ON , DURABILITY = SCHEMA_AND_DATA )
112 |
113 | A benefit analysis for in-mem is in progress. Stay tuned.
114 |
115 | ## Cost
116 |
117 | Current pricing can be found [here on the Azure website](https://azure.microsoft.com/en-us/pricing/details/sql-database/?b=16.50).
118 |
119 | Using the S0 option, pricing can be around $15 per month. It's a simple operation if the database needs to be scaled up. In my testing, I scaled from an S0 to a P1. The scaling operation breaks the connection, so be sure to implement a [transient fault handling solution](https://msdn.microsoft.com/en-us/library/hh680934%28v=pandp.50%29.aspx).
120 |
121 | Thanks to the high storage efficiency of SQL Database (see above), we can choose an edition of SQL Database based on our compute requirements and not storage requirements.
122 |
123 | ## Interpolation
124 |
125 | *Coming Soon*
126 |
127 | ## Interoperability
128 |
129 | SQL Database is *great* for compatibility with other products. Many products have direct support, including Azure Stream Analytics, Azure Machine Learning, PowerBI, and more.
130 |
131 | ## High Availability & Disaster Recovery
132 |
133 | There is [extensive documentation available](https://azure.microsoft.com/en-us/documentation/articles/sql-database-business-continuity/) on the Azure documentation site.
134 |
--------------------------------------------------------------------------------
/storage/sql-database/sample/DbDataReaderSimulator.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections;
3 | using System.Collections.Generic;
4 | using System.Data.Common;
5 | using System.Linq;
6 | using System.Text;
7 | using System.Threading.Tasks;
8 |
9 | namespace sql_database_sample
10 | {
11 | class DbDataReaderSimulator : DbDataReader
12 | {
13 | private DatasourceRecord _currentRecord;
14 | private int _datasourceId;
15 | private DatasourceRecord.DataTypeEnum _dataType;
16 | private Random _random;
17 |
18 | private DateTime _lastTime = DateTime.UtcNow;
19 |
20 | private List _fields = new List { "Timestamp", "DatasourceId", "Value" };
21 |
22 |
23 | public DbDataReaderSimulator(int datasourceId, DatasourceRecord.DataTypeEnum dataType)
24 | {
25 | _datasourceId = datasourceId;
26 | _dataType = dataType;
27 |
28 | _random = new Random();
29 | }
30 |
31 | public override object this[string name]
32 | {
33 | get
34 | {
35 | throw new NotImplementedException();
36 | }
37 | }
38 |
39 | public override object this[int ordinal]
40 | {
41 | get
42 | {
43 | throw new NotImplementedException();
44 | }
45 | }
46 |
47 | public override int Depth
48 | {
49 | get
50 | {
51 | throw new NotImplementedException();
52 | }
53 | }
54 |
55 | public override int FieldCount
56 | {
57 | get
58 | {
59 | return _fields.Count;
60 | }
61 | }
62 |
63 | public override bool HasRows
64 | {
65 | get
66 | {
67 | throw new NotImplementedException();
68 | }
69 | }
70 |
71 | public override bool IsClosed
72 | {
73 | get
74 | {
75 | throw new NotImplementedException();
76 | }
77 | }
78 |
79 | public override int RecordsAffected
80 | {
81 | get
82 | {
83 | throw new NotImplementedException();
84 | }
85 | }
86 |
87 | public override bool GetBoolean(int ordinal)
88 | {
89 | throw new NotImplementedException();
90 | }
91 |
92 | public override byte GetByte(int ordinal)
93 | {
94 | throw new NotImplementedException();
95 | }
96 |
97 | public override long GetBytes(int ordinal, long dataOffset, byte[] buffer, int bufferOffset, int length)
98 | {
99 | throw new NotImplementedException();
100 | }
101 |
102 | public override char GetChar(int ordinal)
103 | {
104 | throw new NotImplementedException();
105 | }
106 |
107 | public override long GetChars(int ordinal, long dataOffset, char[] buffer, int bufferOffset, int length)
108 | {
109 | throw new NotImplementedException();
110 | }
111 |
112 | public override string GetDataTypeName(int ordinal)
113 | {
114 | throw new NotImplementedException();
115 | }
116 |
117 | public override DateTime GetDateTime(int ordinal)
118 | {
119 | throw new NotImplementedException();
120 | }
121 |
122 | public override decimal GetDecimal(int ordinal)
123 | {
124 | throw new NotImplementedException();
125 | }
126 |
127 | public override double GetDouble(int ordinal)
128 | {
129 | throw new NotImplementedException();
130 | }
131 |
132 | public override IEnumerator GetEnumerator()
133 | {
134 | throw new NotImplementedException();
135 | }
136 |
137 | public override Type GetFieldType(int ordinal)
138 | {
139 | throw new NotImplementedException();
140 | }
141 |
142 | public override float GetFloat(int ordinal)
143 | {
144 | throw new NotImplementedException();
145 | }
146 |
147 | public override Guid GetGuid(int ordinal)
148 | {
149 | throw new NotImplementedException();
150 | }
151 |
152 | public override short GetInt16(int ordinal)
153 | {
154 | throw new NotImplementedException();
155 | }
156 |
157 | public override int GetInt32(int ordinal)
158 | {
159 | throw new NotImplementedException();
160 | }
161 |
162 | public override long GetInt64(int ordinal)
163 | {
164 | throw new NotImplementedException();
165 | }
166 |
167 | public override string GetName(int ordinal)
168 | {
169 | throw new NotImplementedException();
170 | }
171 |
172 | public override int GetOrdinal(string name)
173 | {
174 | throw new NotImplementedException();
175 | }
176 |
177 | public override string GetString(int ordinal)
178 | {
179 | throw new NotImplementedException();
180 | }
181 |
182 | public override object GetValue(int ordinal)
183 | {
184 | if(ordinal == 0)
185 | {
186 | return _currentRecord.Timestamp;
187 | }
188 | if(ordinal == 1)
189 | {
190 | return _currentRecord.DatasourceId;
191 | }
192 | if(ordinal == 2)
193 | {
194 | //return _currentRecord.Value;
195 | return _currentRecord.GetDoubleValue();
196 | }
197 |
198 | throw new NotSupportedException("Don't support column " + ordinal);
199 | }
200 |
201 | public override int GetValues(object[] values)
202 | {
203 | throw new NotImplementedException();
204 | }
205 |
206 | public override bool IsDBNull(int ordinal)
207 | {
208 | return false;
209 | }
210 |
211 | public override bool NextResult()
212 | {
213 | //noop
214 | return false;
215 | }
216 |
217 | public override bool Read()
218 | {
219 | _lastTime = _lastTime.AddSeconds(1);
220 |
221 | _currentRecord = new DatasourceRecord();
222 | _currentRecord.Timestamp = _lastTime;
223 | _currentRecord.DatasourceId = _datasourceId;
224 | _currentRecord.DataType = _dataType;
225 |
226 | if(_dataType == DatasourceRecord.DataTypeEnum.Double)
227 | {
228 | _currentRecord.SetDoubleValue(_random.NextDouble());
229 | }
230 | else if(_dataType == DatasourceRecord.DataTypeEnum.String)
231 | {
232 | _currentRecord.SetStringValue(_random.NextDouble().ToString());
233 | }
234 | else
235 | {
236 | throw new NotSupportedException("Simulator doesn't support " + _dataType.ToString());
237 | }
238 |
239 | return true;
240 | }
241 | }
242 | }
243 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "{}"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright {yyyy} {name of copyright owner}
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/storage/sql-database/sample/.vs/config/applicationhost.config:
--------------------------------------------------------------------------------
1 |
2 |
20 |
21 |
22 |
23 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
140 |
141 |
142 |
143 |
144 |
145 |
151 |
152 |
153 |
154 |
155 |
156 |
157 |
158 |
159 |
160 |
161 |
162 |
163 |
164 |
165 |
166 |
167 |
168 |
169 |
170 |
171 |
172 |
173 |
174 |
175 |
176 |
177 |
178 |
179 |
180 |
181 |
182 |
183 |
184 |
185 |
186 |
187 |
188 |
189 |
190 |
191 |
192 |
193 |
194 |
195 |
196 |
197 |
198 |
199 |
200 |
201 |
202 |
203 |
204 |
205 |
211 |
212 |
213 |
214 |
215 |
216 |
217 |
218 |
219 |
220 |
221 |
222 |
223 |
224 |
225 |
226 |
227 |
228 |
229 |
230 |
231 |
232 |
233 |
234 |
235 |
236 |
237 |
238 |
239 |
240 |
241 |
242 |
243 |
244 |
245 |
246 |
247 |
248 |
249 |
250 |
251 |
252 |
253 |
254 |
255 |
256 |
257 |
258 |
259 |
260 |
261 |
262 |
263 |
264 |
265 |
266 |
267 |
268 |
269 |
270 |
271 |
272 |
273 |
274 |
275 |
276 |
277 |
278 |
279 |
280 |
281 |
282 |
283 |
284 |
285 |
286 |
287 |
288 |
289 |
290 |
291 |
292 |
293 |
294 |
295 |
296 |
297 |
298 |
299 |
300 |
301 |
302 |
303 |
304 |
305 |
306 |
307 |
308 |
309 |
310 |
311 |
312 |
313 |
314 |
315 |
316 |
317 |
318 |
319 |
320 |
321 |
322 |
323 |
324 |
325 |
326 |
327 |
328 |
329 |
330 |
331 |
332 |
333 |
334 |
335 |
336 |
337 |
338 |
339 |
340 |
341 |
342 |
343 |
344 |
345 |
346 |
347 |
348 |
349 |
350 |
351 |
352 |
353 |
354 |
355 |
356 |
357 |
358 |
359 |
360 |
361 |
362 |
363 |
364 |
365 |
366 |
367 |
368 |
369 |
370 |
371 |
372 |
373 |
374 |
375 |
376 |
377 |
378 |
379 |
380 |
381 |
382 |
383 |
384 |
385 |
386 |
387 |
388 |
389 |
390 |
391 |
392 |
393 |
394 |
395 |
396 |
397 |
398 |
399 |
400 |
401 |
402 |
403 |
404 |
405 |
406 |
407 |
408 |
409 |
410 |
411 |
412 |
413 |
414 |
415 |
416 |
417 |
418 |
419 |
420 |
421 |
422 |
423 |
424 |
425 |
426 |
427 |
428 |
429 |
430 |
431 |
432 |
433 |
434 |
435 |
436 |
437 |
438 |
439 |
440 |
441 |
442 |
443 |
444 |
445 |
446 |
447 |
448 |
449 |
450 |
451 |
452 |
453 |
454 |
455 |
456 |
457 |
458 |
459 |
460 |
461 |
462 |
463 |
464 |
465 |
466 |
467 |
468 |
469 |
470 |
471 |
472 |
473 |
474 |
475 |
476 |
477 |
478 |
479 |
480 |
481 |
482 |
483 |
484 |
485 |
486 |
487 |
488 |
489 |
490 |
491 |
492 |
493 |
494 |
495 |
496 |
497 |
498 |
499 |
500 |
501 |
502 |
503 |
504 |
505 |
506 |
507 |
508 |
509 |
510 |
511 |
512 |
513 |
514 |
515 |
516 |
517 |
518 |
519 |
520 |
521 |
522 |
523 |
524 |
525 |
526 |
527 |
528 |
529 |
530 |
531 |
532 |
533 |
534 |
535 |
536 |
537 |
538 |
539 |
540 |
541 |
542 |
543 |
544 |
545 |
546 |
547 |
548 |
549 |
550 |
551 |
552 |
553 |
554 |
555 |
556 |
557 |
558 |
559 |
560 |
561 |
562 |
563 |
564 |
565 |
566 |
567 |
568 |
569 |
570 |
571 |
572 |
573 |
574 |
575 |
576 |
577 |
578 |
579 |
580 |
581 |
582 |
583 |
584 |
585 |
586 |
587 |
588 |
589 |
590 |
591 |
592 |
593 |
594 |
595 |
596 |
597 |
598 |
599 |
600 |
601 |
602 |
603 |
604 |
605 |
606 |
607 |
608 |
609 |
610 |
611 |
612 |
613 |
614 |
615 |
616 |
617 |
618 |
619 |
620 |
621 |
622 |
623 |
624 |
625 |
626 |
627 |
628 |
629 |
630 |
631 |
632 |
633 |
634 |
635 |
636 |
637 |
638 |
639 |
640 |
641 |
642 |
643 |
644 |
645 |
646 |
647 |
648 |
649 |
650 |
651 |
652 |
653 |
654 |
655 |
656 |
657 |
658 |
659 |
660 |
661 |
662 |
663 |
664 |
665 |
666 |
667 |
668 |
669 |
670 |
671 |
672 |
673 |
674 |
675 |
676 |
677 |
678 |
679 |
680 |
681 |
682 |
683 |
684 |
685 |
686 |
687 |
688 |
689 |
690 |
691 |
692 |
693 |
694 |
695 |
696 |
697 |
698 |
699 |
700 |
701 |
702 |
703 |
704 |
705 |
706 |
707 |
708 |
709 |
710 |
711 |
712 |
713 |
714 |
715 |
716 |
717 |
718 |
719 |
720 |
721 |
722 |
723 |
724 |
725 |
726 |
727 |
728 |
729 |
730 |
731 |
732 |
733 |
734 |
735 |
736 |
737 |
738 |
739 |
740 |
741 |
742 |
743 |
744 |
745 |
746 |
747 |
748 |
749 |
750 |
751 |
752 |
753 |
754 |
755 |
756 |
757 |
758 |
759 |
760 |
761 |
762 |
763 |
764 |
765 |
766 |
767 |
768 |
769 |
770 |
771 |
772 |
773 |
774 |
775 |
776 |
777 |
778 |
779 |
780 |
781 |
782 |
783 |
784 |
785 |
786 |
787 |
788 |
789 |
790 |
791 |
792 |
793 |
794 |
795 |
796 |
797 |
798 |
799 |
800 |
801 |
802 |
803 |
804 |
805 |
806 |
807 |
808 |
809 |
810 |
811 |
812 |
813 |
814 |
815 |
816 |
817 |
818 |
819 |
820 |
821 |
822 |
823 |
824 |
825 |
826 |
827 |
828 |
829 |
830 |
831 |
832 |
833 |
834 |
835 |
836 |
837 |
838 |
839 |
840 |
841 |
842 |
843 |
844 |
845 |
846 |
847 |
848 |
849 |
850 |
851 |
852 |
853 |
854 |
855 |
856 |
857 |
858 |
859 |
860 |
861 |
862 |
863 |
864 |
865 |
866 |
867 |
868 |
869 |
870 |
871 |
872 |
873 |
874 |
875 |
876 |
877 |
878 |
879 |
880 |
881 |
882 |
883 |
884 |
887 |
888 |
889 |
890 |
891 |
892 |
893 |
894 |
895 |
896 |
897 |
898 |
899 |
900 |
903 |
904 |
905 |
906 |
907 |
908 |
909 |
910 |
911 |
912 |
913 |
914 |
915 |
916 |
917 |
918 |
919 |
920 |
921 |
922 |
923 |
924 |
925 |
926 |
927 |
928 |
929 |
930 |
931 |
932 |
933 |
934 |
935 |
936 |
937 |
938 |
939 |
940 |
941 |
942 |
943 |
944 |
945 |
946 |
947 |
948 |
949 |
950 |
951 |
952 |
953 |
954 |
955 |
956 |
957 |
958 |
959 |
960 |
961 |
962 |
963 |
964 |
965 |
966 |
967 |
968 |
969 |
970 |
971 |
972 |
973 |
974 |
975 |
976 |
977 |
978 |
979 |
980 |
981 |
982 |
983 |
984 |
985 |
986 |
987 |
988 |
989 |
990 |
991 |
992 |
993 |
994 |
995 |
996 |
997 |
998 |
999 |
1000 |
1001 |
1002 |
1003 |
1004 |
1005 |
1006 |
1007 |
1008 |
1009 |
1010 |
1011 |
1012 |
1013 |
1014 |
1015 |
1016 |
1017 |
1018 |
1019 |
1020 |
1021 |
1022 |
1023 |
1024 |
1025 |
1026 |
1027 |
1028 |
1029 |
1030 |
1031 |
--------------------------------------------------------------------------------