├── .gitattributes ├── .gitignore ├── LICENSE.md ├── README.md ├── Richter Azure Storage Patterns.sln ├── StoragePatternsDemos ├── StoragePatternsDemos.cs ├── StoragePatternsDemos.csproj ├── app.config └── packages.config ├── Wintellect.Azure.Storage ├── Properties │ └── AssemblyInfo.cs ├── Storage.Blob.Logs.cs ├── Storage.Blob.cs ├── Storage.Queue.cs ├── Storage.Sas.cs ├── Storage.Table.PropertySerializer.cs ├── Storage.Table.cs ├── Storage.cs ├── Wintellect Azure Storage Library.docx ├── Wintellect.Azure.Storage.csproj ├── Wintellect.IO.cs ├── Wintellect.Periodic.cs ├── Wintellect.cs ├── app.config └── packages.config └── packages └── repositories.config /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.sln.docstates 8 | *.ide 9 | 10 | # Build results 11 | 12 | [Dd]ebug/ 13 | [Rr]elease/ 14 | x64/ 15 | build/ 16 | [Bb]in/ 17 | [Oo]bj/ 18 | 19 | # Enable "build/" folder in the NuGet Packages folder since NuGet packages use it for MSBuild targets 20 | !packages/*/build/ 21 | 22 | # MSTest test Results 23 | [Tt]est[Rr]esult*/ 24 | [Bb]uild[Ll]og.* 25 | 26 | *_i.c 27 | *_p.c 28 | *.ilk 29 | *.meta 30 | *.obj 31 | *.pch 32 | *.pdb 33 | *.pgc 34 | *.pgd 35 | *.rsp 36 | *.sbr 37 | *.tlb 38 | *.tli 39 | *.tlh 40 | *.tmp 41 | *.tmp_proj 42 | *.log 43 | *.vspscc 44 | *.vssscc 45 | .builds 46 | *.pidb 47 | *.log 48 | *.scc 49 | 50 | # Visual C++ cache files 51 | ipch/ 52 | *.aps 53 | *.ncb 54 | *.opensdf 55 | *.sdf 56 | *.cachefile 57 | 58 | # Visual Studio profiler 59 | *.psess 60 | *.vsp 61 | *.vspx 62 | 63 | # Guidance Automation Toolkit 64 | *.gpState 65 | 66 | # ReSharper is a .NET coding add-in 67 | _ReSharper*/ 68 | *.[Rr]e[Ss]harper 69 | 70 | # TeamCity is a build add-in 71 | _TeamCity* 72 | 73 | # DotCover is a Code Coverage Tool 74 | *.dotCover 75 | 76 | # NCrunch 77 | *.ncrunch* 78 | .*crunch*.local.xml 79 | 80 | # Installshield output folder 81 | [Ee]xpress/ 82 | 83 | # DocProject is a documentation generator add-in 84 | DocProject/buildhelp/ 85 | DocProject/Help/*.HxT 86 | DocProject/Help/*.HxC 87 | DocProject/Help/*.hhc 88 | DocProject/Help/*.hhk 89 | DocProject/Help/*.hhp 90 | DocProject/Help/Html2 91 | DocProject/Help/html 92 | 93 | # Click-Once directory 94 | publish/ 95 | 96 | # Publish Web Output 97 | *.Publish.xml 98 | 99 | # NuGet Packages Directory 100 | ## TODO: If you have NuGet Package Restore enabled, uncomment the next line 101 | packages/ 102 | 103 | # Windows Azure Build Output 104 | csx 105 | *.build.csdef 106 | 107 | # Windows Store app package directory 108 | AppPackages/ 109 | 110 | # Others 111 | sql/ 112 | *.Cache 113 | ClientBin/ 114 | [Ss]tyle[Cc]op.* 115 | ~$* 116 | *~ 117 | *.dbmdl 118 | *.[Pp]ublish.xml 119 | *.pfx 120 | *.publishsettings 121 | 122 | # RIA/Silverlight projects 123 | Generated_Code/ 124 | 125 | # Backup & report files from converting an old project file to a newer 126 | # Visual Studio version. Backup files are not needed, because we have git ;-) 127 | _UpgradeReport_Files/ 128 | Backup*/ 129 | UpgradeLog*.XML 130 | UpgradeLog*.htm 131 | 132 | # SQL Server files 133 | App_Data/*.mdf 134 | App_Data/*.ldf 135 | 136 | 137 | #LightSwitch generated files 138 | GeneratedArtifacts/ 139 | _Pvt_Extensions/ 140 | ModelManifest.xml 141 | 142 | # ========================= 143 | # Windows detritus 144 | # ========================= 145 | 146 | # Windows image file caches 147 | Thumbs.db 148 | ehthumbs.db 149 | 150 | # Folder config file 151 | Desktop.ini 152 | 153 | # Recycle Bin used on file shares 154 | $RECYCLE.BIN/ 155 | 156 | # Mac desktop service store files 157 | .DS_Store 158 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Jeffrey Richter 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | AzureStorage 2 | ============ 3 | -------------------------------------------------------------------------------- /Richter Azure Storage Patterns.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 2013 4 | VisualStudioVersion = 12.0.30723.0 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wintellect.Azure.Storage", "Wintellect.Azure.Storage\Wintellect.Azure.Storage.csproj", "{CF2AE096-D1B0-4C33-9921-C1B19948F971}" 7 | EndProject 8 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StoragePatternsDemos", "StoragePatternsDemos\StoragePatternsDemos.csproj", "{6D95DAEA-6E60-4FB3-ACED-B5A151C2CD9C}" 9 | EndProject 10 | Global 11 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 12 | Debug|Any CPU = Debug|Any CPU 13 | Debug|Mixed Platforms = Debug|Mixed Platforms 14 | Debug|x86 = Debug|x86 15 | Release|Any CPU = Release|Any CPU 16 | Release|Mixed Platforms = Release|Mixed Platforms 17 | Release|x86 = Release|x86 18 | EndGlobalSection 19 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 20 | {CF2AE096-D1B0-4C33-9921-C1B19948F971}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 21 | {CF2AE096-D1B0-4C33-9921-C1B19948F971}.Debug|Any CPU.Build.0 = Debug|Any CPU 22 | {CF2AE096-D1B0-4C33-9921-C1B19948F971}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU 23 | {CF2AE096-D1B0-4C33-9921-C1B19948F971}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU 24 | {CF2AE096-D1B0-4C33-9921-C1B19948F971}.Debug|x86.ActiveCfg = Debug|Any CPU 25 | {CF2AE096-D1B0-4C33-9921-C1B19948F971}.Release|Any CPU.ActiveCfg = Release|Any CPU 26 | {CF2AE096-D1B0-4C33-9921-C1B19948F971}.Release|Any CPU.Build.0 = Release|Any CPU 27 | {CF2AE096-D1B0-4C33-9921-C1B19948F971}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU 28 | {CF2AE096-D1B0-4C33-9921-C1B19948F971}.Release|Mixed Platforms.Build.0 = Release|Any CPU 29 | {CF2AE096-D1B0-4C33-9921-C1B19948F971}.Release|x86.ActiveCfg = Release|Any CPU 30 | {6D95DAEA-6E60-4FB3-ACED-B5A151C2CD9C}.Debug|Any CPU.ActiveCfg = Debug|x86 31 | {6D95DAEA-6E60-4FB3-ACED-B5A151C2CD9C}.Debug|Mixed Platforms.ActiveCfg = Debug|x86 32 | {6D95DAEA-6E60-4FB3-ACED-B5A151C2CD9C}.Debug|Mixed Platforms.Build.0 = Debug|x86 33 | {6D95DAEA-6E60-4FB3-ACED-B5A151C2CD9C}.Debug|x86.ActiveCfg = Debug|x86 34 | {6D95DAEA-6E60-4FB3-ACED-B5A151C2CD9C}.Debug|x86.Build.0 = Debug|x86 35 | {6D95DAEA-6E60-4FB3-ACED-B5A151C2CD9C}.Release|Any CPU.ActiveCfg = Release|x86 36 | {6D95DAEA-6E60-4FB3-ACED-B5A151C2CD9C}.Release|Mixed Platforms.ActiveCfg = Release|x86 37 | {6D95DAEA-6E60-4FB3-ACED-B5A151C2CD9C}.Release|Mixed Platforms.Build.0 = Release|x86 38 | {6D95DAEA-6E60-4FB3-ACED-B5A151C2CD9C}.Release|x86.ActiveCfg = Release|x86 39 | {6D95DAEA-6E60-4FB3-ACED-B5A151C2CD9C}.Release|x86.Build.0 = Release|x86 40 | EndGlobalSection 41 | GlobalSection(SolutionProperties) = preSolution 42 | HideSolutionNode = FALSE 43 | EndGlobalSection 44 | EndGlobal 45 | -------------------------------------------------------------------------------- /StoragePatternsDemos/StoragePatternsDemos.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | true 5 | Debug 6 | x86 7 | 8.0.30703 8 | 2.0 9 | {6D95DAEA-6E60-4FB3-ACED-B5A151C2CD9C} 10 | Exe 11 | Properties 12 | AzureStorage 13 | AzureStorage 14 | v4.5.1 15 | 16 | 17 | 512 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | publish\ 27 | true 28 | Disk 29 | false 30 | Foreground 31 | 7 32 | Days 33 | false 34 | false 35 | true 36 | 0 37 | 1.0.0.%2a 38 | false 39 | false 40 | true 41 | 42 | 43 | x86 44 | true 45 | full 46 | false 47 | bin\Debug\ 48 | DEBUG;TRACE 49 | prompt 50 | 4 51 | false 52 | false 53 | true 54 | true 55 | AllRules.ruleset 56 | false 57 | 58 | 59 | x86 60 | pdbonly 61 | true 62 | bin\Release\ 63 | TRACE 64 | prompt 65 | 4 66 | false 67 | true 68 | true 69 | AllRules.ruleset 70 | false 71 | 72 | 73 | 74 | False 75 | ..\packages\Microsoft.Data.Edm.5.6.2\lib\net40\Microsoft.Data.Edm.dll 76 | 77 | 78 | False 79 | ..\packages\Microsoft.Data.OData.5.6.2\lib\net40\Microsoft.Data.OData.dll 80 | 81 | 82 | False 83 | ..\packages\Microsoft.Data.Services.Client.5.6.2\lib\net40\Microsoft.Data.Services.Client.dll 84 | 85 | 86 | False 87 | ..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll 88 | 89 | 90 | False 91 | ..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll 92 | 93 | 94 | ..\packages\Newtonsoft.Json.5.0.8\lib\net45\Newtonsoft.Json.dll 95 | 96 | 97 | 98 | 99 | 100 | False 101 | ..\packages\System.Spatial.5.6.2\lib\net40\System.Spatial.dll 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | False 116 | Microsoft .NET Framework 4 %28x86 and x64%29 117 | true 118 | 119 | 120 | False 121 | .NET Framework 3.5 SP1 Client Profile 122 | false 123 | 124 | 125 | False 126 | .NET Framework 3.5 SP1 127 | false 128 | 129 | 130 | False 131 | Windows Installer 3.1 132 | true 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | {cf2ae096-d1b0-4c33-9921-c1b19948f971} 142 | Wintellect.Azure.Storage 143 | 144 | 145 | 146 | 153 | -------------------------------------------------------------------------------- /StoragePatternsDemos/app.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /StoragePatternsDemos/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /Wintellect.Azure.Storage/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("Wintellect.Azure.Storage")] 9 | [assembly: AssemblyDescription("Wintellect's Azure Storage Library")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("Jeffrey Richter & Wintellect")] 12 | [assembly: AssemblyProduct("Wintellect.Azure.Storage")] 13 | [assembly: AssemblyCopyright("Copyright © Jeffrey Richter")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("257b7065-4cfb-4061-a34d-10fa148b2db1")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.6.0.0")] 36 | [assembly: AssemblyFileVersion("1.6.0.0")] 37 | -------------------------------------------------------------------------------- /Wintellect.Azure.Storage/Storage.Blob.Logs.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Globalization; 4 | using System.Text; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | using Microsoft.WindowsAzure.Storage; 8 | using Microsoft.WindowsAzure.Storage.Blob; 9 | using Wintellect.Periodic; 10 | 11 | namespace Wintellect.Azure.Storage.Blob { 12 | /// The base class for on-going and periodic log blobs. 13 | public abstract class BlobLogBase { 14 | /// The container holding the log blob. 15 | protected readonly CloudBlobContainer m_container; 16 | private readonly AppendBlobBlockInfo m_info; 17 | private readonly String m_extension; 18 | 19 | /// Constructs a BlobLogBase object. 20 | /// Indicates the container where log blobs should be placed. 21 | /// Indicates the extension (initial period is optional) of the log blob. 22 | /// Indicates the mime type of the log blob. 23 | /// Indicates the header (first block) of the log blob. Pass 'null' if you do not want a header. 24 | /// Indicates the maximum number of blocks the blob should have. Old blocks are deleted when going over this limit. 25 | protected internal BlobLogBase(CloudBlobContainer container, String extension, String mimeType, Byte[] header = null, Int32 maxEntries = 49000) { 26 | m_container = container; 27 | m_extension = extension.StartsWith(".") ? extension : "." + extension; 28 | m_info = new AppendBlobBlockInfo { MimeType = mimeType, Header = header, MaxEntries = maxEntries }; 29 | } 30 | 31 | /// Returns the CloudBlockBlob from the blob name. 32 | /// The blob's name. 33 | /// The CloudBlockBlob. 34 | protected CloudBlockBlob GetBlob(String blobName) { 35 | CloudBlockBlob blob = m_container.GetBlockBlobReference(blobName + m_extension); 36 | return blob; 37 | } 38 | 39 | /// Appends a block to the end of the log blob. 40 | /// An identifier identifying the main part of the blob's name. 41 | /// The full name of the blob. 42 | /// The data to append as a block to the end of the log blob. 43 | /// The blob request options. 44 | /// The operation context. 45 | /// The cancellation token. 46 | /// A Task indicating when the operation completes. 47 | protected Task AppendBlockAsync(String Id, String blobName, Byte[] dataToAppend, 48 | BlobRequestOptions options = null, OperationContext operationContext = null, 49 | CancellationToken cancellationToken = default(CancellationToken)) { 50 | return GetBlob(blobName).AppendBlockAsync(m_info, dataToAppend, options, operationContext, cancellationToken); 51 | } 52 | 53 | /// An event that is raised whenever a new log blob is being created. 54 | public event EventHandler NewLog { 55 | add { m_info.NewBlob += value; } 56 | remove { m_info.NewBlob -= value; } 57 | } 58 | 59 | /// Calculates and returns a URI string (containing the blob URI and its SAS query string) for a log blob. 60 | /// The blob to get a URI for. 61 | /// The SAS permissions to apply to the blob. 62 | /// The URI (containing the blob URI and its SAS query string). 63 | public String GetSasUri(CloudBlockBlob blob, SharedAccessBlobPolicy sap) { 64 | // Make sure we use HTTPS so that SAS doesn't transfer in clear text ensuring security 65 | String uri = (blob.Uri.Scheme == Uri.UriSchemeHttps) ? blob.Uri.ToString() : 66 | new StringBuilder(Uri.UriSchemeHttps).Append("://").Append(blob.Uri.Host).Append(blob.Uri.PathAndQuery).ToString(); 67 | return uri + blob.GetSharedAccessSignature(sap); 68 | } 69 | } 70 | 71 | /// Represents an on-going (not periodic) log blob. 72 | public sealed class BlobLog : BlobLogBase { 73 | /// Construct an on-going log blob. 74 | /// The container holding the log blob. 75 | /// Indicates the extension (initial period is optional) of the log blob. 76 | /// Indicates the mime type of the log blob. 77 | /// Indicates the header (first block) of the log blob. Pass 'null' if you do not want a header. 78 | /// Indicates the maximum number of blocks the blob should have. Old blocks are deleted when going over this limit. 79 | public BlobLog(CloudBlobContainer container, String extension, String mimeType, Byte[] header, Int32 maxEntries = 49000) : 80 | base(container, extension, mimeType, header, maxEntries) { 81 | } 82 | 83 | /// Appends a block to the end of the log blob. 84 | /// An identifier identifying the blob's name. 85 | /// The data to append as a block to the end of the log blob. 86 | /// A Task indicating when the operation completes. 87 | public Task AppendAsync(String Id, Byte[] dataToAppend) { 88 | return base.AppendBlockAsync(Id, Id, dataToAppend); 89 | } 90 | 91 | /// Returns the CloudBlockBlob from the blob Id. 92 | /// The log blob's Id. 93 | /// The CloudBlockBlob. 94 | public new CloudBlockBlob GetBlob(String Id) { return base.GetBlob(Id); } 95 | } 96 | 97 | /// Used to indicate the naming convention (and therefore sort order) for PeriodBlobLogs. 98 | public enum PeriodBlobLogOrder { 99 | /// Names/sorts PeriodBlobLogs by their Id. 100 | Id, 101 | /// Names/sorts PeriodBlobLogs by their time (chronological order). 102 | Time 103 | } 104 | 105 | 106 | /// Represents a periodic (not on-going) log blob. 107 | public sealed class PeriodBlobLog : BlobLogBase { 108 | private readonly Period m_period; 109 | private readonly PeriodBlobLogOrder m_logOrder; 110 | private readonly String[] m_delimiter; 111 | 112 | /// Construct an on-going log blob. 113 | /// Indicates the time period that a log blob should cover. 114 | /// Indicates the naming convention (sort order) used when naming periodic log blobs. 115 | /// Indicates the delimiter to use to separate the blob's name, period start date, and period end date. 116 | /// The container holding the log blob. 117 | /// Indicates the extension (initial period is optional) of the log blob. 118 | /// Indicates the mime type of the log blob. 119 | /// Indicates the header (first block) of the log blob. Pass 'null' if you do not want a header. 120 | /// Indicates the maximum number of blocks the blob should have. Old blocks are deleted when going over this limit. 121 | public PeriodBlobLog(Period period, PeriodBlobLogOrder logOrder, String delimiter, 122 | CloudBlobContainer container, String extension, String mimeType, Byte[] header, Int32 maxEntries = 49000) : 123 | base(container, extension, mimeType, header, maxEntries) { 124 | m_period = period; 125 | m_logOrder = logOrder; 126 | m_delimiter = new[] { delimiter }; 127 | } 128 | 129 | private static String ToYMD(DateTimeOffset dto) { return dto.ToString("yyyyMMdd"); } 130 | private static DateTimeOffset FromYMD(String dto) { return DateTimeOffset.ParseExact(dto, "yyyyMMdd", null, DateTimeStyles.AssumeUniversal); } 131 | 132 | /// Returns the CloudBlockBlob from the blob's ID and cycle date. 133 | /// The blob's identifier. 134 | /// The date used to calculate the blob's name. 135 | /// The CloudBlockBlob. 136 | public CloudBlockBlob GetBlob(String Id, DateTimeOffset cycleDate) { 137 | return base.GetBlob(GenerateBlobName(Id, cycleDate)); 138 | } 139 | private String GenerateBlobName(String Id, DateTimeOffset cycleDate) { 140 | DateTimeOffset periodStop, periodStart = PeriodCalculator.CalculatePeriodDates(m_period, cycleDate, 0, out periodStop); 141 | StringBuilder blobName = new StringBuilder(); 142 | String delimiter = m_delimiter[0]; 143 | switch (m_logOrder) { 144 | case PeriodBlobLogOrder.Id: 145 | blobName.Append(Id).Append(delimiter) 146 | .Append(ToYMD(periodStart)).Append(delimiter) 147 | .Append(ToYMD(periodStop)); 148 | break; 149 | case PeriodBlobLogOrder.Time: 150 | blobName.Append(ToYMD(periodStart)).Append(delimiter) 151 | .Append(ToYMD(periodStop)).Append(delimiter) 152 | .Append(Id); 153 | break; 154 | } 155 | return blobName.Append(delimiter).ToString(); 156 | } 157 | 158 | /// A class containing information about a period log blob. 159 | public sealed class PeriodBlobLogNameInfo { 160 | internal PeriodBlobLogNameInfo(String id, DateTimeOffset periodStart, DateTimeOffset periodStop) { 161 | Id = id; 162 | PeriodStart = periodStart; 163 | PeriodStop = periodStop; 164 | } 165 | /// Indicates the log blob's ID. 166 | public readonly String Id; 167 | /// Indicates the period's starts date. 168 | public readonly DateTimeOffset PeriodStart; 169 | /// Indicates the period's end date. 170 | public readonly DateTimeOffset PeriodStop; 171 | } 172 | 173 | /// Parses a blob name returning its ID, and period start and end dates. 174 | /// The blob whose name to parse. 175 | /// The log blob's ID, period start and period end dates. 176 | public PeriodBlobLogNameInfo ParseLogName(CloudBlockBlob blob) { 177 | String[] tokens = blob.Name.Split(m_delimiter, StringSplitOptions.None); 178 | PeriodBlobLogNameInfo logInfo = null; 179 | switch (m_logOrder) { 180 | case PeriodBlobLogOrder.Id: 181 | logInfo = new PeriodBlobLogNameInfo(tokens[0], FromYMD(tokens[1]), FromYMD(tokens[2])); 182 | break; 183 | case PeriodBlobLogOrder.Time: 184 | logInfo = new PeriodBlobLogNameInfo(tokens[2], FromYMD(tokens[0]), FromYMD(tokens[1])); 185 | break; 186 | } 187 | return logInfo; 188 | } 189 | 190 | /// Appends a block to the end of the log blob. 191 | /// An identifier identifying the blob's name. 192 | /// The date used to determine which blob to append the data to. 193 | /// The data to append as a block to the end of the log blob. 194 | /// A Task indicating when the operation completes. 195 | public Task AppendAsync(String Id, DateTimeOffset cycleDate, Byte[] dataToAppend) { 196 | String blobName = GenerateBlobName(Id, cycleDate); 197 | return base.AppendBlockAsync(Id, blobName, dataToAppend); 198 | } 199 | 200 | private String CalculatePrefix(String Id, DateTimeOffset startDate = default(DateTimeOffset)) { 201 | // If blobs are prefixed by Id, we scan the subset of blobs with this Id 202 | // If blobs are not prefixed by Id, we scan all blobs in this container 203 | if (m_logOrder == PeriodBlobLogOrder.Id) return (Id != null) ? (Id + m_delimiter[0]) : null; 204 | return ToYMD(startDate); 205 | } 206 | 207 | /// Deletes log blobs containing data before the specified date. 208 | /// Identifies the log blob. 209 | /// The date before which log blobs should be deleted. 210 | /// A Task indicating when the operation completes. 211 | public Task DeleteOldLogsAsync(String Id, DateTimeOffset oldDate) { 212 | return m_container.ScanBlobsAsync(blob => DeleteOldLogAsync(blob, oldDate), CalculatePrefix(Id)); 213 | } 214 | 215 | private async Task DeleteOldLogAsync(CloudBlockBlob blob, DateTimeOffset oldDate) { 216 | PeriodBlobLogNameInfo lni = ParseLogName(blob); 217 | if (lni.PeriodStop < oldDate) await blob.DeleteAsync(); // Log data ends before old date, delete it 218 | return lni.PeriodStart > oldDate; // Stop when we hit a log newer than the old date 219 | } 220 | 221 | /// Finds all log blobs with the specified ID containing data after the oldest date. 222 | /// The ID of the log blobs to find. 223 | /// The oldest date of blobs to consider. 224 | /// A collection of CloudBlockBlob objects. 225 | public async Task> FindLogsAsync(String Id, DateTimeOffset oldest = default(DateTimeOffset)) { 226 | List blobs = new List(); 227 | await m_container.ScanBlobsAsync(blob => { 228 | var lni = ParseLogName(blob); 229 | if (lni.Id == Id && lni.PeriodStart >= oldest) blobs.Add(blob); 230 | 231 | // If ordered by Id, it will stop when no more blobs match the Id prefix 232 | // If ordered by time, it will stop when all done 233 | return Task.FromResult(false); 234 | }, CalculatePrefix(Id, oldest)); 235 | return blobs; 236 | } 237 | } 238 | } -------------------------------------------------------------------------------- /Wintellect.Azure.Storage/Storage.Blob.cs: -------------------------------------------------------------------------------- 1 | /****************************************************************************** 2 | Module: Storage.Blob.cs 3 | Notices: Copyright (c) by Jeffrey Richter & Wintellect 4 | ******************************************************************************/ 5 | 6 | using System; 7 | using System.Collections.Generic; 8 | using System.Globalization; 9 | using System.IO; 10 | using System.Linq; 11 | using System.Net; 12 | using System.Runtime.InteropServices; 13 | using System.Text; 14 | using System.Threading; 15 | using System.Threading.Tasks; 16 | using Microsoft.WindowsAzure.Storage; 17 | using Microsoft.WindowsAzure.Storage.Blob; 18 | using Microsoft.WindowsAzure.Storage.Blob.Protocol; 19 | #if ResumableFileUploader 20 | using Wintellect.Threading.AsyncProgModel; 21 | #endif 22 | 23 | namespace Wintellect.Azure.Storage.Blob { 24 | /// Defines extension methods useful when working with Azure blob containers. 25 | public static class BlobContainerExtensions { 26 | /// Creates a blob container if it doesn't exist optionally deleting all its blobs. 27 | /// The blob container to create or clear out. 28 | /// True to delete all blobs in this container. 29 | /// The passed-in CloudBlobContainer. 30 | public static async Task EnsureExistsAsync(this CloudBlobContainer container, Boolean clear = false) { 31 | if (clear) { 32 | await container.DeleteIfExistsAsync().ConfigureAwait(false); 33 | await container.RetryUntilCreatedAsync().ConfigureAwait(false); 34 | } else await container.CreateIfNotExistsAsync().ConfigureAwait(false); 35 | return container; 36 | } 37 | #if DeleteMe 38 | public static async Task UploadDirToBlobContainerAsync(this CloudBlobContainer container, String rootPath, SearchOption searchOption = SearchOption.TopDirectoryOnly, Action callback = null) { 39 | await container.EnsureExistsAsync().ConfigureAwait(false); 40 | foreach (var file in Directory.EnumerateFiles(rootPath, "*.*", searchOption)) { 41 | var cloudFile = container.GetBlockBlobReference(file.Substring(rootPath.Length + 1).Replace('\\', '/')); 42 | if (callback != null) callback(cloudFile); 43 | using (var stream = File.OpenRead(file)) { 44 | cloudFile.UploadFromStream(stream); 45 | } 46 | } 47 | } 48 | #endif 49 | 50 | /// Returns True for a null result segment or if the result segment has more. 51 | /// A ContainerResultSegment (can be null). 52 | /// True if crs is null or if its continuation token is not null. 53 | public static Boolean HasMore(this ContainerResultSegment crs) { 54 | return (crs == null) ? true : (crs.SafeContinuationToken() != null); 55 | } 56 | 57 | /// Returns null if crs is null or if its ContinuationToken is null. 58 | /// A ContainerResultSegment. 59 | /// A BlobContinuationToken. 60 | public static BlobContinuationToken SafeContinuationToken(this ContainerResultSegment crs) { 61 | return (crs == null) ? null : crs.ContinuationToken; 62 | } 63 | 64 | /// Scans blobs in a container and invokes a callback method for each blob passing the specified criteria. 65 | /// The container with the blobs to scan. 66 | /// The method to invoke for each blob discovered in the container. 67 | /// The case-sensitive prefix indicating the subset of blobs to find. 68 | /// What additional blob data to retrieve during the scan. 69 | /// Allows you to control retry and timeout options. 70 | /// The context for the operation. 71 | /// A Task which you can use to track the scan operation. 72 | public static async Task ScanBlobsAsync(this CloudBlobContainer container, Func> callback, String prefix = null, 73 | BlobListingDetails blobListingDetails = BlobListingDetails.None, BlobRequestOptions blobRequestOptions = null, OperationContext operationContext = null) { 74 | prefix = container.Name + "/" + prefix; 75 | for (BlobResultSegment brs = null; brs.HasMore(); ) { 76 | brs = await container.ServiceClient.ListBlobsSegmentedAsync(prefix, true, blobListingDetails, null, brs.SafeContinuationToken(), 77 | blobRequestOptions, operationContext); 78 | foreach (CloudBlockBlob blob in brs.Results) { 79 | if (await callback(blob)) return; // Callback returns true to stop 80 | } 81 | } 82 | } 83 | } 84 | } 85 | 86 | namespace Wintellect.Azure.Storage.Blob { 87 | /// Defines extension methods useful when working with Azure blobs. 88 | public static class BlobExtensions { 89 | /// 90 | /// Appends a string to the end of a blob (creating the blob and appending a header if the blob doesn't exist). 91 | /// Before calling this method, set the blob's ContentType Property (ex: "text/csv; charset=utf-8") 92 | /// 93 | /// The blob to have the string appended to. 94 | /// The string to append. 95 | /// The header string to put in the blob if the blob is being created. 96 | /// The maximum block sized in bytes (0=Azure default of 4MB) 97 | /// A Microsoft.WindowsAzure.Storage.Blob.BlobRequestOptions object that specifies additional options for the request. 98 | /// An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation. 99 | /// A System.Threading.CancellationToken to observe while waiting for a task to complete. 100 | /// A System.Threading.Tasks.Task object that represents the current operation. 101 | public static Task AppendAsync(this CloudBlockBlob blob, String @string, Byte[] header, Int32 maxBlockSize = 0, 102 | BlobRequestOptions options = null, OperationContext operationContext = null, 103 | CancellationToken cancellationToken = default(CancellationToken)) { 104 | return blob.AppendAsync(new MemoryStream(@string.Encode()), header, maxBlockSize, 105 | options, operationContext, cancellationToken); 106 | } 107 | 108 | /// 109 | /// Appends a stream's contents to the end of a blob (creating the blob and appending a header if the blob doesn't exist). 110 | /// Before calling this method, set the blob's ContentType Property (ex: "text/csv; charset=utf-8") 111 | /// 112 | /// The blob to have the byte array appended to. 113 | /// The stream with contents to append. 114 | /// The header byte array to put in the blob if the blob is being created. 115 | /// The maximum block sized in bytes (0=Azure default of 4MB) 116 | /// A Microsoft.WindowsAzure.Storage.Blob.BlobRequestOptions object that specifies additional options for the request. 117 | /// An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation. 118 | /// A System.Threading.CancellationToken to observe while waiting for a task to complete. 119 | /// A System.Threading.Tasks.Task object that represents the current operation. 120 | public static async Task AppendAsync(this CloudBlockBlob blob, Stream data, Byte[] header, Int32 maxBlockSize = 0, 121 | BlobRequestOptions options = null, OperationContext operationContext = null, 122 | CancellationToken cancellationToken = default(CancellationToken)) { 123 | 124 | if (maxBlockSize == 0) maxBlockSize = 4 * 1024 * 1024; 125 | if (data.Length > maxBlockSize) 126 | throw new ArgumentOutOfRangeException("data", "A single data object cannot be larger than " + maxBlockSize.ToString() + " bytes."); 127 | if (header != null && header.Length > maxBlockSize) 128 | throw new ArgumentOutOfRangeException("header", "The header cannot be larger than " + maxBlockSize.ToString() + " bytes."); 129 | while (true) { 130 | using (var ms = new MemoryStream()) { 131 | AccessCondition accessCondition; 132 | IEnumerable blockList = Enumerable.Empty(); 133 | try { 134 | blockList = await blob.DownloadBlockListAsync(BlockListingFilter.Committed, 135 | null, options, operationContext, cancellationToken).ConfigureAwait(false); // 404 if blob not found 136 | accessCondition = AccessCondition.GenerateIfMatchCondition(blob.Properties.ETag); // Write if blob matches what we read 137 | } 138 | catch (StorageException se) { 139 | if (!se.Matches(HttpStatusCode.NotFound, BlobErrorCodeStrings.BlobNotFound)) throw; 140 | accessCondition = AccessCondition.GenerateIfNoneMatchCondition("*"); // Write if blob doesn't exist 141 | } 142 | try { 143 | List blockIds = blockList.Select(lbi => lbi.Name).ToList(); 144 | ListBlockItem lastBlock = blockList.LastOrDefault(); 145 | if (lastBlock == null) { 146 | if (header != null) { 147 | // No blocks exist, add header (if specified) 148 | ms.Write(header, 0, header.Length); 149 | } 150 | } else { 151 | // A block exists, can it hold the new data? 152 | if (lastBlock.Length + data.Length < maxBlockSize) { 153 | // Yes, download the last block's current data as long as the blob's etag hasn't changed 154 | Int64 lastBlockOffset = blockList.Sum(lbi => lbi.Length) - lastBlock.Length; 155 | await blob.DownloadRangeToStreamAsync(ms, lastBlockOffset, lastBlock.Length, 156 | accessCondition, options, operationContext, cancellationToken).ConfigureAwait(false); // 412 if blob modified behind our back 157 | blockIds.Remove(lastBlock.Name); // Remove the block we're appending to 158 | } 159 | } 160 | await data.CopyToAsync(ms).ConfigureAwait(false); // Append new data to end of stream 161 | ms.Seek(0, SeekOrigin.Begin); 162 | // Upload new block and append it to the blob 163 | String newBlockId = Guid.NewGuid().ToString("N").Encode().ToBase64String(); 164 | blockIds.Add(newBlockId); // Append new block to end of blob 165 | await blob.PutBlockAsync(newBlockId, ms, null, accessCondition, 166 | options, operationContext, cancellationToken).ConfigureAwait(false); // PutBlock ignores access condition so this always succeeds 167 | await blob.PutBlockListAsync(blockIds, accessCondition, 168 | options, operationContext, cancellationToken).ConfigureAwait(false); // 409 if blob created behind our back; 400 if block ID doesn't exist (happens in another PC calls PutBlockList after our PutBlock) 169 | break; // If successful, we're done; don't retry 170 | } 171 | catch (StorageException se) { 172 | // Blob got created behind our back, retry 173 | if (se.Matches(HttpStatusCode.Conflict, BlobErrorCodeStrings.BlobAlreadyExists)) continue; 174 | 175 | // Blob got created or modified behind our back, retry 176 | if (se.Matches(HttpStatusCode.PreconditionFailed)) continue; 177 | 178 | // Another PC called PutBlockList between our PutBlock & PutBlockList, 179 | // our block got destroyed, retry 180 | if (se.Matches(HttpStatusCode.BadRequest, BlobErrorCodeStrings.InvalidBlockList)) continue; 181 | throw; 182 | } 183 | } 184 | } 185 | } 186 | 187 | /// Appends a block to the end of a block blob. 188 | /// The blob to append a block to. 189 | /// Additional info used when the blob is first being created. 190 | /// The data to append as a block to the end of the blob. 191 | /// The blob request options. 192 | /// The operation context. 193 | /// The cancellation token. 194 | /// A Task indicating when the operation has completed. 195 | public static async Task AppendBlockAsync(this CloudBlockBlob blob, AppendBlobBlockInfo info, 196 | Byte[] dataToAppend, BlobRequestOptions options = null, OperationContext operationContext = null, 197 | CancellationToken cancellationToken = default(CancellationToken)) { 198 | blob.Properties.ContentType = info.MimeType; 199 | //blob.Properties.ContentDisposition = "attachment; filename=\"fname.ext\""; 200 | 201 | // NOTE: Put Block ignores access condition and so it always succeeds 202 | List blockList = new List(); 203 | while (true) { 204 | blockList.Clear(); 205 | AccessCondition accessCondition; 206 | try { 207 | blockList.AddRange((await blob.DownloadBlockListAsync(BlockListingFilter.Committed, 208 | null, options, operationContext, cancellationToken).ConfigureAwait(false)).Select(lbi => lbi.Name)); // 404 if blob not found 209 | accessCondition = AccessCondition.GenerateIfMatchCondition(blob.Properties.ETag); // Write if blob matches what we read 210 | } 211 | catch (StorageException se) { 212 | if (!se.Matches(HttpStatusCode.NotFound, BlobErrorCodeStrings.BlobNotFound)) throw; 213 | accessCondition = AccessCondition.GenerateIfNoneMatchCondition("*"); // Write if blob doesn't exist 214 | } 215 | try { 216 | if (blockList.Count == 0) { // Blob doesn't exist yet, add header (if specified) 217 | // Notify client code that new blob is about to be created 218 | info.OnCreatingBlob(blob); 219 | 220 | if (info.Header != null) { // Write header to new blob 221 | String headerBlockId = Guid.NewGuid().ToString().Encode().ToBase64String(); 222 | await blob.PutBlockAsync(headerBlockId, new MemoryStream(info.Header), null).ConfigureAwait(false); 223 | blockList.Add(headerBlockId); 224 | } 225 | } 226 | // Upload new block & add it's Id to the block list 227 | String blockId = Guid.NewGuid().ToString().Encode().ToBase64String(); 228 | await blob.PutBlockAsync(blockId, new MemoryStream(dataToAppend), null).ConfigureAwait(false); 229 | blockList.Add(blockId); 230 | 231 | // If too many blocks, remove old block (but not the header if it exists) 232 | var maxEntries = info.MaxEntries + ((info.Header == null) ? 0 : 1); 233 | if (blockList.Count > maxEntries) blockList.RemoveAt((info.Header == null) ? 0 : 1); 234 | 235 | // Upload the new block list 236 | await blob.PutBlockListAsync(blockList, AccessCondition.GenerateIfMatchCondition(blob.Properties.ETag), 237 | options, operationContext, cancellationToken).ConfigureAwait(false); // 409 if blob created behind our back; 400 if block Id doesn't exist (happens in another PC calls PutBlockList after our PutBlock) 238 | break; // If successful, we're done; don't retry 239 | } 240 | catch (StorageException se) { 241 | // Blob got created behind our back, retry 242 | if (se.Matches(HttpStatusCode.Conflict, BlobErrorCodeStrings.BlobAlreadyExists)) continue; 243 | 244 | // Blob got created or modified behind our back, retry 245 | if (se.Matches(HttpStatusCode.PreconditionFailed)) continue; 246 | 247 | // Another PC called PutBlockList between our PutBlock & PutBlockList, 248 | // our block(s) got destroyed, retry 249 | if (se.Matches(HttpStatusCode.BadRequest, BlobErrorCodeStrings.InvalidBlockList)) continue; 250 | throw; 251 | } 252 | } 253 | } 254 | 255 | #if DEBUG 256 | private static void InjectFailure(CloudBlockBlob blob) { 257 | String injectBlockId = Convert.ToBase64String(Guid.NewGuid().ToString("N").Encode()); 258 | blob.PutBlock(injectBlockId, new MemoryStream("Inject create behind back".Encode()), null); 259 | blob.PutBlockList(new[] { injectBlockId }); 260 | } 261 | #endif 262 | 263 | #if DELETEME 264 | public static Uri TransformUri(this ICloudBlob blob) { 265 | // Update with Shared Access Signature if required 266 | var creds = blob.ServiceClient.Credentials; 267 | var uri = blob.Uri; 268 | return creds.NeedsTransformUri ? new Uri(creds.TransformUri(uri.ToString())) : uri; 269 | } 270 | public static void MakeRestRequest(this ICloudBlob blob, Func requestMethod, 271 | Action processRequestMethod = null) { 272 | blob.MakeRestRequest(requestMethod, processRequestMethod, response => String.Empty); 273 | } 274 | 275 | public static TResult MakeRestRequest(this ICloudBlob blob, Func requestMethod, 276 | Action processRequestMethod = null, 277 | Func processResponseMethod = null) { 278 | 279 | HttpWebRequest request = requestMethod(blob.TransformUri()); 280 | if (processRequestMethod != null) processRequestMethod(request); 281 | blob.ServiceClient.Credentials.SignRequest(request); 282 | 283 | using (WebResponse response = request.GetResponse()) { 284 | return (processResponseMethod == null) ? default(TResult) : processResponseMethod((HttpWebResponse)response); 285 | } 286 | } 287 | #endif 288 | 289 | /// Returns True for a null result segment or if the result segment has more. 290 | /// A BlobResultSegment (can be null). 291 | /// True if brs is null or if its continuation token is not null. 292 | public static Boolean HasMore(this BlobResultSegment brs) { 293 | return (brs == null) ? true : (brs.SafeContinuationToken() != null); 294 | } 295 | 296 | /// Returns null if brs is null or if its ContinuationToken is null. 297 | /// A BlobResultSegment. 298 | /// A BlobContinuationToken. 299 | public static BlobContinuationToken SafeContinuationToken(this BlobResultSegment brs) { 300 | return (brs == null) ? null : brs.ContinuationToken; 301 | } 302 | } 303 | 304 | /// An event args object passed when a new blob is created by AppendBlockAsync. 305 | public sealed class NewBlobEventArgs : EventArgs { 306 | internal NewBlobEventArgs(CloudBlockBlob blob) { Blob = blob; } 307 | /// Identifies the blob that is about to be created. 308 | public readonly CloudBlockBlob Blob; 309 | } 310 | 311 | /// Describes information required when by the AppendBlockAsync method. 312 | public sealed class AppendBlobBlockInfo { 313 | /// Indicates the Mime type to use when initially creating the blob. 314 | public String MimeType; 315 | /// Indicates the header to be written to blob when it is first created. 316 | public Byte[] Header = null; 317 | /// Indicates the maximum number of blocks the blob should have. Old blocks are deleted when going over this limit. 318 | public Int32 MaxEntries = 50000; 319 | 320 | /// Event raised when a AppendBLockAsync creates a new blob. 321 | public event EventHandler NewBlob; 322 | internal void OnCreatingBlob(CloudBlockBlob blob) { 323 | if (NewBlob != null) NewBlob(this, new NewBlobEventArgs(blob)); 324 | } 325 | } 326 | } 327 | 328 | namespace Wintellect.Azure.Storage.Blob { 329 | /// Defines extension methods on primitive types useful when working with Azure storage. 330 | public static class PrimitiveTypeExtensions { 331 | #if false 332 | /// Converts a timestamp into a string formatted like a blob's snapshot. 333 | /// The timestamp to convert. 334 | /// The string 335 | public static String ToBlobSnapshotString(this DateTimeOffset dateTime) { 336 | return dateTime.UtcDateTime.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fffffff'Z'", CultureInfo.InvariantCulture); 337 | } 338 | #endif 339 | /// Converts an integer to an Azure blob block ID string. 340 | /// The block integer ID (0-50,000). 341 | /// The integer as an Azure block ID. 342 | public static String ToBlockId(this Int32 id) { 343 | // Block ID's must all be the same length (hence the "D5" [for 50000]) and 344 | // be Base-64 encoded to be embedded in an HTTP request 345 | return id.ToString("D5").Encode().ToBase64String(); 346 | } 347 | /// Converts an Azure block ID back to its integer. 348 | /// The Azure blob block ID. 349 | /// The integer. 350 | 351 | public static Int32 FromBlockId(this String id) { 352 | // Block ID's must all be the same length (hence the "D5" [for 50000]) and 353 | // be Base-64 encoded to be embedded in an HTTP request 354 | return Int32.Parse(id.FromBase64ToBytes().Decode()); 355 | } 356 | } 357 | } 358 | 359 | #if ResumableFileUploader 360 | namespace Wintellect.Azure.Storage.Blob { 361 | public sealed class ResumableFileUploader { 362 | private readonly Int32 m_blockSize; 363 | private readonly Int32 m_concurrency; 364 | public Int32 BlockSize { get { return m_blockSize; } } 365 | public Int32 Concurrency { get { return m_concurrency; } } 366 | 367 | public ResumableFileUploader(Int32 blockSize, Int32 concurrency) { 368 | m_blockSize = blockSize; 369 | m_concurrency = concurrency; 370 | } 371 | 372 | public IAsyncResult BeginUpload(String pathname, CloudBlockBlob blob, Action blockUploaded, AsyncCallback callback = null, Object state = null) { 373 | var ae = new AsyncEnumerator(); 374 | return ae.BeginExecute(UploadFileInBlocks(ae, pathname, blob, blockUploaded), callback, state); 375 | } 376 | 377 | public void EndUpload(IAsyncResult asyncResult) { 378 | AsyncEnumerator.FromAsyncResult(asyncResult).EndExecute(asyncResult); 379 | } 380 | 381 | private IEnumerable GetBlockNumbersFromArray(Byte[] uploadedBlocks) { 382 | for (Int32 blockNumber = 0; blockNumber < uploadedBlocks.Length; blockNumber++) { 383 | // Skip already-uploaded blocks 384 | if (uploadedBlocks[blockNumber] == 0) yield return blockNumber; 385 | } 386 | } 387 | 388 | private IEnumerator UploadFileInBlocks(AsyncEnumerator ae, String pathname, CloudBlockBlob blob, Action blockUploaded) { 389 | Int64 fileLength; 390 | using (var fs = new FileStream(pathname, FileMode.Open, FileAccess.Read, FileShare.Read)) 391 | using (var mmf = MemoryMappedFile.CreateFromFile(fs, null, fileLength = fs.Length, MemoryMappedFileAccess.Read, null, HandleInheritability.None, false)) { 392 | Byte[] uploadedBlocks = new Byte[(fileLength - 1) / m_blockSize + 1]; 393 | 394 | // Find out which blocks have been uploaded already? 395 | blob.BeginDownloadBlockList(BlockListingFilter.Uncommitted, AccessCondition.GenerateEmptyCondition(), null, null, ae.End(), null); 396 | yield return 1; 397 | try { 398 | foreach (ListBlockItem lbi in blob.EndDownloadBlockList(ae.DequeueAsyncResult())) { 399 | Int32 blockId = lbi.Name.FromBase64ToInt32(); 400 | uploadedBlocks[blockId] = 1; 401 | blockUploaded(blockId); 402 | } 403 | } 404 | catch (StorageException e) { 405 | if ((HttpStatusCode)e.RequestInformation.HttpStatusCode != HttpStatusCode.NotFound) 406 | throw; 407 | } 408 | 409 | // Start uploading the remaining blocks: 410 | Int32 blocksUploading = 0; 411 | 412 | foreach (var blockNumber in GetBlockNumbersFromArray(uploadedBlocks)) { 413 | // Start uploading up to 'm_concurrency' blocks 414 | blocksUploading++; 415 | var aeBlock = new AsyncEnumerator("Block #" + blockNumber); 416 | aeBlock.BeginExecute(UploadBlock(aeBlock, mmf, blob, blockNumber, 417 | (blockNumber == uploadedBlocks.Length - 1) ? fileLength % m_blockSize : m_blockSize), ae.End(), blockNumber); 418 | if (blocksUploading < m_concurrency) continue; 419 | 420 | // As each block completes uploading, start uploading a new block (if any are left) 421 | yield return 1; 422 | blocksUploading--; 423 | IAsyncResult result = ae.DequeueAsyncResult(); 424 | AsyncEnumerator.FromAsyncResult(result).EndExecute(result); 425 | blockUploaded((Int32)result.AsyncState); 426 | } 427 | 428 | // Wait until all blocks have finished uploading and then commit them all 429 | for (; blocksUploading > 0; blocksUploading--) { 430 | yield return 1; 431 | IAsyncResult result = ae.DequeueAsyncResult(); 432 | AsyncEnumerator.FromAsyncResult(result).EndExecute(result); 433 | blockUploaded((Int32)result.AsyncState); 434 | } 435 | 436 | // Commit all the blocks in order: 437 | blob.BeginPutBlockList(Enumerable.Range(0, uploadedBlocks.Length).Select(b => b.ToBase64()), ae.End(), null); 438 | yield return 1; 439 | blob.EndPutBlockList(ae.DequeueAsyncResult()); 440 | } 441 | } 442 | 443 | private IEnumerator UploadBlock(AsyncEnumerator ae, MemoryMappedFile mmf, CloudBlockBlob blob, Int32 blockNumber, Int64 length) { 444 | using (var stream = mmf.CreateViewStream(blockNumber * m_blockSize, length, MemoryMappedFileAccess.Read)) { 445 | blob.BeginPutBlock(blockNumber.ToBase64(), stream, null, ae.End(), null); 446 | yield return 1; 447 | blob.EndPutBlock(ae.DequeueAsyncResult()); 448 | } 449 | } 450 | } 451 | } 452 | #endif 453 | 454 | #if MimeSupport 455 | #region Mime support 456 | namespace Wintellect.Azure.Storage.Blob { 457 | public static class Mime { 458 | public static String FindMimeFromData(Byte[] data, String mimeProposed) { 459 | if (data == null) throw new ArgumentNullException("data"); 460 | return FindMime(null, false, data, mimeProposed); 461 | } 462 | 463 | public static String FindMimeFromUrl(String url, Boolean urlIsFilename, string mimeProposed) { 464 | if (url == null) throw new ArgumentNullException("url"); 465 | return FindMime(url, urlIsFilename, null, mimeProposed); 466 | } 467 | 468 | public static String FindMimeFromDataAndUrl(Byte[] data, String url, Boolean urlIsFilename, string mimeProposed) { 469 | if (data == null) throw new ArgumentNullException("data"); 470 | if (url == null) throw new ArgumentNullException("url"); 471 | return FindMime(url, urlIsFilename, data, mimeProposed); 472 | } 473 | private const UInt32 E_FAIL = 0x80004005; 474 | private const UInt32 E_INVALIDARG = 0x80000003; 475 | private const UInt32 E_OUTOFMEMORY = 0x80000002; 476 | 477 | private static String FindMime(String url, Boolean urlIsFileName, Byte[] data, String mimeProposed) { 478 | IntPtr outPtr = IntPtr.Zero; 479 | Int32 ret = FindMimeFromData(IntPtr.Zero, url, data, (data == null) ? 0 : data.Length, mimeProposed, 480 | MimeFlags.EnableMimeSniffing | (urlIsFileName ? MimeFlags.UrlAsFilename : MimeFlags.Default), out outPtr, 0); 481 | if (ret == 0 && outPtr != IntPtr.Zero) { 482 | String mimeRet = Marshal.PtrToStringUni(outPtr); 483 | Marshal.FreeHGlobal(outPtr); 484 | return mimeRet.ToLowerInvariant(); 485 | } 486 | return null; 487 | } 488 | 489 | // http://msdn.microsoft.com/en-us/library/ms775147(VS.85).aspx 490 | [DllImport("UrlMon", CharSet = CharSet.Unicode, ExactSpelling = true, SetLastError = false)] 491 | private static extern Int32 FindMimeFromData( 492 | IntPtr pBC, 493 | [MarshalAs(UnmanagedType.LPWStr)] string pwzUrl, 494 | [MarshalAs(UnmanagedType.LPArray, ArraySubType = UnmanagedType.I1, SizeParamIndex = 3)] byte[] pBuffer, 495 | Int32 cbSize, 496 | [MarshalAs(UnmanagedType.LPWStr)] string pwzMimeProposed, 497 | MimeFlags dwMimeFlags, 498 | out IntPtr ppwzMimeOut, 499 | Int32 dwReserved); 500 | 501 | [Flags] 502 | public enum MimeFlags : int { 503 | /// No flags specified. Use default behavior for the function. 504 | Default = 0, 505 | /// Treat the specified pwzUrl as a file name. 506 | UrlAsFilename = 0x00000001, 507 | /// 508 | /// Use MIME-type detection even if FEATURE_MIME_SNIFFING is detected. Usually, this feature control key would disable MIME-type detection. 509 | /// Microsoft Internet Explorer 6 for Microsoft Windows XP Service Pack 2 (SP2) and later. 510 | /// 511 | EnableMimeSniffing = 0x00000002, 512 | /// 513 | /// Perform MIME-type detection if "text/plain" is proposed, even if data sniffing is otherwise disabled. Plain text may be converted to text/html if HTML tags are detected. 514 | /// Microsoft Internet Explorer 6 for Microsoft Windows XP Service Pack 2 (SP2) and later. 515 | /// 516 | IgnoreMimeTextPlain = 0x00000004, 517 | /// 518 | /// Use the authoritative MIME type specified in pwzMimeProposed. Unless IgnoreMimeTextPlain is specified, no data sniffing is performed. 519 | /// Windows Internet Explorer 8. 520 | /// 521 | ServerMime = 0x00000008 522 | } 523 | } 524 | } 525 | #endregion 526 | #endif -------------------------------------------------------------------------------- /Wintellect.Azure.Storage/Storage.Queue.cs: -------------------------------------------------------------------------------- 1 | /****************************************************************************** 2 | Module: Storage.Queue.cs 3 | Notices: Copyright (c) by Jeffrey Richter & Wintellect 4 | ******************************************************************************/ 5 | 6 | using System; 7 | using System.Collections.Generic; 8 | using System.Net; 9 | using System.Threading; 10 | using System.Threading.Tasks; 11 | using Microsoft.WindowsAzure.Storage; 12 | using Microsoft.WindowsAzure.Storage.Queue; 13 | using Microsoft.WindowsAzure.Storage.Queue.Protocol; 14 | 15 | namespace Wintellect.Azure.Storage.Queue { 16 | /// Defines extension methods useful when working with Azure storage queues. 17 | public static class QueueExtensions { 18 | /// Returns True for a null result segment or if the result segment has more. 19 | /// A QueueResultSegment (can be null). 20 | /// True if qrs is null or if its continuation token is not null. 21 | public static Boolean HasMore(this QueueResultSegment qrs) { 22 | return (qrs == null) ? true : (qrs.SafeContinuationToken() != null); 23 | } 24 | /// Returns null if qrs is null or if its ContinuationToken is null. 25 | /// A QueueResultSegment. 26 | /// A QueueContinuationToken. 27 | public static QueueContinuationToken SafeContinuationToken(this QueueResultSegment qrs) { 28 | return (qrs == null) ? null : qrs.ContinuationToken; 29 | } 30 | } 31 | } 32 | 33 | namespace Wintellect.Azure.Storage.Queue { 34 | /// A light-weight struct wrapping a CloudQueue. The methods exposed force the creation of scalable services. 35 | public struct AzureQueue { 36 | /// Implicitly converts an AzureQueue instance to a CloudQueue reference. 37 | /// The AzureQueue instance. 38 | /// The wrapped CloudQueue object reference. 39 | public static implicit operator CloudQueue(AzureQueue aq) { return aq.CloudQueue; } 40 | 41 | 42 | /// Implicitly converts a CloudQueue reference to an AzureQueue instance. 43 | /// A reference to a CloudQueue object. 44 | /// A reference to the CloudQueue object. 45 | /// An AzureQueue instance wrapping the CloudQueue object. 46 | public static implicit operator AzureQueue(CloudQueue cq) { return new AzureQueue(cq); } 47 | 48 | /// Initializes a new instance of the AzureQueue struct. 49 | /// The CloudQueue that this structure wraps. 50 | public AzureQueue(CloudQueue cloudQueue) { CloudQueue = cloudQueue; } 51 | 52 | /// Access to the wrapped CloudQueue. 53 | private readonly CloudQueue CloudQueue; 54 | 55 | /// Ensures the queue exists. Optionally deleting it first to clear it out. 56 | /// Pass true to delete the queue first. 57 | /// The same AzureQueue instance for fluent programming pattern. 58 | public async Task EnsureExistsAsync(Boolean clear = false) { 59 | await CreateIfNotExistsAsync().ConfigureAwait(false); 60 | if (clear) await ClearAsync().ConfigureAwait(false); 61 | return this; 62 | } 63 | 64 | /// Gets the approximate message count for the queue. 65 | public Int32? ApproximateMessageCount { get { return CloudQueue.ApproximateMessageCount; } } 66 | 67 | /// Gets or sets a value indicating whether to apply base64 encoding when adding or retrieving messages. 68 | public Boolean EncodeMessage { get { return CloudQueue.EncodeMessage; } set { CloudQueue.EncodeMessage = value; } } 69 | 70 | /// Gets the queue's metadata. 71 | public IDictionary Metadata { get { return CloudQueue.Metadata; } } 72 | 73 | /// Gets the queue's name. 74 | public string Name { get { return CloudQueue.Name; } } 75 | 76 | /// Gets the service client for the queue. 77 | public CloudQueueClient ServiceClient { get { return CloudQueue.ServiceClient; } } 78 | 79 | /// Gets the queue's URI. 80 | public Uri Uri { get { return CloudQueue.Uri; } } 81 | 82 | /// 83 | /// Returns a task that performs an asynchronous operation to add a message to the queue. 84 | /// 85 | /// The message to add. 86 | /// The maximum time to allow the message to be in the queue, or null. 87 | /// The length of time from now during which the message will be invisible. If null then the message will be visible immediately. 88 | /// A Microsoft.WindowsAzure.Storage.Queue.QueueRequestOptions object that specifies additional options for the request. 89 | /// An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation. 90 | /// A System.Threading.CancellationToken to observe while waiting for a task to complete. 91 | /// A System.Threading.Tasks.Task object that represents the current operation. 92 | [DoesServiceRequest] 93 | public Task AddMessageAsync(CloudQueueMessage message, TimeSpan? timeToLive = null, TimeSpan? initialVisibilityDelay = null, QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken)) { 94 | return CloudQueue.AddMessageAsync(message, timeToLive, initialVisibilityDelay, options, operationContext, cancellationToken); 95 | } 96 | 97 | /// 98 | /// Returns a task that performs an asynchronous operation to clear all messages from the queue. 99 | /// 100 | /// A Microsoft.WindowsAzure.Storage.Queue.QueueRequestOptions object that specifies additional options for the request. 101 | /// An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation. 102 | /// A System.Threading.CancellationToken to observe while waiting for a task to complete. 103 | /// A System.Threading.Tasks.Task object that represents the current operation. 104 | [DoesServiceRequest] 105 | public Task ClearAsync(QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken)) { 106 | return CloudQueue.ClearAsync(options, operationContext, cancellationToken); 107 | } 108 | 109 | /// 110 | /// Returns a task that performs an asynchronous operation to create a queue. 111 | /// 112 | /// A Microsoft.WindowsAzure.Storage.Queue.QueueRequestOptions object that specifies additional options for the request. 113 | /// An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation. 114 | /// A System.Threading.CancellationToken to observe while waiting for a task to complete. 115 | /// A System.Threading.Tasks.Task object that represents the current operation. 116 | [DoesServiceRequest] 117 | public Task CreateAsync(QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken)) { 118 | return CloudQueue.CreateAsync(options, operationContext, cancellationToken); 119 | } 120 | 121 | /// 122 | /// Returns a task that performs an asynchronous request to create the queue if it does not already exist. 123 | /// 124 | /// A Microsoft.WindowsAzure.Storage.Queue.QueueRequestOptions object that specifies additional options for the request. 125 | /// An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation. 126 | /// A System.Threading.CancellationToken to observe while waiting for a task to complete. 127 | /// A System.Threading.Tasks.Task object that represents the current operation. 128 | [DoesServiceRequest] 129 | public Task CreateIfNotExistsAsync(QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken)) { 130 | return CloudQueue.CreateIfNotExistsAsync(options, operationContext, cancellationToken); 131 | } 132 | 133 | /// 134 | /// Returns a task that performs an asynchronous operation to delete a queue. 135 | /// 136 | /// A Microsoft.WindowsAzure.Storage.Queue.QueueRequestOptions object that specifies additional options for the request. 137 | /// An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation. 138 | /// A System.Threading.CancellationToken to observe while waiting for a task to complete. 139 | /// A System.Threading.Tasks.Task object that represents the current operation. 140 | [DoesServiceRequest] 141 | public Task DeleteAsync(QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken)) { 142 | return CloudQueue.DeleteAsync(options, operationContext, cancellationToken); 143 | } 144 | 145 | /// 146 | /// Returns a task that performs an asynchronous request to delete the queue if it already exists. 147 | /// 148 | /// A Microsoft.WindowsAzure.Storage.Queue.QueueRequestOptions object that specifies additional options for the request. 149 | /// An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation. 150 | /// A System.Threading.CancellationToken to observe while waiting for a task to complete. 151 | /// A System.Threading.Tasks.Task object that represents the current operation. 152 | [DoesServiceRequest] 153 | public Task DeleteIfExistsAsync(QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken)) { 154 | return CloudQueue.DeleteIfExistsAsync(options, operationContext, cancellationToken); 155 | } 156 | 157 | /// 158 | /// Returns a task that performs an asynchronous operation to delete a message. 159 | /// 160 | /// The message ID. 161 | /// The pop receipt value. 162 | /// A Microsoft.WindowsAzure.Storage.Queue.QueueRequestOptions object that specifies additional options for the request. 163 | /// An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation. 164 | /// A System.Threading.CancellationToken to observe while waiting for a task to complete. 165 | /// A System.Threading.Tasks.Task object that represents the current operation. 166 | [DoesServiceRequest] 167 | public async Task DeleteMessageAsync(String messageId, String popReceipt, QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken)) { 168 | try { 169 | await CloudQueue.DeleteMessageAsync(messageId, popReceipt, options, operationContext, cancellationToken).ConfigureAwait(false); 170 | return true; 171 | } 172 | catch (StorageException e) { 173 | if (!e.Matches(HttpStatusCode.NotFound, QueueErrorCodeStrings.MessageNotFound)) throw; 174 | // pop receipt must be invalid; ignore or log (so we can tune the visibility timeout) 175 | return false; 176 | } 177 | } 178 | 179 | /// 180 | /// Returns a task that performs an asynchronous operation to delete a message. 181 | /// 182 | /// The message. 183 | /// A Microsoft.WindowsAzure.Storage.Queue.QueueRequestOptions object that specifies additional options for the request. 184 | /// An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation. 185 | /// A System.Threading.CancellationToken to observe while waiting for a task to complete. 186 | /// A System.Threading.Tasks.Task object that represents the current operation. 187 | public Task DeleteMessageAsync(CloudQueueMessage message, QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken)) { 188 | return DeleteMessageAsync(message.Id, message.PopReceipt, options, operationContext, cancellationToken); 189 | } 190 | 191 | /// 192 | /// Returns a task that performs an asynchronous request to check existence of the queue. 193 | /// 194 | /// A Microsoft.WindowsAzure.Storage.Queue.QueueRequestOptions object that specifies additional options for the request. 195 | /// An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation. 196 | /// A System.Threading.CancellationToken to observe while waiting for a task to complete. 197 | /// A System.Threading.Tasks.Task object that represents the current operation. 198 | [DoesServiceRequest] 199 | public Task ExistsAsync(QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken)) { 200 | return CloudQueue.ExistsAsync(options, operationContext, cancellationToken); 201 | } 202 | 203 | /// 204 | /// Returns a task that performs an asynchronous operation to fetch the queue's attributes. 205 | /// 206 | /// A Microsoft.WindowsAzure.Storage.Queue.QueueRequestOptions object that specifies additional options for the request. 207 | /// An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation. 208 | /// A System.Threading.CancellationToken to observe while waiting for a task to complete. 209 | /// A System.Threading.Tasks.Task object that represents the current operation. 210 | [DoesServiceRequest] 211 | public Task FetchAttributesAsync(QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken)) { 212 | return CloudQueue.FetchAttributesAsync(options, operationContext, cancellationToken); 213 | } 214 | 215 | /// 216 | /// Returns a task that performs an asynchronous operation to get a single message 217 | /// from the queue, and specifies how long the message should be reserved before 218 | /// it becomes visible, and therefore available for deletion. 219 | /// 220 | /// The visibility timeout interval. 221 | /// A Microsoft.WindowsAzure.Storage.Queue.QueueRequestOptions object that specifies additional options for the request. 222 | /// An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation. 223 | /// A System.Threading.CancellationToken to observe while waiting for a task to complete. 224 | /// A System.Threading.Tasks.Task object that represents the current operation. 225 | [DoesServiceRequest] 226 | public Task GetMessageAsync(TimeSpan? visibilityTimeout = null, QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken)) { 227 | return CloudQueue.GetMessageAsync(visibilityTimeout, options, operationContext, cancellationToken); 228 | } 229 | 230 | /// 231 | /// Returns a task that performs an asynchronous operation to get the specified 232 | /// number of messages from the queue using the specified request options and 233 | /// operation context. This operation marks the retrieved messages as invisible 234 | /// in the queue for the default visibility timeout period. 235 | /// 236 | /// The number of messages to retrieve. 237 | /// The visibility timeout interval. 238 | /// A Microsoft.WindowsAzure.Storage.Queue.QueueRequestOptions object that specifies additional options for the request. 239 | /// An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation. 240 | /// A System.Threading.CancellationToken to observe while waiting for a task to complete. 241 | /// A System.Threading.Tasks.Task object that represents the current operation. 242 | [DoesServiceRequest] 243 | public Task> GetMessagesAsync(int messageCount, TimeSpan? visibilityTimeout = null, QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken)) { 244 | return CloudQueue.GetMessagesAsync(messageCount, visibilityTimeout, options, operationContext, cancellationToken); 245 | } 246 | 247 | /// 248 | /// Returns a task that performs an asynchronous request to get the permissions settings for the queue. 249 | /// 250 | /// A Microsoft.WindowsAzure.Storage.Queue.QueueRequestOptions object that specifies additional options for the request. 251 | /// An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation. 252 | /// A System.Threading.CancellationToken to observe while waiting for a task to complete. 253 | /// A System.Threading.Tasks.Task object that represents the current operation. 254 | [DoesServiceRequest] 255 | public Task GetPermissionsAsync(QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken)) { 256 | return CloudQueue.GetPermissionsAsync(options, operationContext, cancellationToken); 257 | } 258 | 259 | /// 260 | /// Returns a shared access signature for the queue. 261 | /// 262 | /// The access policy for the shared access signature. 263 | /// A queue-level access policy. 264 | /// The query string returned includes the leading question mark. 265 | public string GetSharedAccessSignature(SharedAccessQueuePolicy policy, string accessPolicyIdentifier) { 266 | return CloudQueue.GetSharedAccessSignature(policy, accessPolicyIdentifier); 267 | } 268 | 269 | /// 270 | /// Returns a task that performs an asynchronous operation to get a single message from the queue. 271 | /// 272 | /// A Microsoft.WindowsAzure.Storage.Queue.QueueRequestOptions object that specifies additional options for the request. 273 | /// An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation. 274 | /// A System.Threading.CancellationToken to observe while waiting for a task to complete. 275 | /// A System.Threading.Tasks.Task object that represents the current operation. 276 | [DoesServiceRequest] 277 | public Task PeekMessageAsync(QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken)) { 278 | return CloudQueue.PeekMessageAsync(options, operationContext, cancellationToken); 279 | } 280 | 281 | /// 282 | /// Returns a task that performs an asynchronous operation to peek messages from the queue. 283 | /// 284 | /// The number of messages to peek. 285 | /// A Microsoft.WindowsAzure.Storage.Queue.QueueRequestOptions object that specifies additional options for the request. 286 | /// An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation. 287 | /// A System.Threading.CancellationToken to observe while waiting for a task to complete. 288 | /// A System.Threading.Tasks.Task object that represents the current operation. 289 | [DoesServiceRequest] 290 | public Task> PeekMessagesAsync(int messageCount, QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken)) { 291 | return CloudQueue.PeekMessagesAsync(messageCount, options, operationContext, cancellationToken); 292 | } 293 | 294 | /// 295 | /// Returns a task that performs an asynchronous operation to set user-defined metadata on the queue. 296 | /// 297 | /// A Microsoft.WindowsAzure.Storage.Queue.QueueRequestOptions object that specifies additional options for the request. 298 | /// An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation. 299 | /// A System.Threading.CancellationToken to observe while waiting for a task to complete. 300 | /// A System.Threading.Tasks.Task object that represents the current operation. 301 | [DoesServiceRequest] 302 | public Task SetMetadataAsync(QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken)) { 303 | return CloudQueue.SetMetadataAsync(options, operationContext, cancellationToken); 304 | } 305 | 306 | /// 307 | /// Returns a task that performs an asynchronous request to set permissions for the queue. 308 | /// 309 | /// The permissions to apply to the queue. 310 | /// A Microsoft.WindowsAzure.Storage.Queue.QueueRequestOptions object that specifies additional options for the request. 311 | /// An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation. 312 | /// A System.Threading.CancellationToken to observe while waiting for a task to complete. 313 | /// A System.Threading.Tasks.Task object that represents the current operation. 314 | [DoesServiceRequest] 315 | public Task SetPermissionsAsync(QueuePermissions permissions, QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken)) { 316 | return CloudQueue.SetPermissionsAsync(permissions, options, operationContext, cancellationToken); 317 | } 318 | 319 | /// 320 | /// Returns a task that performs an asynchronous operation to update the visibility timeout and optionally the content of a message. 321 | /// 322 | /// The message to update. 323 | /// The visibility timeout interval. 324 | /// An EnumSet of Microsoft.WindowsAzure.Storage.Queue.MessageUpdateFields values that specifies which parts of the message are to be updated. 325 | /// A Microsoft.WindowsAzure.Storage.Queue.QueueRequestOptions object that specifies additional options for the request. 326 | /// An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation. 327 | /// A System.Threading.CancellationToken to observe while waiting for a task to complete. 328 | /// A System.Threading.Tasks.Task object that represents the current operation. 329 | [DoesServiceRequest] 330 | public Task UpdateMessageAsync(CloudQueueMessage message, TimeSpan visibilityTimeout, MessageUpdateFields updateFields, QueueRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken)) { 331 | return CloudQueue.UpdateMessageAsync(message, visibilityTimeout, updateFields, options, operationContext, cancellationToken); 332 | } 333 | /// Returns a string that represents the current object. 334 | public override string ToString() { return CloudQueue.ToString(); } 335 | /// Determines whether the specified System.Object is equal to the current System.Object. 336 | /// The object to compare with the current object. 337 | /// true if the specified object is equal to the current object; otherwise, false. 338 | public override bool Equals(object obj) { return CloudQueue.Equals(obj); } 339 | /// Serves as a hash function for a particular type. 340 | /// A hash code for the current System.Object. 341 | public override int GetHashCode() { return CloudQueue.GetHashCode(); } 342 | } 343 | } -------------------------------------------------------------------------------- /Wintellect.Azure.Storage/Storage.Sas.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Globalization; 3 | using System.Runtime.InteropServices.WindowsRuntime; 4 | using System.Text; 5 | using Windows.Security.Cryptography; 6 | using Windows.Security.Cryptography.Core; 7 | using Windows.Storage.Streams; 8 | 9 | namespace Wintellect.Azure.Storage.Sas { 10 | // Service versioning: http://msdn.microsoft.com/en-us/library/windowsazure/dd894041.aspx 11 | // Constructing the Shared Access Signature URI: http://msdn.microsoft.com/en-us/library/windowsazure/dn140255.aspx 12 | // Permissions order: Container=rwdl, Blob=rwd, Queue=raup, Table=raud 13 | /// The set of blob container permissions. 14 | [Flags] 15 | public enum SasContainerPermissions { 16 | /// No permission. 17 | None = 0, 18 | /// Read permission. 19 | Read = 'r', 20 | /// Write permission. 21 | Write = 'w' << 8, 22 | /// Delete permission. 23 | Delete = 'd' << 16, 24 | /// List permission. 25 | List = 'l' << 24 26 | } 27 | /// The set of blob permissions. 28 | [Flags] 29 | public enum SasBlobPermissions { 30 | /// No permission. 31 | None = 0, 32 | /// Read permission. 33 | Read = 'r', 34 | /// Write permission. 35 | Write = 'w' << 8, 36 | /// Delete permission. 37 | Delete = 'd' << 16 38 | } 39 | /// The set of queue permissions. 40 | [Flags] 41 | public enum SasQueuePermissions { 42 | /// No permission. 43 | None = 0, 44 | /// Read permission. 45 | Read = 'r', 46 | /// Add permission. 47 | Add = 'a' << 8, 48 | /// Update permission. 49 | Update = 'u' << 16, 50 | /// Process messages permission. 51 | ProcessMessages = 'p' << 24 52 | } 53 | /// The set of table permissions. 54 | [Flags] 55 | public enum SasTablePermissions { 56 | /// No permission. 57 | None = 0, 58 | /// Query permission. 59 | Query = 'r', 60 | /// Add permission. 61 | Add = 'a' << 8, 62 | /// Update permission. 63 | Update = 'u' << 16, 64 | /// Delete permission. 65 | Delete = 'd' << 24 66 | } 67 | 68 | /// Defines methods for manipulating Azure storage SAS strings. 69 | public static class AzureStorageSas { 70 | /// Parses a SAS URL returning its start and end times. 71 | /// The SAS URL to parse. 72 | /// The start time in the SAS URL. 73 | /// The end time in the SAS URL. 74 | public static void ParseSasUrlTimes(String sasUrl, out DateTimeOffset startTime, out DateTimeOffset endTime) { 75 | startTime = GetTimeAfterPrefix("st=", sasUrl) ?? DateTimeOffset.UtcNow; 76 | endTime = GetTimeAfterPrefix("se=", sasUrl) ?? startTime.AddHours(1); 77 | } 78 | 79 | private static DateTimeOffset? GetTimeAfterPrefix(String prefix, String url) { 80 | Int32 startIndex = url.IndexOf(prefix); 81 | if (startIndex == -1) return null; 82 | Int32 endIndex = url.IndexOf("&", startIndex += prefix.Length); 83 | if (endIndex == -1) return null; 84 | String time = url.Substring(startIndex, endIndex - startIndex).Replace("%3A", ":"); 85 | DateTimeOffset dt; 86 | 87 | if (!DateTimeOffset.TryParse(time, null, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out dt)) 88 | return null; 89 | return dt; 90 | } 91 | 92 | // URL = https://myaccount.blob.core.windows.net/music: canonicalizedresource = "/myaccount/music" 93 | // URL = https://myaccount.blob.core.windows.net/music/intro.mp3: canonicalizedresource = "/myaccount/music/intro.mp3" 94 | // URL = https://myaccount.queue.core.windows.net/thumbnails: canonicalizedresource = "/myaccount/thumbnails" 95 | // URL = https://myaccount.table.core.windows.net/Employees(PartitionKey='Jeff',RowKey='Price') canonicalizedresource = "/myaccount/employees" 96 | private static String GetCanonicalName(String account, String containerTableOrQueueName, String blobName = null) { 97 | // For container & queue: return "/" + accountName + absolutePath; 98 | // For a blob 99 | String str = "/" + account + "/" + containerTableOrQueueName.ToLowerInvariant(); 100 | if (blobName != null) str += "/" + blobName.Replace('\\', '/'); 101 | return str; 102 | } 103 | 104 | /// Creates a SAS string for a blob container. 105 | /// Identifies the Azure storage account's name. 106 | /// Identifies the Azure storage accounts key. 107 | /// The SAS version. 108 | /// The name of the blob container. 109 | /// An optional signature identifier. 110 | /// The desired permissions. 111 | /// The optional start time. 112 | /// The optional end time. 113 | /// The SAS string. 114 | public static String CreateContainerSas(String accountName, IBuffer accountKey, String version, String containerName, 115 | String signatureIdentifier, SasContainerPermissions permissions, DateTimeOffset? startTime, DateTimeOffset? expiryTime) { 116 | String canonicalResourceName = GetCanonicalName(accountName, containerName); 117 | return CreateSas(accountKey, version, "c", canonicalResourceName, signatureIdentifier, 118 | (UInt32)permissions, startTime, expiryTime); 119 | } 120 | 121 | /// Creates a SAS string for a blob. 122 | /// Identifies the Azure storage account's name. 123 | /// Identifies the Azure storage accounts key. 124 | /// The SAS version. 125 | /// The name of the blob container. 126 | /// The name of the blob within the container. 127 | /// An optional signature identifier. 128 | /// The desired permissions. 129 | /// The optional start time. 130 | /// The optional end time. 131 | /// The SAS string. 132 | public static String CreateBlobSas(String accountName, IBuffer accountKey, String version, String containerName, String blobName, 133 | String signatureIdentifier, SasBlobPermissions permissions, DateTimeOffset? startTime, DateTimeOffset? expiryTime) { 134 | String canonicalResourceName = GetCanonicalName(accountName, containerName, blobName); 135 | return CreateSas(accountKey, version, "b", canonicalResourceName, signatureIdentifier, 136 | (UInt32)permissions, startTime, expiryTime); 137 | } 138 | 139 | /// Creates a SAS string for a queue. 140 | /// Identifies the Azure storage account's name. 141 | /// Identifies the Azure storage accounts key. 142 | /// The SAS version. 143 | /// The name of the queue. 144 | /// An optional signature identifier. 145 | /// The desired permissions. 146 | /// The optional start time. 147 | /// The optional end time. 148 | /// The SAS string. 149 | public static String CreateQueueSas(String accountName, IBuffer accountKey, String version, String queueName, 150 | String signatureIdentifier, SasQueuePermissions permissions, DateTimeOffset? startTime, DateTimeOffset? expiryTime) { 151 | String canonicalResourceName = GetCanonicalName(accountName, queueName); 152 | return CreateSas(accountKey, version, null, canonicalResourceName, signatureIdentifier, 153 | (UInt32)permissions, startTime, expiryTime); 154 | } 155 | /// Creates a SAS string for a table. 156 | /// Identifies the Azure storage account's name. 157 | /// Identifies the Azure storage accounts key. 158 | /// The SAS version. 159 | /// The name of the table. 160 | /// An optional signature identifier. 161 | /// The desired permissions. 162 | /// The optional start time. 163 | /// The optional end time. 164 | /// The optional start partition key. 165 | /// The optional start row key. 166 | /// The optional end partition key. 167 | /// The optional end row key. 168 | /// The SAS string. 169 | public static String CreateTableSas(String accountName, IBuffer accountKey, String version, String tableName, 170 | String signatureIdentifier, SasTablePermissions permissions, DateTimeOffset? startTime, DateTimeOffset? expiryTime, 171 | String startPartitionKey = null, String startRowKey = null, String endPartitionKey = null, String endRowKey = null) { 172 | String canonicalResourceName = GetCanonicalName(accountName, tableName); 173 | return CreateSas(accountKey, version, null, canonicalResourceName, signatureIdentifier, 174 | (UInt32)permissions, startTime, expiryTime, 175 | tableName, startPartitionKey, startRowKey, endPartitionKey, endRowKey); 176 | } 177 | private static String ToPermissionString(UInt32 permissions) { 178 | String perm = String.Empty; 179 | for (Char c = (Char)(permissions & 0xff); permissions != 0; c = (Char)((permissions >>= 8) & 0xff)) 180 | if (c != 0) perm += c.ToString(); 181 | return perm; 182 | } 183 | 184 | private static String GetDateTimeOrEmpty(DateTimeOffset? value) { 185 | return value.HasValue ? value.Value.UtcDateTime.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture) : String.Empty; 186 | } 187 | private static String CreateSas(IBuffer accountKey, String version, 188 | String resourceType, String canonicalResourceName, 189 | String signatureIdentifier, UInt32 permissions, DateTimeOffset? startTime, DateTimeOffset? expiryTime, 190 | String tableName = null, 191 | String startPartitionKey = null, String startRowKey = null, 192 | String endPartitionKey = null, String endRowKey = null) { 193 | 194 | String signature; 195 | #if true 196 | MacAlgorithmProvider mac = MacAlgorithmProvider.OpenAlgorithm(MacAlgorithmNames.HmacSha256); 197 | CryptographicHash hash = mac.CreateHash(accountKey); 198 | // String to sign: http://msdn.microsoft.com/en-us/library/azure/dn140255.aspx 199 | StringBuilder stringToSign = new StringBuilder(); 200 | stringToSign.Append(ToPermissionString(permissions)) 201 | .Append("\n").Append(GetDateTimeOrEmpty(startTime)) 202 | .Append("\n").Append(GetDateTimeOrEmpty(expiryTime)) 203 | .Append("\n").Append(canonicalResourceName) 204 | .Append("\n").Append(signatureIdentifier) 205 | .Append("\n").Append(version); 206 | if (version != "2012-02-12") { 207 | String CacheControl = String.Empty; // rscc 208 | String ContentDisposition = String.Empty; // rscd 209 | String ContentEncoding = String.Empty; // rsce 210 | String ContentLanguage = String.Empty; // rscl 211 | String ContentType = String.Empty; // rsct 212 | stringToSign.Append("\n").Append(CacheControl) 213 | .Append("\n").Append(ContentDisposition) 214 | .Append("\n").Append(ContentEncoding) 215 | .Append("\n").Append(ContentLanguage) 216 | .Append("\n").Append(ContentType); 217 | } 218 | if (tableName != null) { 219 | stringToSign.Append("\n").Append(startPartitionKey) 220 | .Append("\n").Append(startRowKey) 221 | .Append("\n").Append(endPartitionKey) 222 | .Append("\n").Append(endRowKey); 223 | } 224 | hash.Append(stringToSign.ToString().Encode().AsBuffer()); 225 | signature = CryptographicBuffer.EncodeToBase64String(hash.GetValueAndReset()); 226 | #else 227 | using (HashAlgorithm hashAlgorithm = new HMACSHA256(accountKey)) { 228 | String stringToSign = String.Format(CultureInfo.InvariantCulture, "{0}\n{1}\n{2}\n{3}\n{4}\n{5}", 229 | permissions, GetDateTimeOrEmpty(startTime), GetDateTimeOrEmpty(expiryTime), 230 | canonicalResourceName, signatureIdentifier, version); 231 | if (tableName != null) 232 | stringToSign = String.Format(CultureInfo.InvariantCulture, "{0}\n{1}\n{2}\n{3}\n{4}", 233 | stringToSign, startPartitionKey, startRowKey, endPartitionKey, endRowKey); 234 | signature = Convert.ToBase64String(hashAlgorithm.ComputeHash(Encoding.UTF8.GetBytes(stringToSign))); 235 | } 236 | #endif 237 | // NOTE: The order of query parameters is important 238 | return new StringBuilder() 239 | .Add("sv", version) 240 | .Add("st", startTime == null ? null : GetDateTimeOrEmpty(startTime)) 241 | .Add("se", expiryTime == null ? null : GetDateTimeOrEmpty(expiryTime)) 242 | .Add("sr", resourceType) 243 | .Add("tn", tableName) 244 | .Add("sp", ToPermissionString(permissions)) 245 | .Add("spk", startPartitionKey).Add("srk", startRowKey) 246 | .Add("epk", endPartitionKey).Add("erk", endRowKey) 247 | .Add("si", signatureIdentifier) 248 | .Add("sk", null/*accountKeyName*/) 249 | .Add("sig", signature).ToString(); 250 | } 251 | private static StringBuilder Add(this StringBuilder sb, String key, String value) { 252 | if (String.IsNullOrWhiteSpace(value)) return sb; 253 | sb.Append(sb.Length == 0 ? "?" : "&"); // delimiter 254 | sb.Append(key + "=" + Uri.EscapeDataString(value)); 255 | return sb; 256 | } 257 | } 258 | } -------------------------------------------------------------------------------- /Wintellect.Azure.Storage/Storage.Table.PropertySerializer.cs: -------------------------------------------------------------------------------- 1 | /****************************************************************************** 2 | Module: Storage.Table.PropertySerializer.cs 3 | Notices: Copyright (c) by Jeffrey Richter & Wintellect 4 | ******************************************************************************/ 5 | 6 | using System; 7 | using System.Collections.Generic; 8 | using System.Linq; 9 | using System.Text; 10 | 11 | namespace Wintellect.Azure.Storage.Table { 12 | /// Use this class to serialize a collection of records to an Azure table property. 13 | public sealed class PropertySerializer { 14 | /// Serializes a collection of records to an Azure table property. 15 | /// The type of records in the collection. 16 | /// The maximum number of records to serialize. 17 | /// The version number of the serialization format. 18 | /// The collection of records. 19 | /// The approximate record size in bytes. This is a hint to improve performance. 20 | /// The method that serializes a single record to bytes. 21 | /// The array of bytes containing the serialized records. 22 | public static Byte[] ToProperty(Int32 maxRecordsPerEntity, UInt16 version, 23 | IEnumerable records, Int32 approxRecordSize, 24 | Action recordToProperty) { 25 | Int32 maxRecords = Math.Min(records.Count() + 1, maxRecordsPerEntity); 26 | var serializer = new PropertySerializer(records.Count() * approxRecordSize).Add(version); // Put version number in 27 | Int32 count = 0; 28 | foreach (TRecord record in records) { 29 | if (count++ >= maxRecords) break; // Don't go above 'maxRecords' 30 | recordToProperty(record, version, serializer); 31 | } 32 | return serializer.ToArray(); 33 | } 34 | 35 | private static readonly Encoding s_utf8 = Encoding.UTF8; 36 | private Byte[] m_bytes; 37 | private Int32 m_size = 0; 38 | 39 | /// Returns the number of bytes in serialized into the byte array. 40 | public Int32 Size { get { return m_size; } } 41 | 42 | /// Gets and sets the byte array's capacity. 43 | public Int32 Capacity { 44 | get { return m_bytes.Length; } 45 | set { 46 | if (value != m_bytes.Length) { 47 | if (value < m_size) { 48 | throw new ArgumentOutOfRangeException("Attempt to set capacity smaller than what's currently in use."); 49 | } 50 | Array.Resize(ref m_bytes, value); 51 | } 52 | } 53 | } 54 | /// Constructs a PropertySerializer setting its initial capacity. 55 | /// The initialize size of the internal byte array which the records get serialized into. 56 | public PropertySerializer(Int32 initialCapacity) { m_bytes = new Byte[initialCapacity]; } 57 | private void EnsureCapacity(Int32 min) { 58 | if (m_bytes.Length < min) Capacity = Math.Max(min, m_bytes.Length * 2); 59 | } 60 | private PropertySerializer Add(UInt64 value, Int32 numBytes) { 61 | EnsureCapacity(m_size + numBytes); 62 | for (Int32 n = 0; n < numBytes; n++) { 63 | m_bytes[m_size++] = (Byte)(value & 0xff); 64 | value >>= 8; 65 | } 66 | return this; 67 | } 68 | 69 | /// Serializes the value into the byte array. 70 | /// The value to serialize. 71 | /// The same PropertySerializer so you can chain Add calls together. 72 | public PropertySerializer Add(Byte value) { return Add(value, sizeof(Byte)); } 73 | /// Serializes the value into the byte array. 74 | /// The value to serialize. 75 | /// The same PropertySerializer so you can chain Add calls together. 76 | public PropertySerializer Add(Int16 value) { return Add(unchecked((UInt64)value), sizeof(Int16)); } 77 | /// Serializes the value into the byte array. 78 | /// The value to serialize. 79 | /// The same PropertySerializer so you can chain Add calls together. 80 | public PropertySerializer Add(UInt16 value) { return Add(value, sizeof(UInt16)); } 81 | /// Serializes the value into the byte array. 82 | /// The value to serialize. 83 | /// The same PropertySerializer so you can chain Add calls together. 84 | public PropertySerializer Add(Int32 value) { return Add(unchecked((UInt64)value), sizeof(Int32)); } 85 | /// Serializes the value into the byte array. 86 | /// The value to serialize. 87 | /// The same PropertySerializer so you can chain Add calls together. 88 | public PropertySerializer Add(UInt32 value) { return Add(value, sizeof(UInt32)); } 89 | /// Serializes the value into the byte array. 90 | /// The value to serialize. 91 | /// The same PropertySerializer so you can chain Add calls together. 92 | public PropertySerializer Add(Int64 value) { return Add(unchecked((UInt64)value), sizeof(Int64)); } 93 | /// Serializes the value into the byte array. 94 | /// The value to serialize. 95 | /// The same PropertySerializer so you can chain Add calls together. 96 | public PropertySerializer Add(UInt64 value) { return Add(unchecked(value), sizeof(Int64)); } 97 | /// Serializes the value into the byte array. 98 | /// The value to serialize. 99 | /// The maximum characters you expect any string. 100 | /// The same PropertySerializer so you can chain Add calls together. 101 | public PropertySerializer Add(String value, Int32 maxChars) { 102 | if (maxChars < 1 || maxChars > 65535) 103 | throw new ArgumentOutOfRangeException("maxChars", "maxChars must >0 and <=65535"); 104 | if (value.Length > maxChars) 105 | throw new ArgumentException(String.Format("value > {0} characters", maxChars.ToString())); 106 | 107 | Int32 byteCount = s_utf8.GetByteCount(value); 108 | Add(unchecked((UInt32) byteCount), maxChars < 256 ? sizeof(Byte) : sizeof(UInt16)); 109 | EnsureCapacity(m_size + byteCount); 110 | s_utf8.GetBytes(value, 0, value.Length, m_bytes, m_size); 111 | m_size += byteCount; 112 | return this; 113 | } 114 | /// Returns the byte array containing the serialized collection of records./// 115 | public Byte[] ToArray() { Capacity = m_size; return m_bytes; } 116 | } 117 | 118 | /// Use this class to deserialize a collection of records from an Azure table property. 119 | public sealed class PropertyDeserializer { 120 | /// Deserializes a collection of records from an Azure table property. 121 | /// The type of records in the collection. 122 | /// The Azure table property's byte array. 123 | /// The method that deserializes bytes to a single record. 124 | /// The collection of records. 125 | public static List FromProperty(Byte[] propertyBytes, Func propertyToRecord) { 126 | var records = new List(); 127 | if (propertyBytes == null || propertyBytes.Length == 0) return records; 128 | 129 | var deserializer = new PropertyDeserializer(propertyBytes); 130 | UInt16 version = deserializer.ToUInt16(); 131 | 132 | // If version != 0, this property is before we had version #s at all, so upgrade to version-supporting 133 | while (deserializer.BytesRemaining > 0) 134 | records.Add(propertyToRecord(version, deserializer)); 135 | return records; 136 | } 137 | 138 | private readonly Byte[] m_bytes; 139 | private Int32 m_offset = 0; 140 | /// Constructs a PropertyDeserializer over a byte array and starting offset within the byte array. 141 | /// The byte array to deserialize records from. 142 | /// The starting offset into the byte array. 143 | public PropertyDeserializer(Byte[] bytes, Int32 startOffset = 0) { m_bytes = bytes; m_offset = startOffset; } 144 | 145 | /// Returns the number of bytes remaining to be deserialized. 146 | public Int32 BytesRemaining { get { return m_bytes.Length - m_offset; } } 147 | //public PropertyDeserializer Backup(Int32 numBytes) { m_offset -= numBytes; return this; } 148 | 149 | /// Deserializes a value from the byte array. 150 | /// The deserialized value. 151 | public Byte ToByte() { 152 | Byte v = m_bytes[m_offset]; 153 | m_offset += sizeof(Byte); 154 | return v; 155 | } 156 | /// Deserializes a value from the byte array. 157 | /// The deserialized value. 158 | public UInt16 ToUInt16() { 159 | UInt16 v = BitConverter.ToUInt16(m_bytes, m_offset); 160 | m_offset += sizeof(UInt16); 161 | return v; 162 | } 163 | /// Deserializes a value from the byte array. 164 | /// The deserialized value. 165 | public Int16 ToInt16() { 166 | Int16 v = BitConverter.ToInt16(m_bytes, m_offset); 167 | m_offset += sizeof(Int16); 168 | return v; 169 | } 170 | /// Deserializes a value from the byte array. 171 | /// The deserialized value. 172 | public UInt32 ToUInt32() { 173 | UInt32 v = BitConverter.ToUInt32(m_bytes, m_offset); 174 | m_offset += sizeof(UInt32); 175 | return v; 176 | } 177 | /// Deserializes a value from the byte array. 178 | /// The deserialized value. 179 | public Int32 ToInt32() { 180 | Int32 v = BitConverter.ToInt32(m_bytes, m_offset); 181 | m_offset += sizeof(Int32); 182 | return v; 183 | } 184 | /// Deserializes a value from the byte array. 185 | /// The deserialized value. 186 | public UInt64 ToUInt64() { 187 | UInt64 v = BitConverter.ToUInt64(m_bytes, m_offset); 188 | m_offset += sizeof(UInt64); 189 | return v; 190 | } 191 | /// Deserializes a value from the byte array. 192 | /// The deserialized value. 193 | public Int64 ToInt64() { 194 | Int64 v = BitConverter.ToInt64(m_bytes, m_offset); 195 | m_offset += sizeof(UInt64); 196 | return v; 197 | } 198 | /// Deserializes a value from the byte array. 199 | /// The maximum number of characters in the string. 200 | /// The deserialized value. 201 | public String ToString(Int32 maxChars) { 202 | if (maxChars < 1 || maxChars > 65535) 203 | throw new ArgumentOutOfRangeException("maxChars", "maxChars must >0 and <65536"); 204 | 205 | Int32 encodedLength = (maxChars < 256) ? ToByte() : ToInt16(); 206 | String v = Encoding.UTF8.GetString(m_bytes, m_offset, encodedLength); 207 | m_offset += encodedLength; 208 | return v; 209 | } 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /Wintellect.Azure.Storage/Storage.cs: -------------------------------------------------------------------------------- 1 | /****************************************************************************** 2 | Module: Storage.cs 3 | Notices: Copyright (c) by Jeffrey Richter & Wintellect 4 | ******************************************************************************/ 5 | 6 | using System; 7 | using System.Diagnostics; 8 | using System.Net; 9 | using System.Threading; 10 | using System.Threading.Tasks; 11 | using Microsoft.WindowsAzure.Storage; 12 | using Microsoft.WindowsAzure.Storage.Blob; 13 | using Microsoft.WindowsAzure.Storage.Blob.Protocol; 14 | using Microsoft.WindowsAzure.Storage.RetryPolicies; 15 | using Wintellect.Azure.Storage.Table; 16 | 17 | namespace Wintellect.Azure.Storage { 18 | /// Defines extensions methods useful with Azure storage. 19 | public static class StorageExtensions { 20 | #region Primitive type extensions 21 | /// Improves performance when talking to Azure storage by setting UseNagleAlorithm to false, Expect100Continue to false, and setting the specific connection limit. 22 | /// The specific URI to apply these performance improvements to; if null, these performance improvements are applied to all URIs. 23 | /// The connection limit. 24 | public static void ImprovePerformance(this Uri uri, Int32 connectionLimit = 100) { 25 | if (uri != null) { 26 | // Improve performance for this 1 Uri 27 | ServicePoint sp = ServicePointManager.FindServicePoint(uri); 28 | sp.UseNagleAlgorithm = false; 29 | sp.Expect100Continue = false; 30 | sp.ConnectionLimit = connectionLimit; 31 | return; 32 | } else { 33 | // Improve performance for all Uri 34 | ServicePointManager.DefaultConnectionLimit = connectionLimit; 35 | 36 | // Turn off "expect 100 continue" so PUT/POST requests don't send expect 37 | // header & wait for 100-continue from server before sending payload body 38 | ServicePointManager.Expect100Continue = false; 39 | 40 | // Turn off the Nagle algorithm to send small packets quickly 41 | ServicePointManager.UseNagleAlgorithm = false; 42 | } 43 | } 44 | 45 | #if DeleteMe 46 | [DebuggerStepThrough] 47 | public static HttpWebResponse WebResponse(this WebException webException) { 48 | return (HttpWebResponse)webException.Response; 49 | } 50 | #endif 51 | #endregion 52 | 53 | #region Methods that apply to Blobs, Tables, and Queues 54 | /// Compares a StorageException against a specific HTTP code and storage error code string. 55 | /// The StorageException. 56 | /// The HTTP code to compare against. 57 | /// The optional storage error code string. 58 | /// True if the StorageException is due to the specific HTTP code and storage error code string. 59 | public static Boolean Matches(this StorageException se, HttpStatusCode code, String storageErrorCode = null) { 60 | if ((HttpStatusCode)se.RequestInformation.HttpStatusCode != code) return false; 61 | if (storageErrorCode == null) return true; 62 | return se.RequestInformation.ExtendedErrorInformation.ErrorCode == storageErrorCode; 63 | } 64 | 65 | #if DeleteMe 66 | 67 | public static OperationContext ChangeRequestVersionHeader(this OperationContext context, String versionHeaderValue) { 68 | context.SendingRequest += (sender, e) => { 69 | ((HttpWebRequest)e.Request).Headers["x-ms-version"] = versionHeaderValue; 70 | }; 71 | return context; 72 | } 73 | #endif 74 | 75 | [DebuggerStepThrough] 76 | private static async Task RetryUntilCreatedAsync(TStorage storage, Func action, Int32? msBetweenTries = null) { 77 | while (true) { 78 | try { await action(storage).ConfigureAwait(false); return storage; } 79 | catch (StorageException ex) { 80 | if (ex.RequestInformation.HttpStatusCode != 0/*StorageErrorCode.ResourceAlreadyExists*/) throw; 81 | } 82 | await Task.Delay(msBetweenTries ?? 10).ConfigureAwait(false); 83 | } 84 | } 85 | 86 | /// Retries an operation against a CloudBlobContainer until it succeeds. 87 | /// The blob container. 88 | /// The time to wait between retries. 89 | /// The passed-in CloudBlobContainer. 90 | [DebuggerStepThrough] 91 | public static Task RetryUntilCreatedAsync(this CloudBlobContainer container, Int32? msBetweenTries = null) { 92 | return RetryUntilCreatedAsync(container, c => c.CreateIfNotExistsAsync(), msBetweenTries); 93 | } 94 | 95 | /// Retries an operation against an AzureTable until it succeeds. 96 | /// The table. 97 | /// The time to wait between retries. 98 | /// The passed-in AzureTable. 99 | [DebuggerStepThrough] 100 | public static Task RetryUntilCreatedAsync(this AzureTable table, Int32? msBetweenTries = null) { 101 | return RetryUntilCreatedAsync(table, tc => tc.CreateIfNotExistsAsync(), msBetweenTries); 102 | } 103 | #endregion 104 | } 105 | 106 | /// Elects 1 machine to perform some operation periodically. 107 | 108 | public static class PeriodicElector { 109 | private enum ElectionError { 110 | Unknown, 111 | BlobNotFound, 112 | LeaseAlreadyPresent, 113 | ElectionOver 114 | } 115 | /// Ensures that 1 machine will execute an operation periodically. 116 | /// The base time; all machines should use the same exact base time. 117 | /// Indicates how frequently you want the operation performed. 118 | /// Indicates how frequently a machine that is not elected to perform the work should retry the work in case the elected machine crashes while performing the work. 119 | /// The blob that all the machines should be using to acquire a lease. 120 | /// Indicates when the operation should no longer be performed in the future. 121 | /// A method that is called periodically by whatever machine wins the election. 122 | /// A Task which you can use to catch an exception or known then this method has been canceled. 123 | public static async Task RunAsync(DateTimeOffset startTime, TimeSpan period, 124 | TimeSpan timeBetweenLeaseRetries, ICloudBlob blob, 125 | CancellationToken cancellationToken, Func winnerWorker) { 126 | var bro = new BlobRequestOptions { RetryPolicy = new ExponentialRetry() }; 127 | const Int32 leaseDurationSeconds = 60; // 15-60 seconds 128 | DateTimeOffset nextElectionTime = NextElectionTime(startTime, period); 129 | while (true) { 130 | TimeSpan timeToWait = nextElectionTime - DateTimeOffset.UtcNow; 131 | timeToWait = (timeToWait < TimeSpan.Zero) ? TimeSpan.Zero : timeToWait; 132 | await Task.Delay(timeToWait, cancellationToken).ConfigureAwait(false); // Wait until time to check 133 | ElectionError electionError = ElectionError.Unknown; 134 | try { 135 | // Try to acquire lease & check metadata to see if this period has been processed 136 | String leaseId = await blob.AcquireLeaseAsync(TimeSpan.FromSeconds(leaseDurationSeconds), null, 137 | AccessCondition.GenerateIfNotModifiedSinceCondition(nextElectionTime), bro, null, CancellationToken.None).ConfigureAwait(false); 138 | try { 139 | // Got lease: do elected work (periodically renew lease) 140 | Task winnerWork = Task.Run(winnerWorker); 141 | while (true) { 142 | // Verify if winnerWork throws, we exit this loop & release the lease to try again 143 | Task wakeUp = await Task.WhenAny(Task.Delay(TimeSpan.FromSeconds(leaseDurationSeconds - 15)), winnerWork).ConfigureAwait(false); 144 | if (wakeUp == winnerWork) { // Winner work is done 145 | if (winnerWork.IsFaulted) throw winnerWork.Exception; else break; 146 | } 147 | await blob.RenewLeaseAsync(AccessCondition.GenerateLeaseCondition(leaseId)).ConfigureAwait(false); 148 | } 149 | // After work done, write to blob to indicate elected winner sucessfully did work 150 | blob.UploadFromByteArray(new Byte[1], 0, 1, AccessCondition.GenerateLeaseCondition(leaseId), bro, null); 151 | nextElectionTime = NextElectionTime(nextElectionTime + period, period); 152 | } 153 | finally { 154 | blob.ReleaseLease(AccessCondition.GenerateLeaseCondition(leaseId)); 155 | } 156 | } 157 | catch (StorageException ex) { 158 | if (ex.Matches(HttpStatusCode.Conflict, BlobErrorCodeStrings.LeaseAlreadyPresent)) electionError = ElectionError.LeaseAlreadyPresent; 159 | else if (ex.Matches(HttpStatusCode.PreconditionFailed)) electionError = ElectionError.ElectionOver; 160 | else throw; 161 | } 162 | switch (electionError) { 163 | case ElectionError.ElectionOver: 164 | // If access condition failed, the election is over, wait until next election time 165 | nextElectionTime = NextElectionTime(nextElectionTime + period, period); 166 | break; 167 | case ElectionError.LeaseAlreadyPresent: 168 | // if failed to get lease, wait a bit and retry again 169 | await Task.Delay(timeBetweenLeaseRetries).ConfigureAwait(false); 170 | break; 171 | } 172 | } 173 | // We never get here 174 | } 175 | 176 | private static DateTimeOffset NextElectionTime(DateTimeOffset marker, TimeSpan period) { 177 | // From marker time, figure out closest period to NOW that is in the past 178 | Int32 periodSeconds = (Int32)period.TotalSeconds; 179 | var secs = TimeSpan.FromSeconds(((Int32)(DateTimeOffset.UtcNow - marker).TotalSeconds / periodSeconds) * periodSeconds); 180 | return marker + secs; // Return the most recent past period to this VM immediately tries to win the election 181 | } 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /Wintellect.Azure.Storage/Wintellect Azure Storage Library.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Wintellect/AzureStorage/3d16cc58d40afdd8925b9c6c86831f90b0cf992a/Wintellect.Azure.Storage/Wintellect Azure Storage Library.docx -------------------------------------------------------------------------------- /Wintellect.Azure.Storage/Wintellect.Azure.Storage.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 8.1 5 | Debug 6 | AnyCPU 7 | 8.0.30703 8 | 2.0 9 | {CF2AE096-D1B0-4C33-9921-C1B19948F971} 10 | Library 11 | Properties 12 | Wintellect.Azure.Storage 13 | Wintellect.Azure.Storage 14 | v4.5 15 | 512 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | publish\ 25 | true 26 | Disk 27 | false 28 | Foreground 29 | 7 30 | Days 31 | false 32 | false 33 | true 34 | 0 35 | 1.0.0.%2a 36 | false 37 | false 38 | true 39 | 40 | 41 | 42 | true 43 | full 44 | false 45 | bin\Debug\ 46 | DEBUG;TRACE 47 | prompt 48 | 4 49 | false 50 | false 51 | AllRules.ruleset 52 | false 53 | true 54 | bin\Debug\Wintellect.Azure.Storage.XML 55 | 56 | 57 | pdbonly 58 | true 59 | bin\Release\ 60 | TRACE 61 | prompt 62 | 4 63 | true 64 | false 65 | AllRules.ruleset 66 | false 67 | true 68 | bin\Release\Wintellect.Azure.Storage.XML 69 | 70 | 71 | 72 | False 73 | ..\packages\Microsoft.Data.Edm.5.6.2\lib\net40\Microsoft.Data.Edm.dll 74 | 75 | 76 | False 77 | ..\packages\Microsoft.Data.OData.5.6.2\lib\net40\Microsoft.Data.OData.dll 78 | 79 | 80 | False 81 | ..\packages\Microsoft.Data.Services.Client.5.6.2\lib\net40\Microsoft.Data.Services.Client.dll 82 | 83 | 84 | False 85 | ..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll 86 | 87 | 88 | False 89 | ..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll 90 | 91 | 92 | ..\packages\Newtonsoft.Json.5.0.8\lib\net45\Newtonsoft.Json.dll 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | False 103 | ..\..\..\..\..\..\..\..\Program Files (x86)\Reference Assemblies\Microsoft\Framework\.NETCore\v4.5.1\System.Runtime.WindowsRuntime.dll 104 | 105 | 106 | 107 | 108 | False 109 | ..\packages\System.Spatial.5.6.2\lib\net40\System.Spatial.dll 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | False 132 | Microsoft .NET Framework 4 %28x86 and x64%29 133 | true 134 | 135 | 136 | False 137 | .NET Framework 3.5 SP1 Client Profile 138 | false 139 | 140 | 141 | False 142 | .NET Framework 3.5 SP1 143 | false 144 | 145 | 146 | False 147 | Windows Installer 3.1 148 | true 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 164 | -------------------------------------------------------------------------------- /Wintellect.Azure.Storage/Wintellect.IO.cs: -------------------------------------------------------------------------------- 1 | /****************************************************************************** 2 | Module: Wintellect.IO.cs 3 | Notices: Copyright (c) by Jeffrey Richter & Wintellect 4 | ******************************************************************************/ 5 | 6 | using System; 7 | using System.Collections.Generic; 8 | using System.IO; 9 | using System.Linq; 10 | using System.Threading; 11 | using System.Threading.Tasks; 12 | 13 | namespace Wintellect.IO { 14 | // http://www.goodreads.com/author_blog_posts/2312502-asynchronous-batch-logging-without-blocking-threads 15 | using Batch = System.Collections.Generic.List; 16 | 17 | /// A class that batches log information together persisting it when a batch (number of bytes) has been reached or when a period of time has expired. 18 | public sealed class BatchLogger { 19 | private static readonly TimeSpan Infinite = TimeSpan.FromMilliseconds(Timeout.Infinite); 20 | private SpinLock m_lock = new SpinLock(false); // Can't be read-only since this is a struct 21 | private readonly TimeSpan m_maxInterval; 22 | private readonly Int32 m_maxBatchBytes; 23 | private readonly Func m_transferBatch; 24 | private Int32 m_outstandingBatches = 0; 25 | private Batch m_batch = null; 26 | private Int32 m_batchBytes = 0; 27 | private Timer m_timer = null; 28 | private Boolean m_minimalLossEnabled = false; 29 | 30 | /// Constructs a batch logger. 31 | /// Indicates how many bytes need to accumulate in the batch in memory before persisting the batch. 32 | /// The maximum size (in milliseconds) that bytes should sit in memory before being persisted. 33 | /// The method to call to persist the batch. 34 | public BatchLogger(Int32 maxBatchBytes, TimeSpan maxInterval, Func transferBatch) { 35 | m_maxBatchBytes = maxBatchBytes; 36 | m_maxInterval = maxInterval; 37 | m_transferBatch = transferBatch; 38 | CreateNewBatch(); 39 | } 40 | 41 | // Returns reference to old batch so it can be flushed to the log 42 | private Batch CreateNewBatch() { 43 | // NOTE: This method MUST be called under the SpinLock unless called by the ctor 44 | if (m_batch != null && m_batchBytes == 0) return null; // Batch exists & is empty, nothing to do 45 | Batch oldbatch = m_batch; 46 | m_batch = new Batch(m_maxBatchBytes); 47 | m_batchBytes = 0; 48 | if (m_timer != null) m_timer.Dispose(); 49 | m_timer = new Timer(TimeExpired, m_batch, m_maxInterval, Infinite); 50 | return oldbatch; 51 | } 52 | 53 | private void TimeExpired(Object timersBatch) { 54 | Boolean taken = false; 55 | m_lock.Enter(ref taken); 56 | Batch oldBatch = null; 57 | if (timersBatch != m_batch) { 58 | // This timer is not for the current batch; do nothing 59 | // Solves race condition where time fires AS batch is changing 60 | } else { 61 | if (m_batchBytes == 0) { 62 | // Batch empty: reset timer & use the same batch 63 | m_timer.Change(m_maxInterval, Infinite); 64 | } else { 65 | // Batch not empty: (force flush &) create a new batch 66 | oldBatch = CreateNewBatch(); 67 | } 68 | } 69 | m_lock.Exit(); 70 | // If there was an old batch, transfer it 71 | TransferBatchAsync(this, oldBatch); 72 | } 73 | 74 | /// 75 | /// Call this method to flush the current batch and to disable batching. Future 76 | /// calls to Append will immediately call the transferBatch callback. You can call this 77 | /// method when a VM wants to shut down to reduce the chance of losing log data. 78 | /// 79 | public void EnableMinimalLoss() { 80 | // Flush any existing batch 81 | Boolean taken = false; 82 | m_lock.Enter(ref taken); 83 | m_minimalLossEnabled = true; // Future log entries are not batched 84 | Batch oldBatch = CreateNewBatch(); 85 | m_lock.Exit(); 86 | // If batch swapped, transfer it 87 | TransferBatchAsync(this, oldBatch); 88 | } 89 | 90 | /// Appends a byte array to the current batch. 91 | /// The byte array containing the data to append. 92 | public void Append(Byte[] data) { 93 | if (data.Length > m_maxBatchBytes) 94 | throw new ArgumentOutOfRangeException("data", "A single data object cannot be larger than " + m_maxBatchBytes.ToString() + " bytes."); 95 | if (m_minimalLossEnabled) { TransferBatchAsync(this, new Batch { data }); return; } 96 | 97 | Boolean taken = false; 98 | m_lock.Enter(ref taken); 99 | Batch oldBatch = null; 100 | if (m_batchBytes + data.Length > m_maxBatchBytes) { 101 | // Data won't fit in current batch, create a new batch so we can transfer the old batch 102 | oldBatch = CreateNewBatch(); 103 | } 104 | // Add new data to current (or new) batch 105 | m_batch.Add(data); 106 | m_batchBytes += data.Length; 107 | m_lock.Exit(); 108 | // If batch swapped, transfer it 109 | TransferBatchAsync(this, oldBatch); 110 | } 111 | 112 | // TransferBatch is static to make it clear that this method is 113 | // NOT attached to the object in any way; this is an independent operation 114 | // that does not rely on any object state or a lock 115 | private static Task TransferBatchAsync(BatchLogger batchLogger, Batch batch) { 116 | // NOTE: this should be called while NOT under a lock 117 | if (batch == null) return Task.FromResult(true); // No batch to transfer, return 118 | 119 | // Start transfer of batch (asynchronously) to the persistent store... 120 | Interlocked.Increment(ref batchLogger.m_outstandingBatches); 121 | return batchLogger.m_transferBatch(batch) 122 | .ContinueWith(_ => Interlocked.Decrement(ref batchLogger.m_outstandingBatches)); 123 | } 124 | } 125 | } 126 | 127 | namespace Wintellect.IO { 128 | /// A stream that efficiently stores a collection of byte arrays allowing you to read from them as a stream. 129 | public sealed class GatherStream : Stream { 130 | private readonly IEnumerable m_enumerable; 131 | private readonly Int64 m_length; 132 | private IEnumerator m_buffers; 133 | private Byte[] m_currentBuffer = null; 134 | private Int32 m_bufferOffset = 0; 135 | private Int64 m_position = 0; 136 | 137 | /// Constructs a GatherStream initializing it with a collection of byte arrays. 138 | /// The collection of bytes arrays to be read from as a stream. 139 | public GatherStream(IEnumerable buffers) { 140 | m_length = buffers.Sum(b => b.Length); 141 | m_enumerable = buffers; 142 | Seek(0, SeekOrigin.Begin); 143 | } 144 | /// Always returns true. 145 | public override Boolean CanSeek { get { return true; } } 146 | /// Seeks to a byte offset within the logical stream. 147 | /// The offset of the byte within the logical stream (must be 0). 148 | /// The origin (must be SeekOrigin.Begin). 149 | /// Always returns 0. 150 | public override Int64 Seek(Int64 offset, SeekOrigin origin) { 151 | if (offset != 0 || origin != SeekOrigin.Begin) throw new NotImplementedException(); 152 | m_buffers = m_enumerable.GetEnumerator(); 153 | m_currentBuffer = m_buffers.MoveNext() ? m_buffers.Current : null; 154 | m_bufferOffset = 0; 155 | m_position = 0; 156 | return 0; 157 | } 158 | /// The number of bytes in the logical stream. 159 | public override Int64 Length { get { return m_length; } } 160 | /// Always returns true. 161 | public override Boolean CanRead { get { return true; } } 162 | /// Copies bytes from the logical stream into a byte array. 163 | /// The buffer where the stream's bytes should be placed. 164 | /// The offset the buffer to place the stream's next byte. 165 | /// The number of bytes from the stream to copy into the buffer. 166 | /// The number of bytes copied into the buffer. 167 | public override Int32 Read(Byte[] buffer, Int32 offset, Int32 count) { 168 | Int32 bytesRead = 0; 169 | // Loop while we still have buffers and we need to read more bytes 170 | while (m_currentBuffer != null && bytesRead < count) { 171 | // Copy bytes from internal buffer to output buffer 172 | Int32 bytesToTransferThisTime = Math.Min(count - bytesRead, m_currentBuffer.Length - m_bufferOffset); 173 | Buffer.BlockCopy(m_currentBuffer, m_bufferOffset, buffer, offset, bytesToTransferThisTime); 174 | bytesRead += bytesToTransferThisTime; 175 | offset += bytesToTransferThisTime; 176 | m_bufferOffset += bytesToTransferThisTime; 177 | if (m_bufferOffset == m_currentBuffer.Length) { 178 | // Read all of this Byte[], move to next Byte[] 179 | m_bufferOffset = 0; 180 | m_currentBuffer = m_buffers.MoveNext() ? m_buffers.Current : null; 181 | } 182 | } 183 | m_position += bytesRead; 184 | return bytesRead; 185 | } 186 | #region Output members 187 | /// You can set the Position to 0 (only). You cannot get the position. 188 | public override Int64 Position { 189 | get { return m_position; } 190 | set { if (value != 0) throw new ArgumentOutOfRangeException("value must be 0"); Seek(0, SeekOrigin.Begin); } 191 | } 192 | /// Always returns false. 193 | public override Boolean CanWrite { get { return false; } } 194 | /// Throws a NotImplementedException. 195 | /// This argument is ignored. 196 | public override void SetLength(Int64 value) { throw new NotImplementedException(); } 197 | /// Throws a NotImplementedException. 198 | /// This argument is ignored. 199 | /// This argument is ignored. 200 | /// This argument is ignored. 201 | public override void Write(Byte[] buffer, Int32 offset, Int32 count) { throw new NotImplementedException(); } 202 | /// Throws a NotImplementedException. 203 | public override void Flush() { throw new NotImplementedException(); } 204 | #endregion 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /Wintellect.Azure.Storage/Wintellect.Periodic.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | 4 | 5 | namespace Wintellect.Periodic { 6 | /// A static class with helper methods used to calculate when time periods start and end. 7 | public static class PeriodCalculator { 8 | /// Returns the start date of a cycle. 9 | /// The period to use to calculate the period start date. 10 | /// Used for MonthlyCycle and YearlyCycle to determine the cross-over date. 11 | /// The number of periods to add (use a negative number to subtract). 12 | /// The period's start date. 13 | public static DateTimeOffset CalculatePeriodStartDate(Period period, DateTimeOffset cycleDate, Int32 periodsToAdd = 0) { 14 | DateTimeOffset periodStop; 15 | return CalculatePeriodDates(period, cycleDate, periodsToAdd, out periodStop); 16 | } 17 | 18 | /// Returns the start date of a cycle. 19 | /// The period to use to calculate the period start and end dates. 20 | /// Used for MonthlyCycle and YearlyCycle to determine the cross-over date. 21 | /// The number of periods to add (use a negative number to subtract). 22 | /// The period's end date. 23 | /// The period's start date. 24 | public static DateTimeOffset CalculatePeriodDates(Period period, DateTimeOffset cycleDate, Int32 periodsToAdd, out DateTimeOffset periodStop) { 25 | DateTimeOffset periodStart, today = DateTimeOffset.UtcNow; 26 | switch (period) { 27 | case Period.Day: periodStart = periodStop = today.AddDays(periodsToAdd); break; 28 | case Period.MonthCycle: 29 | periodStart = GetClosestMonthlyCycleStart(cycleDate).AddMonths(periodsToAdd); 30 | periodStop = periodStart.AddMonths(1).AddDays(-1); 31 | break; 32 | case Period.Month: 33 | periodStart = new DateTimeOffset(today.Year, today.Month, 1, 0, 0, 0, TimeSpan.Zero).AddMonths(periodsToAdd); 34 | periodStop = periodStart.AddMonths(1).AddDays(-1); 35 | break; 36 | case Period.Quarter: 37 | periodStart = new DateTimeOffset(today.Year, (((today.Month) / 3) * 3) + 1, 1, 0, 0, 0, TimeSpan.Zero).AddMonths(3 * periodsToAdd); 38 | periodStop = periodStart.AddMonths(3).AddDays(-1); 39 | break; 40 | case Period.YearCycle: 41 | periodStart = GetClosestYearlyCycleStart(cycleDate).AddYears(periodsToAdd); 42 | periodStop = periodStart.AddYears(1).AddDays(-1); 43 | break; 44 | case Period.Year: 45 | periodStart = new DateTimeOffset(today.Year, 1, 1, 0, 0, 0, TimeSpan.Zero).AddYears(periodsToAdd); 46 | periodStop = periodStart.AddYears(1).AddDays(-1); 47 | break; 48 | default: 49 | throw new ArgumentException("Invalid period:" + period, "period"); 50 | } 51 | return periodStart; 52 | } 53 | private static DateTimeOffset GetClosestMonthlyCycleStart(DateTimeOffset date) { 54 | // Jan 31 -> Feb 28 (or 29 on leap year) 55 | var today = DateTimeOffset.UtcNow; 56 | var desiredDate = new DateTime(today.Year, today.Month, 57 | Math.Min(date.Day, DateTime.DaysInMonth(today.Year, today.Month))); 58 | if (desiredDate <= today) return desiredDate; 59 | 60 | // If desired is in the future, go back a month (if today is Feb 20 -> Jan 31) 61 | today = today.AddMonths(-1); 62 | return new DateTime(today.Year, today.Month, 63 | Math.Min(date.Day, DateTime.DaysInMonth(today.Year, today.Month))); 64 | } 65 | 66 | private static DateTimeOffset GetClosestYearlyCycleStart(DateTimeOffset date) { 67 | Debug.Assert(date <= DateTimeOffset.UtcNow); 68 | // Jan 31,2013 -> Feb 28,2014 (or 29 on leap year) 69 | var today = DateTimeOffset.UtcNow; 70 | var desiredDate = new DateTime(today.Year, date.Month, 71 | Math.Min(date.Day, DateTime.DaysInMonth(today.Year, date.Month))); 72 | if (desiredDate <= today) return desiredDate; 73 | 74 | // If desired is in the future, go back a year 75 | today = today.AddYears(-1).AddDays(-1); 76 | return new DateTime(today.Year, date.Month, 77 | Math.Min(date.Day, DateTime.DaysInMonth(today.Year, date.Month))); 78 | } 79 | } 80 | 81 | /// An enum with the Periods supported by the PeriodCalculator's methods. 82 | public enum Period { 83 | /// Indicates a daily period. 84 | Day, 85 | /// Indicates a monthly period starting at a specific day within a month. 86 | MonthCycle, 87 | /// Indicates a monthly period starting at the first of the month. 88 | Month, 89 | /// Indicates a quarterly period. 90 | Quarter, 91 | /// Indicates a yearly period starting at January 1st. 92 | Year, 93 | /// Indicates a yearly period starting at a specific day within a year. 94 | YearCycle 95 | } 96 | } -------------------------------------------------------------------------------- /Wintellect.Azure.Storage/Wintellect.cs: -------------------------------------------------------------------------------- 1 | /****************************************************************************** 2 | Module: Wintellect.cs 3 | Notices: Copyright (c) by Jeffrey Richter & Wintellect 4 | ******************************************************************************/ 5 | 6 | using System; 7 | using System.Collections.Generic; 8 | using System.Diagnostics; 9 | using System.IO; 10 | using System.Linq.Expressions; 11 | using System.Runtime.InteropServices; 12 | using System.Security.Cryptography.Pkcs; 13 | using System.Security.Cryptography.X509Certificates; 14 | using System.Text; 15 | 16 | #if false 17 | namespace Wintellect { 18 | public static class Diagnostics { 19 | [Conditional("TRACE"), MethodImpl(MethodImplOptions.NoInlining), DebuggerStepThrough] 20 | public static void WhenWhere(Action callback, [CallerMemberName] String member = null, [CallerFilePath] String file = null, [CallerLineNumber] Int32 line = 0) { 21 | var sb = new StringBuilder("[") 22 | .Append(DateTimeOffset.UtcNow.ToString()).Append(" ") 23 | .Append(member).Append(" ") 24 | .Append(file).Append(" ") 25 | .Append(line.ToString()).Append("] "); 26 | callback(sb); 27 | } 28 | } 29 | } 30 | #endif 31 | 32 | namespace Wintellect { 33 | /// Defines String extension methods. 34 | public static class StringEx { 35 | /// Indicates whether a specified string is null, empty, or consists only of white-space characters. 36 | /// The string to test. 37 | /// true if the value parameter is null or System.String.Empty, or if value consists exclusively of white-space characters. 38 | [DebuggerStepThrough] 39 | public static Boolean IsNullOrWhiteSpace(this String s) { return String.IsNullOrWhiteSpace(s); } 40 | 41 | /// Converts a base-64 string to a byte array. 42 | /// The base-64 string to convert. 43 | /// The byte array. 44 | [DebuggerStepThrough] 45 | public static Byte[] FromBase64ToBytes(this String s) { return Convert.FromBase64String(s); } 46 | /// Converts a byte array to a base-64 string. 47 | /// The byte array to convert. 48 | /// The base-64 string. 49 | [DebuggerStepThrough] 50 | public static String ToBase64String(this Byte[] data) { return Convert.ToBase64String(data); } 51 | 52 | /// Converts an Int32 to a base-64 string. 53 | /// The Int32 to convert. 54 | /// The base-64 string. 55 | [DebuggerStepThrough] 56 | public static String ToBase64(this Int32 value) { 57 | // Convert 4-byte Int32 to Byte[4] and Base-64 encode it 58 | return Convert.ToBase64String(BitConverter.GetBytes(value)); // Byte[<=64] 59 | } 60 | 61 | /// Converts a base-64 string to an Int32. 62 | /// The base-64 string to convert. 63 | /// The Int32. 64 | [DebuggerStepThrough] 65 | public static Int32 FromBase64ToInt32(this String value) { 66 | return BitConverter.ToInt32(Convert.FromBase64String(value), 0); 67 | } 68 | 69 | /// HTML-encodes a string. 70 | /// The string to HTML encode. 71 | /// The HTML-encoded string. 72 | [DebuggerStepThrough] 73 | public static String HtmlEncode(this String s) { return System.Web.HttpUtility.HtmlEncode(s); } 74 | 75 | /// Encodes a string to a byte array. 76 | /// The string to encode. 77 | /// The encoding to use (default is UTF-8). 78 | /// The encoded string as a byte array. 79 | [DebuggerStepThrough] 80 | public static Byte[] Encode(this String s, Encoding encoding = null) { return (encoding ?? Encoding.UTF8).GetBytes(s); } 81 | 82 | /// Decodes a byte array to a string. 83 | /// The byte array to decode. 84 | /// The encoding to use (default is UTF-8). 85 | /// The decoded string. 86 | [DebuggerStepThrough] 87 | public static String Decode(this Byte[] data, Encoding encoding = null) { return (encoding ?? Encoding.UTF8).GetString(data); } 88 | /// Decodes a byte array to a string. 89 | /// The byte array to decode. 90 | /// The starting index within the byte array. 91 | /// The number of bytes to decode. 92 | /// The encoding to use (default is UTF-8). 93 | /// The decoded string. 94 | [DebuggerStepThrough] 95 | public static String Decode(this Byte[] data, Int32 index, Int32 count, Encoding encoding = null) { return (encoding ?? Encoding.UTF8).GetString(data, index, count); } 96 | 97 | /// Performs a case-insensitive comparison against the start of a string. 98 | /// The string to compare against. 99 | /// The prefix you want to see if string starts with. 100 | /// true if string starts with prefix (in a case insensitive way). 101 | public static Boolean StartsWithInsensitive(this String @string, String prefix) { 102 | return String.Compare(@string, 0, prefix, 0, prefix.Length, StringComparison.OrdinalIgnoreCase) == 0; 103 | } 104 | 105 | /// Returns a comma-separated string consisting of the elements. Each string element is enclosed in quotes. 106 | /// The elements to quote and comma separate. 107 | /// The CSV string. 108 | public static String ToCsvLine(this IEnumerable elements) { 109 | // This methods purposely does NOT accept Objects to avoid unnecessary boxings. 110 | // This forces callers to call ToString thereby reducing memory overhead and improving performance. 111 | var sb = new StringBuilder(); 112 | foreach (String e in elements) { 113 | if (sb.Length > 0) sb.Append(','); 114 | sb.Append('\"').Append(e).Append('\"'); 115 | } 116 | return sb.AppendLine().ToString(); 117 | } 118 | 119 | /// Returns a comma-separated string consisting of the elements. Each string element is enclosed in quotes. 120 | /// The elements to quote and comma separate. 121 | /// The CSV string. 122 | public static String ToCsvLine(params String[] elements) { 123 | // This methods purposely does NOT accept Objects to avoid unnecessary boxings. 124 | // This forces callers to call ToString thereby reducing memory overhead and improving performance. 125 | return ToCsvLine((IEnumerable)elements); 126 | } 127 | 128 | #region Cryptographic Message Syntax 129 | // Encrypts/decrypts messages via http://www.ietf.org/rfc/rfc3852.txt 130 | // CMS is used to digitally sign, digest, authenticate, or encrypt arbitrary message content. 131 | // CMS supports digital signatures & encryption. One encapsulation envelope can be nested 132 | // inside another. Likewise, one party can digitally sign some previously encapsulated data. 133 | // It also allows arbitrary attributes, such as signing time, to be signed along with the message content, 134 | // and provides for other attributes such as countersignatures to be associated with a signature. 135 | 136 | /// Encrypts a string using the Cryptographic Message Syntax. 137 | /// The string data to encrypt. 138 | /// The certificate (must be marked for key exchange). 139 | /// The encrypted string. 140 | public static String EncryptTextUsingCms(this String clearText, X509Certificate2 certificate) { 141 | // Good way to get a certificate: new X509Certificate2(pfxCertificatePathname, pfxPassword); 142 | 143 | // Create an envelope with the text as its contents 144 | var envelopedCms = new EnvelopedCms(new ContentInfo(Encoding.UTF8.GetBytes(clearText))); 145 | 146 | // Encrypt the envelope for the recipient & return the encoded value; 147 | // The recipient of the envelope is the owner of the certificate's private key 148 | // NOTE: the certificate info is embedded; no need to remember the thumbprint separately. 149 | envelopedCms.Encrypt(new CmsRecipient(certificate)); 150 | return Convert.ToBase64String(envelopedCms.Encode()); 151 | } 152 | 153 | /// Decrypts a string using the Cryptographic Message Syntax. The certificate (with private key) must be in the current user's or local compute's Personal (My) store or passed via the extraStore parameter. 154 | /// The string data to decrypt. 155 | /// If not null, represents additional certificates to use for the decryption. 156 | /// The decrypted string. 157 | public static String DecryptTextFromCms(this String cipherText, X509Certificate2Collection extraStore = null) { 158 | var envelopedCms = new EnvelopedCms(); 159 | envelopedCms.Decode(Convert.FromBase64String(cipherText)); 160 | if (extraStore == null) envelopedCms.Decrypt(); 161 | else envelopedCms.Decrypt(extraStore); 162 | return Encoding.UTF8.GetString(envelopedCms.ContentInfo.Content); 163 | } 164 | #endregion 165 | } 166 | } 167 | 168 | namespace Wintellect { 169 | /// Defines methods that operate on enums. 170 | public static class EnumEx { 171 | /// Returns an enum's values. 172 | /// The enum type. 173 | /// The enum type's values. 174 | public static TEnum[] GetValues() where TEnum : struct { 175 | return (TEnum[])Enum.GetValues(typeof(TEnum)); 176 | } 177 | /// Parses a string to an enum type's value. 178 | /// The enum type. 179 | /// The string symbol to parse. 180 | /// true to do a case-insensitive comparison. 181 | /// The enum type's value. 182 | public static TEnum Parse(String value, Boolean ignoreCase = false) where TEnum : struct { 183 | return (TEnum)Enum.Parse(typeof(TEnum), value, ignoreCase); 184 | } 185 | } 186 | } 187 | 188 | namespace Wintellect { 189 | /// A helper struct that returns the string name for a type's property. 190 | /// The type defining the property you wish to convert to a string. 191 | public struct PropertyName { 192 | /// Returns a type's property name as a string. 193 | /// An expression that returns the desired property. 194 | /// The property name as a string. 195 | public String this[LambdaExpression propertyExpression] { 196 | get { 197 | Expression body = propertyExpression.Body; 198 | MemberExpression me = (body is UnaryExpression) 199 | ? (MemberExpression)((UnaryExpression)body).Operand 200 | : (MemberExpression)body; 201 | return me.Member.Name; 202 | } 203 | } 204 | 205 | /// Returns a type's property name as a string. 206 | /// An expression that returns the desired property. 207 | /// The property name as a string. 208 | public String this[Expression> propertyExpression] { 209 | get { 210 | return this[(LambdaExpression)propertyExpression]; 211 | } 212 | } 213 | 214 | /// Returns several types property names as a string collection. 215 | /// The expressions; each returns a desired property. 216 | /// The property names as a string collection. 217 | public String[] this[params Expression>[] propertyExpressions] { 218 | get { 219 | var propertyNames = new String[propertyExpressions.Length]; 220 | for (Int32 i = 0; i < propertyNames.Length; i++) propertyNames[i] = this[(LambdaExpression)propertyExpressions[i]]; 221 | return propertyNames; 222 | } 223 | } 224 | } 225 | } 226 | 227 | namespace Wintellect { 228 | /// The class defines guid extension methods. 229 | public static class GuidExtensions { 230 | /// Write a Guid to a BinaryWriter. 231 | /// The BinaryWriter object. 232 | /// The Guid to write. 233 | public static void Write(this BinaryWriter stream, Guid value) { 234 | GuidBytes gb = value.ToBytes(); 235 | stream.Write(gb.UInt64_0); 236 | stream.Write(gb.UInt64_1); 237 | } 238 | /// Reads a Guid from a BinaryWriter. 239 | /// The BinaryWriter. 240 | /// The Guid. 241 | public static Guid ReadGuid(this BinaryReader stream) { 242 | return new GuidBytes(stream.ReadUInt64(), stream.ReadUInt64()).Guid; 243 | } 244 | /// Efficiently returns a Guid's bytes without making any memory allocations. 245 | /// The Guid whose bytes you wish to obtain. 246 | /// A GuidBytes allowing you to get the Guid's byte values. 247 | public static GuidBytes ToBytes(this Guid guid) { return new GuidBytes(guid); } 248 | } 249 | 250 | /// Efficiently manipulates the byes within a Guid instance. 251 | [StructLayout(LayoutKind.Explicit)] 252 | public struct GuidBytes { 253 | /// Returns the Guid as a Guid. 254 | [FieldOffset(0)] 255 | public readonly Guid Guid; 256 | 257 | /// Returns the first 64 bits within the Guid . 258 | [FieldOffset(0)] 259 | public readonly UInt64 UInt64_0; 260 | /// Returns the second 64 bits within the Guid . 261 | [FieldOffset(8)] 262 | public readonly UInt64 UInt64_1; 263 | 264 | /// Returns the a part of the Guid . 265 | [FieldOffset(0)] 266 | public readonly uint a; 267 | /// Returns the b part of the Guid . 268 | [FieldOffset(4)] 269 | public readonly ushort b; 270 | /// Returns the c part of the Guid . 271 | [FieldOffset(6)] 272 | public readonly ushort c; 273 | /// Returns the d part of the Guid . 274 | [FieldOffset(8)] 275 | public readonly byte d; 276 | /// Returns the e part of the Guid . 277 | [FieldOffset(9)] 278 | public readonly byte e; 279 | /// Returns the f part of the Guid . 280 | [FieldOffset(10)] 281 | public readonly byte f; 282 | /// Returns the g part of the Guid . 283 | [FieldOffset(11)] 284 | public readonly byte g; 285 | /// Returns the h part of the Guid . 286 | [FieldOffset(12)] 287 | public readonly byte h; 288 | /// Returns the i part of the Guid . 289 | [FieldOffset(13)] 290 | public readonly byte i; 291 | /// Returns the j part of the Guid . 292 | [FieldOffset(14)] 293 | public readonly byte j; 294 | /// Returns the k part of the Guid . 295 | [FieldOffset(15)] 296 | public readonly byte k; 297 | 298 | /// Wraps a Guid instance with a GuidBytes instance allowing efficient access to the Guid's bytes. 299 | /// The Guid to wrap. 300 | public GuidBytes(Guid guid) 301 | : this() { 302 | Guid = guid; 303 | } 304 | /// Creates a GuidBytes instance from a Guid's two 64-bit integer values. 305 | /// The first 64 bits of the Guid. 306 | /// The second 64 bits of the Guid. 307 | public GuidBytes(UInt64 UInt64_0, UInt64 UInt64_1) 308 | : this() { 309 | this.UInt64_0 = UInt64_0; 310 | this.UInt64_1 = UInt64_1; 311 | } 312 | } 313 | } -------------------------------------------------------------------------------- /Wintellect.Azure.Storage/app.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /Wintellect.Azure.Storage/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /packages/repositories.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | --------------------------------------------------------------------------------