├── .gitignore ├── .nuget ├── NuGet.exe ├── NuGet.Config └── NuGet.targets ├── packages.config ├── WazStorageExtensions.nuspec ├── WazStorageExtensions.sln ├── InitializationExtensions.cs ├── ExistenceExtensions.cs ├── Properties └── AssemblyInfo.cs ├── LICENSE ├── README.md ├── WazStorageExtensions.csproj ├── LeaseExtensions.cs └── AutoRenewLease.cs /.gitignore: -------------------------------------------------------------------------------- 1 | obj 2 | bin 3 | *.user 4 | *.suo 5 | *.cache 6 | *.nupkg 7 | packages/ 8 | -------------------------------------------------------------------------------- /.nuget/NuGet.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smarx/WazStorageExtensions/HEAD/.nuget/NuGet.exe -------------------------------------------------------------------------------- /.nuget/NuGet.Config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /packages.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /WazStorageExtensions.nuspec: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | $id$ 5 | $version$ 6 | $author$ 7 | $author$ 8 | false 9 | $description$ 10 | azure storage 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /WazStorageExtensions.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 11.00 3 | # Visual Studio 2010 4 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WazStorageExtensions", "WazStorageExtensions.csproj", "{DC796415-5760-4021-8E92-CB615569D84E}" 5 | EndProject 6 | Global 7 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 8 | Debug|Any CPU = Debug|Any CPU 9 | Release|Any CPU = Release|Any CPU 10 | EndGlobalSection 11 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 12 | {DC796415-5760-4021-8E92-CB615569D84E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 13 | {DC796415-5760-4021-8E92-CB615569D84E}.Debug|Any CPU.Build.0 = Debug|Any CPU 14 | {DC796415-5760-4021-8E92-CB615569D84E}.Release|Any CPU.ActiveCfg = Release|Any CPU 15 | {DC796415-5760-4021-8E92-CB615569D84E}.Release|Any CPU.Build.0 = Release|Any CPU 16 | EndGlobalSection 17 | GlobalSection(SolutionProperties) = preSolution 18 | HideSolutionNode = FALSE 19 | EndGlobalSection 20 | EndGlobal 21 | -------------------------------------------------------------------------------- /InitializationExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using Microsoft.WindowsAzure; 6 | using Microsoft.WindowsAzure.StorageClient; 7 | 8 | namespace smarx.WazStorageExtensions 9 | { 10 | public static class InitializationExtensions 11 | { 12 | public static void Ensure(this CloudStorageAccount account, IEnumerable tables = null, IEnumerable containers = null, IEnumerable queues = null) 13 | { 14 | var tableClient = account.CreateCloudTableClient(); 15 | var blobClient = account.CreateCloudBlobClient(); 16 | var queueClient = account.CreateCloudQueueClient(); 17 | 18 | if (tables != null) foreach (var table in tables) tableClient.CreateTableIfNotExist(table); 19 | if (containers != null) foreach (var container in containers) blobClient.GetContainerReference(container).CreateIfNotExist(); 20 | if (queues != null) foreach (var queue in queues) queueClient.GetQueueReference(queue).CreateIfNotExist(); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /ExistenceExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using Microsoft.WindowsAzure.StorageClient; 6 | 7 | namespace smarx.WazStorageExtensions 8 | { 9 | public static class ExistenceExtensions 10 | { 11 | public static bool Exists(this CloudBlob blob) 12 | { 13 | try 14 | { 15 | blob.FetchAttributes(); 16 | return true; 17 | } 18 | catch (StorageClientException e) 19 | { 20 | if (e.ErrorCode == StorageErrorCode.ResourceNotFound) 21 | { 22 | return false; 23 | } 24 | else 25 | { 26 | throw; 27 | } 28 | } 29 | } 30 | public static bool Exists(this CloudBlobContainer container) 31 | { 32 | try 33 | { 34 | container.FetchAttributes(); 35 | return true; 36 | } 37 | catch (StorageClientException e) 38 | { 39 | if (e.ErrorCode == StorageErrorCode.ResourceNotFound) 40 | { 41 | return false; 42 | } 43 | else 44 | { 45 | throw; 46 | } 47 | } 48 | } 49 | } 50 | } -------------------------------------------------------------------------------- /Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("WazStorageExtensions")] 9 | [assembly: AssemblyDescription("Useful extension methods for Windows Azure storage operations that aren't covered by the .NET client library")] 10 | [assembly: AssemblyCompany("smarx")] 11 | [assembly: AssemblyProduct("WazStorageExtensions")] 12 | 13 | // Setting ComVisible to false makes the types in this assembly not visible 14 | // to COM components. If you need to access a type in this assembly from 15 | // COM, set the ComVisible attribute to true on that type. 16 | [assembly: ComVisible(false)] 17 | 18 | // The following GUID is for the ID of the typelib if this project is exposed to COM 19 | [assembly: Guid("6e9fb729-e9d4-4966-a2bd-a7ca00257e15")] 20 | 21 | // Version information for an assembly consists of the following four values: 22 | // 23 | // Major Version 24 | // Minor Version 25 | // Build Number 26 | // Revision 27 | // 28 | // You can specify all the values or you can default the Build and Revision Numbers 29 | // by using the '*' as shown below: 30 | // [assembly: AssemblyVersion("1.0.*")] 31 | [assembly: AssemblyVersion("5.2.0.0")] 32 | [assembly: AssemblyFileVersion("5.2.0.0")] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Microsoft Public License (Ms-PL) 2 | 3 | This license governs use of the accompanying software. If you use the software, you accept this license. If you do not accept the license, do not use the software. 4 | 5 | 1. Definitions 6 | 7 | The terms "reproduce," "reproduction," "derivative works," and "distribution" have the same meaning here as under U.S. copyright law. 8 | 9 | A "contribution" is the original software, or any additions or changes to the software. 10 | 11 | A "contributor" is any person that distributes its contribution under this license. 12 | 13 | "Licensed patents" are a contributor's patent claims that read directly on its contribution. 14 | 15 | 2. Grant of Rights 16 | 17 | (A) Copyright Grant- Subject to the terms of this license, including the license conditions and limitations in section 3, each contributor grants you a non-exclusive, worldwide, royalty-free copyright license to reproduce its contribution, prepare derivative works of its contribution, and distribute its contribution or any derivative works that you create. 18 | 19 | (B) Patent Grant- Subject to the terms of this license, including the license conditions and limitations in section 3, each contributor grants you a non-exclusive, worldwide, royalty-free license under its licensed patents to make, have made, use, sell, offer for sale, import, and/or otherwise dispose of its contribution in the software or derivative works of the contribution in the software. 20 | 21 | 3. Conditions and Limitations 22 | 23 | (A) No Trademark License- This license does not grant you rights to use any contributors' name, logo, or trademarks. 24 | 25 | (B) If you bring a patent claim against any contributor over patents that you claim are infringed by the software, your patent license from such contributor to the software ends automatically. 26 | 27 | (C) If you distribute any portion of the software, you must retain all copyright, patent, trademark, and attribution notices that are present in the software. 28 | 29 | (D) If you distribute any portion of the software in source code form, you may do so only under this license by including a complete copy of this license with your distribution. If you distribute any portion of the software in compiled or object code form, you may only do so under a license that complies with this license. 30 | 31 | (E) The software is licensed "as-is." You bear the risk of using it. The contributors give no express warranties, guarantees or conditions. You may have additional consumer rights under your local laws which this license cannot change. To the extent permitted under your local laws, the contributors exclude the implied warranties of merchantability, fitness for a particular purpose and non-infringement. 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | WazStorageExtensions 2 | ==================== 3 | 4 | smarx.WazStorageExtensions is a collection of useful extension methods for Windows Azure storage operations that aren't covered by the .NET client library. 5 | 6 | It can be install using the [NuGet package](http://nuget.org/List/Packages/smarx.WazStorageExtensions) via `install-package smarx.WazStorageExtensions` and contains extension methods and classes for the following: 7 | 8 | * Storage Analytics API ([MSDN documentation](http://msdn.microsoft.com/en-us/library/hh343270.aspx)) 9 | * Working with blob leases ([blog post](http://blog.smarx.com/posts/leasing-windows-azure-blobs-using-the-storage-client-library)) 10 | * Testing existence of blobs and containers ([blog post](http://blog.smarx.com/posts/testing-existence-of-a-windows-azure-blob)) 11 | * Updating queue message visibility timeouts and content ([storage team blog post](http://blogs.msdn.com/b/windowsazurestorage/archive/2011/09/15/windows-azure-queues-improved-leases-progress-tracking-and-scheduling-of-future-work.aspx)) 12 | * Upsert and server-side query projection ([storage team blog post](http://blogs.msdn.com/b/windowsazurestorage/archive/2011/09/15/windows-azure-tables-introducing-upsert-and-query-projection.aspx)) 13 | * A convenience method to initialize storage by creating containers, tables, and queues in a single call 14 | 15 | Basic Usage 16 | ----------- 17 | 18 | This console app initializes storage by creating a container, queue, and table: 19 | 20 | public static void Main(string[] args) 21 | { 22 | var account = CloudStorageAccount.Parse(args[0]); 23 | account.Ensure(containers: new [] { "mycontainer" }, queues: new [] { "myqueue" }, tables: new [] { "mytable" }); 24 | } 25 | 26 | This console app tries to acquire a lease on a blob, and (if it succeeds), writes "Hello World" in the blob: 27 | 28 | static void Main(string[] args) 29 | { 30 | var blob = CloudStorageAccount.Parse(args[0]).CreateCloudBlobClient().GetBlobReference(args[1]); 31 | var leaseId = blob.TryAcquireLease(); 32 | if (leaseId != null) 33 | { 34 | blob.UploadText("Hello, World!", leaseId); 35 | blob.ReleaseLease(leaseId); 36 | Console.WriteLine("Blob written!"); 37 | } 38 | else 39 | { 40 | Console.WriteLine("Blob could not be leased."); 41 | } 42 | } 43 | 44 | This console app tests for the existence of a blob: 45 | 46 | static void Main(string[] args) 47 | { 48 | var blob = CloudStorageAccount.Parse(args[0]).CreateCloudBlobClient().GetBlobReference(args[1]); 49 | Console.WriteLine("The blob {0}.", blob.Exists() ? "exists" : "doesn't exist"); 50 | } 51 | -------------------------------------------------------------------------------- /WazStorageExtensions.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Debug 5 | AnyCPU 6 | 8.0.30703 7 | 2.0 8 | {DC796415-5760-4021-8E92-CB615569D84E} 9 | Library 10 | Properties 11 | smarx.WazStorageExtensions 12 | smarx.WazStorageExtensions 13 | v4.0 14 | 512 15 | .\ 16 | true 17 | 18 | 19 | true 20 | full 21 | false 22 | bin\Debug\ 23 | DEBUG;TRACE 24 | prompt 25 | 4 26 | 27 | 28 | pdbonly 29 | true 30 | bin\Release\ 31 | TRACE 32 | prompt 33 | 4 34 | 35 | 36 | 37 | packages\Microsoft.WindowsAzure.ConfigurationManager.1.7.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll 38 | 39 | 40 | packages\WindowsAzure.Storage.1.7.0.0\lib\net35-full\Microsoft.WindowsAzure.StorageClient.dll 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 70 | -------------------------------------------------------------------------------- /.nuget/NuGet.targets: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | $(MSBuildProjectDirectory)\..\ 5 | 6 | 7 | $([System.IO.Path]::Combine($(SolutionDir), ".nuget")) 8 | $([System.IO.Path]::Combine($(ProjectDir), "packages.config")) 9 | $([System.IO.Path]::Combine($(SolutionDir), "packages")) 10 | 11 | 12 | $(SolutionDir).nuget 13 | packages.config 14 | $(SolutionDir)packages 15 | 16 | 17 | $(NuGetToolsPath)\nuget.exe 18 | "$(NuGetExePath)" 19 | mono --runtime=v4.0.30319 $(NuGetExePath) 20 | 21 | $(TargetDir.Trim('\\')) 22 | 23 | 24 | "" 25 | 26 | 27 | false 28 | 29 | 30 | false 31 | 32 | 33 | $(NuGetCommand) install "$(PackagesConfig)" -source $(PackageSources) -o "$(PackagesDir)" 34 | $(NuGetCommand) pack "$(ProjectPath)" -p Configuration=$(Configuration) -o "$(PackageOutputDir)" -symbols 35 | 36 | 37 | 38 | RestorePackages; 39 | $(BuildDependsOn); 40 | 41 | 42 | 43 | 44 | $(BuildDependsOn); 45 | BuildPackage; 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 57 | 58 | 61 | 62 | 63 | 64 | 66 | 67 | 70 | 71 | -------------------------------------------------------------------------------- /LeaseExtensions.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.WindowsAzure.StorageClient; 2 | using Microsoft.WindowsAzure.StorageClient.Protocol; 3 | using System; 4 | using System.IO; 5 | using System.Threading; 6 | using System.Net; 7 | 8 | namespace smarx.WazStorageExtensions 9 | { 10 | public static class LeaseBlobExtensions 11 | { 12 | public static string TryAcquireLease(this CloudBlob blob) 13 | { 14 | try { return blob.AcquireLease(); } 15 | catch (WebException e) 16 | { 17 | if ((e.Response == null) || ((HttpWebResponse)e.Response).StatusCode != HttpStatusCode.Conflict) // 409, already leased 18 | { 19 | throw; 20 | } 21 | e.Response.Close(); 22 | return null; 23 | } 24 | } 25 | 26 | public static string AcquireLease(this CloudBlob blob) 27 | { 28 | var creds = blob.ServiceClient.Credentials; 29 | var transformedUri = new Uri(creds.TransformUri(blob.Uri.AbsoluteUri)); 30 | var req = BlobRequest.Lease(transformedUri, 31 | 90, // timeout (in seconds) 32 | LeaseAction.Acquire, // as opposed to "break" "release" or "renew" 33 | null); // name of the existing lease, if any 34 | blob.ServiceClient.Credentials.SignRequest(req); 35 | using (var response = req.GetResponse()) 36 | { 37 | return response.Headers["x-ms-lease-id"]; 38 | } 39 | } 40 | 41 | private static void DoLeaseOperation(CloudBlob blob, string leaseId, LeaseAction action) 42 | { 43 | var creds = blob.ServiceClient.Credentials; 44 | var transformedUri = new Uri(creds.TransformUri(blob.Uri.AbsoluteUri)); 45 | var req = BlobRequest.Lease(transformedUri, 90, action, leaseId); 46 | creds.SignRequest(req); 47 | req.GetResponse().Close(); 48 | } 49 | 50 | public static void ReleaseLease(this CloudBlob blob, string leaseId) 51 | { 52 | DoLeaseOperation(blob, leaseId, LeaseAction.Release); 53 | } 54 | 55 | public static bool TryRenewLease(this CloudBlob blob, string leaseId) 56 | { 57 | try { blob.RenewLease(leaseId); return true; } 58 | catch { return false; } 59 | } 60 | 61 | public static void RenewLease(this CloudBlob blob, string leaseId) 62 | { 63 | DoLeaseOperation(blob, leaseId, LeaseAction.Renew); 64 | } 65 | 66 | public static void BreakLease(this CloudBlob blob) 67 | { 68 | DoLeaseOperation(blob, null, LeaseAction.Break); 69 | } 70 | 71 | // NOTE: This method doesn't do everything that the regular UploadText does. 72 | // Notably, it doesn't update the BlobProperties of the blob (with the new 73 | // ETag and LastModifiedTimeUtc). It also, like all the methods in this file, 74 | // doesn't apply any retry logic. Use this at your own risk! 75 | public static void UploadText(this CloudBlob blob, string text, string leaseId) 76 | { 77 | string url = blob.Uri.AbsoluteUri; 78 | if (blob.ServiceClient.Credentials.NeedsTransformUri) 79 | { 80 | url = blob.ServiceClient.Credentials.TransformUri(url); 81 | } 82 | var req = BlobRequest.Put(new Uri(blob.ServiceClient.Credentials.TransformUri(blob.Uri.AbsoluteUri)), 83 | 90, new BlobProperties(), BlobType.BlockBlob, leaseId, 0); 84 | using (var writer = new StreamWriter(req.GetRequestStream())) 85 | { 86 | writer.Write(text); 87 | } 88 | blob.ServiceClient.Credentials.SignRequest(req); 89 | req.GetResponse().Close(); 90 | } 91 | 92 | public static void SetMetadata(this CloudBlob blob, string leaseId) 93 | { 94 | var req = BlobRequest.SetMetadata(new Uri(blob.ServiceClient.Credentials.TransformUri(blob.Uri.AbsoluteUri)), 90, leaseId); 95 | foreach (string key in blob.Metadata.Keys) 96 | { 97 | req.Headers.Add("x-ms-meta-" + key, blob.Metadata[key]); 98 | } 99 | blob.ServiceClient.Credentials.SignRequest(req); 100 | req.GetResponse().Close(); 101 | } 102 | } 103 | } -------------------------------------------------------------------------------- /AutoRenewLease.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.WindowsAzure.StorageClient; 2 | using System; 3 | using System.Threading; 4 | using System.Net; 5 | using System.Globalization; 6 | 7 | namespace smarx.WazStorageExtensions 8 | { 9 | public class AutoRenewLease : IDisposable 10 | { 11 | public bool HasLease { get { return leaseId != null; } } 12 | 13 | private CloudBlob blob; 14 | private string leaseId; 15 | private Thread renewalThread; 16 | private bool disposed = false; 17 | 18 | public static void DoOnce(CloudBlob blob, Action action) { DoOnce(blob, action, TimeSpan.FromSeconds(5)); } 19 | public static void DoOnce(CloudBlob blob, Action action, TimeSpan pollingFrequency) 20 | { 21 | // blob.Exists has the side effect of calling blob.FetchAttributes, which populates the metadata collection 22 | while (!blob.Exists() || blob.Metadata["progress"] != "done") 23 | { 24 | using (var arl = new AutoRenewLease(blob)) 25 | { 26 | if (arl.HasLease) 27 | { 28 | action(); 29 | blob.Metadata["progress"] = "done"; 30 | blob.SetMetadata(arl.leaseId); 31 | } 32 | else 33 | { 34 | Thread.Sleep(pollingFrequency); 35 | } 36 | } 37 | } 38 | } 39 | 40 | public static void DoEvery(CloudBlob blob, TimeSpan interval, Action action) 41 | { 42 | while (true) 43 | { 44 | var lastPerformed = DateTimeOffset.MinValue; 45 | using (var arl = new AutoRenewLease(blob)) 46 | { 47 | if (arl.HasLease) 48 | { 49 | blob.FetchAttributes(); 50 | DateTimeOffset.TryParseExact(blob.Metadata["lastPerformed"], "R", CultureInfo.CurrentCulture, DateTimeStyles.AdjustToUniversal, out lastPerformed); 51 | if (DateTimeOffset.UtcNow >= lastPerformed + interval) 52 | { 53 | action(); 54 | lastPerformed = DateTimeOffset.UtcNow; 55 | blob.Metadata["lastPerformed"] = lastPerformed.ToString("R"); 56 | blob.SetMetadata(arl.leaseId); 57 | } 58 | } 59 | } 60 | var timeLeft = (lastPerformed + interval) - DateTimeOffset.UtcNow; 61 | var minimum = TimeSpan.FromSeconds(5); // so we're not polling the leased blob too fast 62 | Thread.Sleep( 63 | timeLeft > minimum 64 | ? timeLeft 65 | : minimum); 66 | } 67 | } 68 | 69 | public AutoRenewLease(CloudBlob blob) 70 | { 71 | this.blob = blob; 72 | blob.Container.CreateIfNotExist(); 73 | try 74 | { 75 | blob.UploadByteArray(new byte[0], new BlobRequestOptions { AccessCondition = AccessCondition.IfNoneMatch("*") }); 76 | } 77 | catch (StorageClientException e) 78 | { 79 | if (e.ErrorCode != StorageErrorCode.BlobAlreadyExists 80 | && e.StatusCode != HttpStatusCode.PreconditionFailed) // 412 from trying to modify a blob that's leased 81 | { 82 | throw; 83 | } 84 | } 85 | leaseId = blob.TryAcquireLease(); 86 | if (HasLease) 87 | { 88 | renewalThread = new Thread(() => 89 | { 90 | while (true) 91 | { 92 | Thread.Sleep(TimeSpan.FromSeconds(40)); 93 | blob.RenewLease(leaseId); 94 | } 95 | }); 96 | renewalThread.Start(); 97 | } 98 | } 99 | 100 | public void Dispose() 101 | { 102 | Dispose(true); 103 | GC.SuppressFinalize(this); 104 | } 105 | 106 | protected virtual void Dispose(bool disposing) 107 | { 108 | if (!disposed) 109 | { 110 | if (disposing) 111 | { 112 | if (renewalThread != null) 113 | { 114 | renewalThread.Abort(); 115 | blob.ReleaseLease(leaseId); 116 | renewalThread = null; 117 | } 118 | } 119 | disposed = true; 120 | } 121 | } 122 | 123 | ~AutoRenewLease() 124 | { 125 | Dispose(false); 126 | } 127 | } 128 | } --------------------------------------------------------------------------------