├── nuget ├── publish.cmd ├── StorageTypeProvider.fsx ├── README.md ├── FSharp.Azure.StorageTypeProvider.nuspec └── init.ps1 ├── .paket ├── paket.exe └── paket.targets ├── docs ├── files │ ├── img │ │ └── logo.png │ └── deedle.css ├── content │ ├── sample.config │ ├── TableSchema.json │ ├── BlobSchema.json │ ├── license.md │ ├── index.fsx │ ├── hot-schema-loading.fsx │ ├── release-notes.md │ ├── quickstart.fsx │ ├── queues.fsx │ └── blobs.fsx └── tools │ ├── references.fsx │ ├── templates │ └── template.cshtml │ ├── formatters.fsx │ └── generate.fsx ├── tests └── IntegrationTests │ ├── paket.references │ ├── Program.fs │ ├── ExpectoHelpers.fs │ ├── TableSchema.json │ ├── BlobSchema.json │ ├── QueueHelpers.fs │ ├── ConnectionTests.fs │ ├── IntegrationTests.fsproj │ ├── ResetTestData.fsx │ ├── TableHelpers.fs │ ├── QueueUnitTests.fs │ └── BlobUnitTests.fs ├── appveyor.yml ├── src └── FSharp.Azure.StorageTypeProvider │ ├── paket.references │ ├── AssemblyInfo.fs │ ├── Queue │ ├── QueueRepository.fs │ ├── QueueMemberFactory.fs │ └── ProvidedQueueTypes.fs │ ├── Table │ ├── SharedTableTypes.fs │ ├── StaticSchema.fs │ ├── TableMemberFactory.fs │ ├── TableQueryBuilder.fs │ ├── ProvidedTableTypes.fs │ └── TableRepository.fs │ ├── FSharp.Azure.StorageTypeProvider.fsproj │ ├── Blob │ ├── StaticSchema.fs │ ├── BlobMemberFactory.fs │ ├── BlobRepository.fs │ ├── MimeTypes.fs │ ├── BlobRepositoryAsync.fs │ └── ProvidedBlobTypes.fs │ ├── Configuration.fs │ ├── Shared.fs │ └── AzureTypeProvider.fs ├── .github └── ISSUE_TEMPLATE.md ├── README.md ├── paket.dependencies ├── LICENSE.txt ├── sampleparser.fsx ├── UnitTests.sln ├── .gitattributes ├── FSharp.Azure.StorageTypeProvider.sln ├── .gitignore └── RELEASE_NOTES.md /nuget/publish.cmd: -------------------------------------------------------------------------------- 1 | @for %%f in (..\bin\*.nupkg) do @..\.nuget\NuGet.exe push %%f -------------------------------------------------------------------------------- /.paket/paket.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fsprojects/AzureStorageTypeProvider/HEAD/.paket/paket.exe -------------------------------------------------------------------------------- /docs/files/img/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fsprojects/AzureStorageTypeProvider/HEAD/docs/files/img/logo.png -------------------------------------------------------------------------------- /tests/IntegrationTests/paket.references: -------------------------------------------------------------------------------- 1 | Unquote 2 | WindowsAzure.Storage 3 | Expecto 4 | FSharp.Compiler.Tools 5 | Newtonsoft.Json 6 | Microsoft.NET.Test.Sdk 7 | Taskbuilder.fs -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | init: 2 | - git config --global core.autocrlf input 3 | build_script: 4 | - cmd: build.cmd 5 | test: off 6 | artifacts: 7 | - path: bin\*.nupkg 8 | version: '2.0.{build}' -------------------------------------------------------------------------------- /nuget/StorageTypeProvider.fsx: -------------------------------------------------------------------------------- 1 | #I @"lib/netstandard2.0" 2 | 3 | #r "netstandard" 4 | #r @"Newtonsoft.Json.dll" 5 | #r @"Microsoft.WindowsAzure.Storage.dll" 6 | #r @"FSharp.Azure.StorageTypeProvider.dll" -------------------------------------------------------------------------------- /tests/IntegrationTests/Program.fs: -------------------------------------------------------------------------------- 1 | open Expecto 2 | 3 | [] 4 | let main args = 5 | let config = { defaultConfig with verbosity = Logging.LogLevel.Info; ``parallel`` = false } 6 | Expecto.Tests.runTestsInAssembly config args -------------------------------------------------------------------------------- /docs/content/sample.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /src/FSharp.Azure.StorageTypeProvider/paket.references: -------------------------------------------------------------------------------- 1 | WindowsAzure.Storage 2 | FSharp.Core 3 | System.Configuration.ConfigurationManager 4 | System.Net.Http 5 | File:ProvidedTypes.fsi 6 | File:ProvidedTypes.fs 7 | FSharp.Compiler.Tools 8 | Taskbuilder.fs -------------------------------------------------------------------------------- /tests/IntegrationTests/ExpectoHelpers.fs: -------------------------------------------------------------------------------- 1 | [] 2 | module Expecto.AzureHelpers 3 | 4 | let beforeAfter before after test () = 5 | try before(); test() |> ignore 6 | finally after() 7 | let beforeAfterAsync before after test = async { 8 | before() 9 | try return! test 10 | finally after() } 11 | 12 | let shouldEqual a b = Expect.equal b a "" 13 | -------------------------------------------------------------------------------- /docs/content/TableSchema.json: -------------------------------------------------------------------------------- 1 | { 2 | "Employee": { 3 | "Name": { 4 | "Type": "String" 5 | }, 6 | "YearsWorking": { 7 | "Type": "int32" 8 | }, 9 | "Dob": { 10 | "Type": "DateTime" 11 | }, 12 | "Salary": { 13 | "Type": "double", 14 | "Optional": true 15 | }, 16 | "IsManager": { 17 | "Type": "boolean", 18 | "Optional": true 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /tests/IntegrationTests/TableSchema.json: -------------------------------------------------------------------------------- 1 | { 2 | "MyTable": { 3 | "ColumnA": { 4 | "Type": "String", 5 | "Optional": true 6 | }, 7 | "ColumnB": { 8 | "Type": "DateTime", 9 | "Optional": true 10 | }, 11 | "ColumnC": { 12 | "Type": "Boolean" 13 | } 14 | }, 15 | "YourTable": { 16 | "ColumnA": { 17 | "Type": "String" 18 | }, 19 | "ColumnB": { 20 | "Type": "DateTime" 21 | } 22 | } 23 | } -------------------------------------------------------------------------------- /docs/content/BlobSchema.json: -------------------------------------------------------------------------------- 1 | { 2 | "samples": { 3 | "file1.txt": { "Type": "blockblob" }, 4 | "file2.txt": null, 5 | "file3.txt": { "Type": "pageblob" }, 6 | "folder/": { 7 | "childFile.txt": null 8 | }, 9 | "folder2/": { 10 | "child/": { 11 | "descendant4.txt": null 12 | } 13 | } 14 | }, 15 | "random": { 16 | "file.txt": null, 17 | "folder/": { 18 | "emptyFolder/": null 19 | } 20 | }, 21 | "emptyContainer": { } 22 | } -------------------------------------------------------------------------------- /nuget/README.md: -------------------------------------------------------------------------------- 1 | This file is in the `nuget` directory. 2 | 3 | You should use this directory to store any artifacts required to produce a NuGet package for your project. 4 | This typically includes a `.nuspec` file and some `.ps1` scripts, for instance. 5 | Additionally, this example project includes a `.cmd` file suitable for manual deployment of packages to http://nuget.org. 6 | 7 | --- 8 | NOTE: 9 | 10 | This file is a placeholder, used to preserve directory structure in Git. 11 | 12 | This file does not need to be edited. 13 | -------------------------------------------------------------------------------- /tests/IntegrationTests/BlobSchema.json: -------------------------------------------------------------------------------- 1 | { 2 | "samples": { 3 | "file1.txt": { "Type": "blockblob" }, 4 | "file2.txt": null, 5 | "file3.txt": { "Type": "pageblob" }, 6 | "folder/": { 7 | "childFile.txt": null 8 | }, 9 | "folder2/": { 10 | "child/": { 11 | "descendant4.txt": null 12 | } 13 | } 14 | }, 15 | "random": { 16 | "file.txt": null, 17 | "folder/": { 18 | "emptyFolder/": null 19 | } 20 | }, 21 | "emptyContainer": {} 22 | } -------------------------------------------------------------------------------- /docs/tools/references.fsx: -------------------------------------------------------------------------------- 1 | #r @"..\..\bin\FSharp.Azure.StorageTypeProvider.dll" 2 | #r @"..\..\bin\Microsoft.Data.OData.dll" 3 | #r @"..\..\bin\Microsoft.Data.Services.Client.dll" 4 | #r @"..\..\bin\Microsoft.Data.Edm.dll" 5 | #r @"..\..\bin\Microsoft.WindowsAzure.Storage.dll" 6 | #r @"..\..\bin\Newtonsoft.Json.dll" 7 | #r @"..\..\bin\System.Spatial.dll" 8 | #r @"System.Xml.Linq.dll" 9 | #r @"..\..\packages\Deedle\lib\net40\Deedle.dll" 10 | do fsi.AddPrinter(fun (printer:Deedle.Internal.IFsiFormattable) -> "\n" + (printer.Format())) 11 | #load @"..\..\tests\integrationtests\ResetTestData.fsx" 12 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Description 2 | 3 | Please provide a succinct description of your issue. 4 | 5 | ### Repro steps 6 | 7 | Please provide the steps required to reproduce the problem 8 | 9 | 1. Step A 10 | 11 | 2. Step B 12 | 13 | ### Expected behavior 14 | 15 | Please provide a description of the behavior you expect. 16 | 17 | ### Actual behavior 18 | 19 | Please provide a description of the actual behavior you observe. 20 | 21 | ### Known workarounds 22 | 23 | Please provide a description of any known workarounds. 24 | 25 | ### Related information 26 | 27 | * Operating system 28 | * Branch 29 | * .NET Runtime, CoreCLR or Mono Version 30 | * Performance information, links to performance testing scripts 31 | -------------------------------------------------------------------------------- /nuget/FSharp.Azure.StorageTypeProvider.nuspec: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | @project@ 5 | @build.number@ 6 | @title@ 7 | @authors@ 8 | @authors@ 9 | https://github.com/fsprojects/AzureStorageTypeProvider 10 | http://fsprojects.github.io/AzureStorageTypeProvider/img/logo.png 11 | false 12 | @description@ 13 | @tags@ 14 | @summary@ 15 | @releaseNotes@ 16 | @dependencies@ 17 | @references@ 18 | 19 | @files@ 20 | -------------------------------------------------------------------------------- /src/FSharp.Azure.StorageTypeProvider/AssemblyInfo.fs: -------------------------------------------------------------------------------- 1 | // Auto-Generated by FAKE; do not edit 2 | namespace System 3 | open System.Reflection 4 | 5 | [] 6 | [] 7 | [] 8 | [] 9 | [] 10 | do () 11 | 12 | module internal AssemblyVersionInformation = 13 | let [] AssemblyTitle = "FSharp.Azure.StorageTypeProvider" 14 | let [] AssemblyProduct = "FSharp.Azure.StorageTypeProvider" 15 | let [] AssemblyDescription = "Allows easy access to Azure Storage assets through F# scripts." 16 | let [] AssemblyVersion = "2.0.1" 17 | let [] AssemblyFileVersion = "2.0.1" 18 | -------------------------------------------------------------------------------- /nuget/init.ps1: -------------------------------------------------------------------------------- 1 | # This script copies the Azure Storage dlls into the package folder where Azure Type Provider lives 2 | param($installPath, $toolsPath, $package) 3 | 4 | # This is where we want to copy Azure Storage dlls to. 5 | $destPath = $installPath + "\lib\net452\" 6 | 7 | # Get the Windows Storage folder 8 | $folders = ("WindowsAzure.Storage", "Microsoft.Data.Services.Client", "Microsoft.Data.OData", "Microsoft.Data.Edm", "System.Spatial") 9 | 10 | Foreach($folder in $folders) 11 | { 12 | $deps = Get-ChildItem -Path ($installPath + "\..\") | Where-Object { $_.Name.Contains($folder) } 13 | Foreach($d in $deps) 14 | { 15 | $files = Get-ChildItem ($d.FullName + "\lib\net452") | where {$_.PSIsContainer -eq $False} 16 | Foreach ($file in $files) 17 | { 18 | # This gets executed each time project is loaded, so skip files if they exist already 19 | Copy-Item $file.FullName ($destPath + $file.Name) -Force -ErrorAction SilentlyContinue 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /tests/IntegrationTests/QueueHelpers.fs: -------------------------------------------------------------------------------- 1 | module FSharp.Azure.StorageTypeProvider.QueueHelpers 2 | 3 | open Microsoft.WindowsAzure.Storage 4 | open Microsoft.WindowsAzure.Storage.Queue 5 | 6 | let private queueClient = CloudStorageAccount.DevelopmentStorageAccount.CreateCloudQueueClient() 7 | 8 | let private resetQueue name = 9 | let queue = queueClient.GetQueueReference name 10 | queue.DeleteIfExistsAsync().Result |> ignore 11 | queue.CreateAsync() |> Async.AwaitTask |> Async.RunSynchronously 12 | 13 | let private addMessage (queue:CloudQueue) (text:string) = queue.AddMessageAsync(CloudQueueMessage text) |> Async.AwaitTask |> Async.RunSynchronously 14 | 15 | let resetData() = 16 | [ "sample-queue"; "second-sample"; "third-sample" ] 17 | |> List.map resetQueue 18 | |> ignore 19 | 20 | let secondQueue = queueClient.GetQueueReference "second-sample" 21 | 22 | [ "Hello from Azure Type Provider" 23 | "F# is cool" 24 | "Azure is also pretty great" ] 25 | |> List.iter (addMessage secondQueue) 26 | 27 | 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Azure Storage Type Provider 2 | =========================== 3 | 4 | An F# Azure Type Provider which can be used to explore Azure Storage assets quickly and easily. 5 | 6 | The goal is to create a provider which allows lightweight access to your Azure storage data within the context of e.g. F# scripts or applications to allow CRUD operations quickly and easily. 7 | 8 | Support exists reading and writing from Blobs, Tables and Queues, as well as fallback to the standard .NET Azure SDK. 9 | 10 | The provider currently does not support .NET Core for the reasons specified [here](https://github.com/fsprojects/AzureStorageTypeProvider/issues/111#issuecomment-417208922). 11 | 12 | [![AppVeyor build status](https://ci.appveyor.com/api/projects/status/github/fsprojects/AzureStorageTypeProvider?svg=true)](https://ci.appveyor.com/project/fsprojectsgit/azurestoragetypeprovider) 13 | 14 | ### Maintainer(s) 15 | 16 | - [@isaacabraham](https://github.com/isaacabraham) 17 | 18 | The default maintainer account for projects under "fsprojects" is [@fsprojectsgit](https://github.com/fsprojectsgit) - F# Community Project Incubation Space (repo management) 19 | -------------------------------------------------------------------------------- /paket.dependencies: -------------------------------------------------------------------------------- 1 | redirects: force 2 | source https://www.nuget.org/api/v2/ 3 | framework: netstandard2.0, netcoreapp2.0 4 | generate_load_scripts: true 5 | 6 | nuget FSharp.Core 7 | nuget WindowsAzure.Storage 9.3.2 8 | nuget Newtonsoft.Json 10.0.2 9 | nuget System.Net.Http 10 | nuget FSharp.Compiler.Tools 11 | 12 | nuget FAKE 13 | nuget Nuget.CommandLine 14 | 15 | nuget Deedle 16 | nuget FSharp.Charting 17 | nuget FSharp.Formatting 18 | 19 | nuget Expecto 20 | nuget Unquote 21 | nuget Microsoft.NET.Test.Sdk 22 | nuget Taskbuilder.fs 23 | 24 | github fsprojects/FSharp.TypeProviders.SDK src/ProvidedTypes.fsi 25 | github fsprojects/FSharp.TypeProviders.SDK src/ProvidedTypes.fs 26 | nuget Microsoft.NETCore.Runtime.CoreCLR 27 | nuget System.Configuration.ConfigurationManager 28 | 29 | group build 30 | 31 | source https://www.nuget.org/api/v2/ 32 | framework: netstandard2.0, netcoreapp2.0 33 | storage: none 34 | 35 | nuget Fake.Azure.Emulators 36 | nuget Fake.BuildServer.AppVeyor 37 | nuget Fake.Core 38 | nuget Fake.Core.ReleaseNotes 39 | nuget Fake.Core.Targets 40 | nuget Fake.DotNet 41 | nuget Fake.DotNet.Cli 42 | nuget Fake.DotNet.Fsi 43 | nuget Fake.DotNet.AssemblyInfoFile 44 | nuget Fake.DotNet.NuGet 45 | nuget Fake.IO.FileSystem 46 | nuget Fake.Tools.Git -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | -------------------------------------------------------------------------------- /docs/content/license.md: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | -------------------------------------------------------------------------------- /docs/files/deedle.css: -------------------------------------------------------------------------------- 1 | #main .deedleframe, #main .deedleseries { 2 | margin:0px 0px 20px 20px; 3 | } 4 | 5 | #main .deedleframe td, #main .deedleframe th { 6 | padding:4px 10px 4px 10px; 7 | min-width:50px; 8 | } 9 | #main .deedleseries td, #main .deedleseries th { 10 | padding:4px 20px 4px 10px; 11 | } 12 | #main .deedleframe td:first-child, #main .deedleframe th:first-child { 13 | min-width:30px; 14 | } 15 | #main .deedleseries thead th, 16 | #main .deedleseries td:first-child, 17 | #main .deedleframe thead th, 18 | #main .deedleframe td:first-child { 19 | padding:4px 20px 4px 10px; 20 | text-align:left; 21 | font-weight:bold; 22 | } 23 | #main .deedleseries table, 24 | #main .deedleframe table { 25 | border-left:1px solid #e0e0e0; 26 | border-top:1px solid #e0e0e0; 27 | } 28 | #main .deedleseries td, #main .deedleseries th, 29 | #main .deedleframe td, #main .deedleframe th { 30 | border-right:1px solid #e0e0e0; 31 | border-bottom:1px solid #e0e0e0; 32 | } 33 | #main .deedleseries thead th, 34 | #main .deedleframe thead th, 35 | #main .deedleframe tr.even { 36 | background:white; 37 | } 38 | #main .deedleseries tr.odd, 39 | #main .deedleframe tr.odd { 40 | background:#f8f8f8; 41 | } 42 | 43 | #main .deedleseries p, 44 | #main .deedleframe p { 45 | font-size:90%; 46 | margin:0px; 47 | padding:0px; 48 | line-height:normal; 49 | } 50 | -------------------------------------------------------------------------------- /tests/IntegrationTests/ConnectionTests.fs: -------------------------------------------------------------------------------- 1 | module ConnectionTests 2 | 3 | open FSharp.Azure.StorageTypeProvider 4 | open Swensen.Unquote 5 | open Expecto 6 | 7 | type SecondBlank = AzureTypeProvider<"","Bar"> 8 | type FirstBlank = AzureTypeProvider<"", "Foo"> 9 | //type TwoPart = AzureTypeProvider<"devstoreaccount1", "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="> 10 | //type FullPath = AzureTypeProvider<"DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;", "test"> 11 | 12 | [] 13 | let connectionTests = 14 | testList "Connection Tests" [ 15 | testCase "SecondBlank connection string works" (fun _ -> Expect.isTrue (SecondBlank.Containers.CloudBlobClient.GetContainerReference("samples").ExistsAsync() |> Async.AwaitTask |> Async.RunSynchronously) "") 16 | testCase "FirstBlank connection string works" (fun _ -> Expect.isTrue (FirstBlank.Containers.CloudBlobClient.GetContainerReference("samples").ExistsAsync() |> Async.AwaitTask |> Async.RunSynchronously) "") 17 | ] 18 | //[] 19 | //let ``TwoPart connection string works``() = 20 | // TwoPart.Containers.CloudBlobClient.GetContainerReference("samples").Exists() =! true 21 | // 22 | //[] 23 | //let ``FullPath connection string works``() = 24 | // FullPath.Containers.CloudBlobClient.GetContainerReference("samples").Exists() =! true 25 | // 26 | -------------------------------------------------------------------------------- /tests/IntegrationTests/IntegrationTests.fsproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | UnitTests 4 | UnitTests 5 | IntegrationTests 6 | netcoreapp2.0 7 | Exe 8 | true 9 | netstandard2.0 10 | win10-x64 11 | false 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | ..\..\bin\netstandard2.0\publish\FSharp.Azure.StorageTypeProvider.dll 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /sampleparser.fsx: -------------------------------------------------------------------------------- 1 | #load @".paket\load\windowsazure.storage.fsx" 2 | 3 | module Raw = 4 | type Column = { Column : string; Type : string; Optional : bool } 5 | type Table = { Table : string; Columns : Column array } 6 | type Schema = { Tables : Table array } 7 | 8 | module Parsed = 9 | open System 10 | open Microsoft.WindowsAzure.Storage.Table 11 | let parseEdmType (value:string) = 12 | match value.ToLower() with 13 | | "binary" -> EdmType.Binary 14 | | "boolean" -> EdmType.Boolean 15 | | "datetime" -> EdmType.DateTime 16 | | "double" -> EdmType.Double 17 | | "guid" -> EdmType.Guid 18 | | "int32" -> EdmType.Int32 19 | | "int64" -> EdmType.Int64 20 | | "string" -> EdmType.String 21 | | value -> failwithf "Unknown column type '%s'" value 22 | 23 | type Column = { Column : string; Type : EdmType; Optional : bool } 24 | type Table = { Table : string; Columns : Column array } 25 | type Schema = { Tables : Table array } 26 | 27 | open Newtonsoft.Json 28 | 29 | let schema = 30 | let schema = JsonConvert.DeserializeObject(System.IO.File.ReadAllText "table-schema.json") 31 | 32 | { Parsed.Schema.Tables = 33 | schema.Tables 34 | |> Array.map(fun t -> 35 | { Table = t.Table 36 | Columns = 37 | t.Columns 38 | |> Array.map(fun c -> 39 | { Column = c.Column 40 | Type = Parsed.parseEdmType c.Type 41 | Optional = c.Optional }) }) } -------------------------------------------------------------------------------- /UnitTests.sln: -------------------------------------------------------------------------------- 1 | Microsoft Visual Studio Solution File, Format Version 12.00 2 | # Visual Studio 15 3 | VisualStudioVersion = 15.0.28010.2046 4 | MinimumVisualStudioVersion = 10.0.40219.1 5 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".paket", ".paket", "{CBD0ED53-D40D-4CC2-B4DE-DE97ECEE3A6B}" 6 | ProjectSection(SolutionItems) = preProject 7 | paket.dependencies = paket.dependencies 8 | paket.lock = paket.lock 9 | EndProjectSection 10 | EndProject 11 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{B7A3503A-B8A9-4C26-8DCD-8E66425AFC91}" 12 | ProjectSection(SolutionItems) = preProject 13 | build.fsx = build.fsx 14 | README.md = README.md 15 | RELEASE_NOTES.md = RELEASE_NOTES.md 16 | EndProjectSection 17 | EndProject 18 | Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "IntegrationTests", "tests\IntegrationTests\IntegrationTests.fsproj", "{C579EF90-6907-4E8B-831A-CF9FE8F6F901}" 19 | EndProject 20 | Global 21 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 22 | Debug|Any CPU = Debug|Any CPU 23 | Release|Any CPU = Release|Any CPU 24 | EndGlobalSection 25 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 26 | {C579EF90-6907-4E8B-831A-CF9FE8F6F901}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 27 | {C579EF90-6907-4E8B-831A-CF9FE8F6F901}.Debug|Any CPU.Build.0 = Debug|Any CPU 28 | {C579EF90-6907-4E8B-831A-CF9FE8F6F901}.Release|Any CPU.ActiveCfg = Release|Any CPU 29 | {C579EF90-6907-4E8B-831A-CF9FE8F6F901}.Release|Any CPU.Build.0 = Release|Any CPU 30 | EndGlobalSection 31 | GlobalSection(SolutionProperties) = preSolution 32 | HideSolutionNode = FALSE 33 | EndGlobalSection 34 | GlobalSection(ExtensibilityGlobals) = postSolution 35 | SolutionGuid = {874626DB-FA47-4F63-A762-6F05CBAFC611} 36 | EndGlobalSection 37 | EndGlobal 38 | -------------------------------------------------------------------------------- /src/FSharp.Azure.StorageTypeProvider/Queue/QueueRepository.fs: -------------------------------------------------------------------------------- 1 | module internal FSharp.Azure.StorageTypeProvider.Queue.QueueRepository 2 | 3 | open FSharp.Azure.StorageTypeProvider 4 | open Microsoft.WindowsAzure.Storage 5 | open Microsoft.WindowsAzure.Storage.Queue 6 | open System 7 | 8 | let internal getQueueClient connectionString = CloudStorageAccount.Parse(connectionString).CreateCloudQueueClient() 9 | 10 | [] 11 | module private SdkExtensions = 12 | type CloudQueueClient with 13 | member cloudQueueClient.ListQueuesAsync() = 14 | let getTables token = async { 15 | let! result = cloudQueueClient.ListQueuesSegmentedAsync token |> Async.AwaitTask 16 | return result.ContinuationToken, result.Results } 17 | Async.segmentedAzureOperation getTables 18 | 19 | let getQueues connectionString = async { 20 | try 21 | let client = getQueueClient connectionString 22 | let! queues = client.ListQueuesAsync() 23 | return queues |> Array.map (fun q -> q.Name) 24 | with 25 | | :? StorageException as ex when ex.RequestInformation.HttpStatusCode = 501 -> return Array.empty } 26 | 27 | let getQueueRef name = getQueueClient >> (fun q -> q.GetQueueReference name) 28 | 29 | let peekMessages connectionString name = getQueueRef name connectionString |> (fun x -> x.PeekMessagesAsync >> fun t -> t.Result) 30 | 31 | let generateSas start duration queuePermissions (queue:CloudQueue) = 32 | let policy = SharedAccessQueuePolicy(Permissions = queuePermissions, 33 | SharedAccessStartTime = (start |> Option.map(fun start -> DateTimeOffset start) |> Option.toNullable), 34 | SharedAccessExpiryTime = Nullable(DateTimeOffset.UtcNow.Add duration)) 35 | queue.GetSharedAccessSignature(policy, null) -------------------------------------------------------------------------------- /src/FSharp.Azure.StorageTypeProvider/Table/SharedTableTypes.fs: -------------------------------------------------------------------------------- 1 | namespace FSharp.Azure.StorageTypeProvider.Table 2 | 3 | open Microsoft.WindowsAzure.Storage.Table 4 | open System 5 | 6 | /// The different types of insertion mechanism to use. 7 | type TableInsertMode = 8 | /// Insert if the entity does not already exist. 9 | | Insert = 0 10 | /// Insert if the entity does not already exist; otherwise overwrite the entity. 11 | | Upsert = 1 12 | 13 | 14 | /// The type of property 15 | type internal PropertyNeed = 16 | /// The property is optional. 17 | | Optional 18 | /// The property is mandatory. 19 | | Mandatory 20 | 21 | type internal ColumnDefinition = { Name : string; ColumnType : EdmType; PropertyNeed : PropertyNeed } 22 | 23 | /// The name of the partition. 24 | type Partition = | Partition of string 25 | /// The row key. 26 | type Row = | Row of string 27 | /// Represents a Partition and Row combined to key a single entity. 28 | type EntityId = Partition * Row 29 | 30 | /// Different responses from a table operation. 31 | type TableResponse = 32 | /// The operation was successful. 33 | | SuccessfulResponse of EntityId * HttpCode : int 34 | /// The operation for this specific entity failed. 35 | | EntityError of EntityId * HttpCode : int * ErrorCode : string 36 | /// The operation for this specific entity was not carried out because an operation for another entity in the same batch failed. 37 | | BatchOperationFailedError of EntityId 38 | /// An unknown error occurred in this batch. 39 | | BatchError of EntityId * HttpCode : int * ErrorCode : string 40 | 41 | /// Represents a single table entity. 42 | type LightweightTableEntity 43 | internal (partitionKey:Partition, rowKey:Row, timestamp:DateTimeOffset, values:Map) = 44 | 45 | let (Partition pkey) = partitionKey 46 | let (Row rkey) = rowKey 47 | 48 | /// The Partition Key of the entity. 49 | member __.PartitionKey with get () = pkey 50 | /// The Row Key of the entity. 51 | member __.RowKey with get () = rkey 52 | /// The timestamp of the entity. 53 | member __.Timestamp with get () = timestamp 54 | /// A collection of key/value pairs of all other properties on this entity. 55 | member __.Values with get () = values -------------------------------------------------------------------------------- /src/FSharp.Azure.StorageTypeProvider/FSharp.Azure.StorageTypeProvider.fsproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | FSharp.Azure.StorageTypeProvider 5 | FSharp.Azure.StorageTypeProvider 6 | FSharp.Azure.StorageTypeProvider 7 | netstandard2.0 8 | true 9 | ..\..\bin\ 10 | ..\..\bin\FSharp.Azure.StorageTypeProvider.XML 11 | true 12 | false 13 | 14 | 15 | 16 | 17 | True 18 | paket-files/ProvidedTypes.fsi 19 | 20 | 21 | True 22 | paket-files/ProvidedTypes.fs 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /src/FSharp.Azure.StorageTypeProvider/Blob/StaticSchema.fs: -------------------------------------------------------------------------------- 1 | module internal FSharp.Azure.StorageTypeProvider.Blob.StaticSchema 2 | 3 | open BlobRepository 4 | open FSharp.Azure.StorageTypeProvider.Configuration 5 | open Microsoft.WindowsAzure.Storage.Blob 6 | open Newtonsoft.Json.Linq 7 | open System.IO 8 | 9 | let failInvalidJson() = failwith "Invalid JSON structure in static blob schema." 10 | 11 | let private (|Folder|File|) (name:string) = 12 | if name.EndsWith "/" then Folder else File 13 | 14 | let rec buildBlobItem prevPath (elementName, contents) = 15 | match elementName, contents with 16 | | File, Json.ObjectOrNull _ -> 17 | let blobType = 18 | match contents.TryGetProperty "Type" with 19 | | Some (Json.String "blockblob") -> BlobType.BlockBlob 20 | | Some (Json.String "pageblob") -> BlobType.PageBlob 21 | | Some content -> failwithf "Unknown value for blob type ('%A'). Must be either 'blockblob' or 'pageblob'." content 22 | | None -> BlobType.BlockBlob 23 | Blob (prevPath + elementName, elementName, blobType, None) 24 | | Folder, Json.ObjectOrNull children -> 25 | let path = prevPath + elementName 26 | Folder (path, elementName, async { return children |> Array.map (buildBlobItem path) }) 27 | | _ -> failInvalidJson() 28 | 29 | let buildBlobSchema (json:Json.Json) = 30 | json.AsObject 31 | |> Array.map (fun (containerName, containerElements) -> 32 | { Name = containerName 33 | Contents = 34 | async { 35 | return 36 | match containerElements with 37 | | Json.ObjectOrNull elements -> elements |> Array.map (buildBlobItem "") 38 | | _ -> failInvalidJson() } } ) 39 | |> Array.toList 40 | 41 | let createSchema resolutionFolder path = 42 | path 43 | |> Option.map(fun path -> 44 | let paths = [ path; Path.Combine(resolutionFolder, path) ] 45 | match paths |> List.tryFind File.Exists with 46 | | None -> Error (exn (sprintf "Could not locate schema file. Searched: %A " paths)) 47 | | Some file -> 48 | try 49 | file 50 | |> File.ReadAllText 51 | |> JToken.Parse 52 | |> Json.ofJToken 53 | |> buildBlobSchema 54 | |> Ok 55 | with ex -> Error ex) 56 | |> defaultArg <| Ok [] -------------------------------------------------------------------------------- /src/FSharp.Azure.StorageTypeProvider/Table/StaticSchema.fs: -------------------------------------------------------------------------------- 1 | module FSharp.Azure.StorageTypeProvider.Table.StaticSchema 2 | 3 | open FSharp.Azure.StorageTypeProvider.Configuration 4 | open Newtonsoft.Json.Linq 5 | open System.IO 6 | 7 | module internal Parsed = 8 | open Microsoft.WindowsAzure.Storage.Table 9 | let parseEdmType (value:string) = 10 | match value.ToLower() with 11 | | "binary" -> EdmType.Binary 12 | | "boolean" -> EdmType.Boolean 13 | | "datetime" -> EdmType.DateTime 14 | | "double" -> EdmType.Double 15 | | "guid" -> EdmType.Guid 16 | | "int32" -> EdmType.Int32 17 | | "int64" -> EdmType.Int64 18 | | "string" -> EdmType.String 19 | | value -> failwithf "Unknown column type '%s'" value 20 | 21 | type Table = { Table : string; Columns : ColumnDefinition list } 22 | type TableSchema = { Tables : Table array } 23 | 24 | let internal buildTableSchema rawJson = 25 | let json = rawJson |> JToken.Parse |> Json.ofJToken 26 | 27 | { Parsed.TableSchema.Tables = 28 | json.AsObject 29 | |> Array.map (fun (tableName, columns) -> 30 | { Table = tableName 31 | Columns = 32 | columns.AsObject 33 | |> Array.map (fun (columnName, properties) -> 34 | let colType = Parsed.parseEdmType (properties.GetProperty "Type").AsString 35 | let propNeed = 36 | match properties.TryGetProperty "Optional" with 37 | | Some (Json.Boolean true) -> Optional 38 | | Some (Json.Boolean false) | None -> Mandatory 39 | | Some _ -> failwith "Optional must be a boolean value." 40 | { Name = columnName 41 | ColumnType = colType 42 | PropertyNeed = propNeed }) 43 | |> Array.toList }) } 44 | 45 | let internal createSchema resolutionFolder path = 46 | path 47 | |> Option.map(fun path -> 48 | let paths = [ path; Path.Combine(resolutionFolder, path) ] 49 | match paths |> List.tryFind File.Exists with 50 | | None -> Error (exn (sprintf "Could not locate schema file. Searched: %A " paths)) 51 | | Some file -> 52 | try 53 | file 54 | |> File.ReadAllText 55 | |> buildTableSchema 56 | |> Some 57 | |> Ok 58 | with ex -> Error ex) 59 | |> defaultArg <| Ok None 60 | 61 | -------------------------------------------------------------------------------- /tests/IntegrationTests/ResetTestData.fsx: -------------------------------------------------------------------------------- 1 | // This script sets up local azure storage to a well-known state for integration tests. 2 | 3 | #load @"..\..\.paket\load\netstandard2.0\WindowsAzure.Storage.fsx" 4 | #r "netstandard" 5 | #r "System.Text.Encoding" 6 | 7 | open Microsoft.WindowsAzure.Storage 8 | open System.Text 9 | 10 | let logWith entity func = 11 | printfn "Resetting %s data..." entity 12 | func() 13 | printfn "Done!" 14 | 15 | let createData _ = 16 | let blobClient = CloudStorageAccount.DevelopmentStorageAccount.CreateCloudBlobClient() 17 | let container = blobClient.GetContainerReference "samples" 18 | 19 | if container.ExistsAsync().Result then container.DeleteAsync().Wait() 20 | container.CreateAsync().Wait() 21 | 22 | let createBlockBlob fileName contents = 23 | let blob = container.GetBlockBlobReference fileName 24 | blob.UploadTextAsync(contents).Wait() 25 | 26 | let createPageBlob fileName (contents:string) = 27 | let blob = container.GetPageBlobReference fileName 28 | 29 | let bytes = 30 | let data = contents |> Encoding.UTF8.GetBytes |> ResizeArray 31 | let output = Array.init (data.Count - (data.Count % 512) + 512) (fun _ -> 0uy) 32 | data.CopyTo output 33 | output 34 | blob.UploadFromByteArrayAsync(bytes, 0, bytes.Length).Wait() 35 | 36 | createBlockBlob "file1.txt" "stuff" 37 | createBlockBlob "file2.txt" "more stuff" 38 | createBlockBlob "file3.txt" "even more stuff" 39 | createBlockBlob "folder/childFile.txt" "child file stuff" 40 | createBlockBlob "sample.txt" "the quick brown fox jumps over the lazy dog\nLorem ipsum dolor sit amet, consectetur adipiscing elit. Cras malesuada.\nLorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla porttitor." 41 | createBlockBlob "data.xml" "thing" 42 | createBlockBlob "folder2/child/grandchild1/descedant1.txt" "not important" 43 | createBlockBlob "folder2/child/grandchild1/descedant2.txt" "not important" 44 | createBlockBlob "folder2/child/grandchild2/descedant3.txt" "not important" 45 | createBlockBlob "folder2/child/descedant4.txt" "not important" 46 | createBlockBlob "folder/pageDataChild.txt" "hello from child page blob" 47 | createPageBlob "pageData.bin" "hello from page blob" 48 | 49 | createData |> logWith "blob" 50 | 51 | #load "TableHelpers.fs" 52 | #load "QueueHelpers.fs" 53 | 54 | open FSharp.Azure.StorageTypeProvider 55 | TableHelpers.resetData |> logWith "table" 56 | QueueHelpers.resetData |> logWith "queue" 57 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /.paket/paket.targets: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | true 6 | 7 | true 8 | $(MSBuildThisFileDirectory) 9 | $(MSBuildThisFileDirectory)..\ 10 | 11 | 12 | 13 | $(PaketToolsPath)paket.exe 14 | $(PaketToolsPath)paket.bootstrapper.exe 15 | "$(PaketExePath)" 16 | mono --runtime=v4.0.30319 $(PaketExePath) 17 | "$(PaketBootStrapperExePath)" 18 | mono --runtime=v4.0.30319 $(PaketBootStrapperExePath) 19 | 20 | $(MSBuildProjectDirectory)\paket.references 21 | $(MSBuildProjectFullPath).paket.references 22 | $(PaketCommand) restore --references-files "$(PaketReferences)" 23 | $(PaketBootStrapperCommand) 24 | 25 | RestorePackages; $(BuildDependsOn); 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /FSharp.Azure.StorageTypeProvider.sln: -------------------------------------------------------------------------------- 1 | Microsoft Visual Studio Solution File, Format Version 12.00 2 | # Visual Studio 15 3 | VisualStudioVersion = 15.0.28010.2046 4 | MinimumVisualStudioVersion = 10.0.40219.1 5 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".paket", ".paket", "{94066CAA-4417-4C43-A286-40182E920E43}" 6 | ProjectSection(SolutionItems) = preProject 7 | paket.dependencies = paket.dependencies 8 | paket.lock = paket.lock 9 | EndProjectSection 10 | EndProject 11 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{B7A3503A-B8A9-4C26-8DCD-8E66425AFC91}" 12 | ProjectSection(SolutionItems) = preProject 13 | build.fsx = build.fsx 14 | docs\tools\generate.fsx = docs\tools\generate.fsx 15 | README.md = README.md 16 | RELEASE_NOTES.md = RELEASE_NOTES.md 17 | EndProjectSection 18 | EndProject 19 | Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Azure.StorageTypeProvider", "src\FSharp.Azure.StorageTypeProvider\FSharp.Azure.StorageTypeProvider.fsproj", "{FB7CA8F3-C158-49E9-B816-501741E2921F}" 20 | EndProject 21 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docs", "docs", "{7CAF8D38-6250-41CB-A65B-ABD760B09281}" 22 | ProjectSection(SolutionItems) = preProject 23 | docs\tools\references.fsx = docs\tools\references.fsx 24 | docs\tools\templates\template.cshtml = docs\tools\templates\template.cshtml 25 | EndProjectSection 26 | EndProject 27 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "content", "content", "{15350981-90B9-44A0-B356-776735ACC7F1}" 28 | ProjectSection(SolutionItems) = preProject 29 | docs\content\blobs.fsx = docs\content\blobs.fsx 30 | docs\content\BlobSchema.json = docs\content\BlobSchema.json 31 | docs\content\index.fsx = docs\content\index.fsx 32 | docs\content\queues.fsx = docs\content\queues.fsx 33 | docs\content\quickstart.fsx = docs\content\quickstart.fsx 34 | docs\content\sample.config = docs\content\sample.config 35 | docs\content\tables.fsx = docs\content\tables.fsx 36 | EndProjectSection 37 | EndProject 38 | Global 39 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 40 | Debug|Any CPU = Debug|Any CPU 41 | Release|Any CPU = Release|Any CPU 42 | EndGlobalSection 43 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 44 | {FB7CA8F3-C158-49E9-B816-501741E2921F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 45 | {FB7CA8F3-C158-49E9-B816-501741E2921F}.Debug|Any CPU.Build.0 = Debug|Any CPU 46 | {FB7CA8F3-C158-49E9-B816-501741E2921F}.Release|Any CPU.ActiveCfg = Release|Any CPU 47 | {FB7CA8F3-C158-49E9-B816-501741E2921F}.Release|Any CPU.Build.0 = Release|Any CPU 48 | EndGlobalSection 49 | GlobalSection(SolutionProperties) = preSolution 50 | HideSolutionNode = FALSE 51 | EndGlobalSection 52 | GlobalSection(NestedProjects) = preSolution 53 | {15350981-90B9-44A0-B356-776735ACC7F1} = {7CAF8D38-6250-41CB-A65B-ABD760B09281} 54 | EndGlobalSection 55 | GlobalSection(ExtensibilityGlobals) = postSolution 56 | SolutionGuid = {C68560AB-A7F2-49A3-BEED-B56A8D7074D2} 57 | EndGlobalSection 58 | EndGlobal 59 | -------------------------------------------------------------------------------- /docs/content/index.fsx: -------------------------------------------------------------------------------- 1 | (*** hide ***) 2 | #load @"..\tools\references.fsx" 3 | 4 | (** 5 | About the Azure Storage Type Provider 6 | ===================================== 7 | The F# Azure Storage Type Provider allows quick and easy exploration of your 8 | Azure Storage assets (Blobs, Tables and Queues) through the F# type system, 9 | allowing for rapid access to large amounts of data cheaply, both through 10 | scripts and applications. A fall-back to the standard .NET Azure SDK is also 11 | provided. 12 | 13 | Example 14 | ------- 15 | 16 | This example illustrates some of the features available from the type provider. 17 | 18 | *) 19 | 20 | open FSharp.Azure.StorageTypeProvider 21 | 22 | // Get a handle to my local storage emulator 23 | type Azure = AzureTypeProvider<"UseDevelopmentStorage=true"> 24 | 25 | // Navigate through the containers to a specific file and read the contents. 26 | let blobContents = 27 | Azure.Containers.samples.``folder/``.``childFile.txt``.Read() 28 | 29 | // Perform a strongly-typed query against a table with automatic schema generation. 30 | let results = 31 | Azure.Tables.employee.Query() 32 | .``Where Name Is``.``Equal To``("fred") 33 | .Execute() 34 | |> Array.map(fun row -> row.Name, row.Dob) 35 | 36 | // Navigate through storage queues and get messages 37 | let queueMessage = Azure.Queues.``sample-queue``.Dequeue() 38 | 39 | (** 40 | 41 | Samples & documentation 42 | ----------------------- 43 | 44 | The library comes with comprehensible documentation. 45 | 46 | * The [quickstart](quickstart.html) contains further examples of how to get up and running as well the guiding principles for the library. 47 | 48 | * There are detailed tutorials for the [Blob](blobs.html), [Table](tables.html) and [Queue](queues.html) 49 | APIs. 50 | 51 | * [API Reference](reference/index.html) contains automatically generated documentation for all types, modules 52 | and functions in the library. This includes additional brief samples on using most of the 53 | functions. 54 | 55 | Contributing and copyright 56 | -------------------------- 57 | 58 | The project is hosted on [GitHub][gh] where you can [report issues][issues], fork 59 | the project and submit pull requests. If you're adding a new public API, please also 60 | consider adding [samples][content] that can be turned into a documentation. You might 61 | also want to read the [library design notes][readme] to understand how it works. 62 | 63 | The library is available under Public Domain license, which allows modification and 64 | redistribution for both commercial and non-commercial purposes. For more information see the 65 | [License file][license] in the GitHub repository. 66 | 67 | [content]: https://github.com/fsprojects/FSharp.ProjectScaffold/tree/master/docs/content 68 | [gh]: https://github.com/fsprojects/FSharp.ProjectScaffold 69 | [issues]: https://github.com/fsprojects/FSharp.ProjectScaffold/issues 70 | [readme]: https://github.com/fsprojects/FSharp.ProjectScaffold/blob/master/README.md 71 | [license]: https://github.com/fsprojects/FSharp.ProjectScaffold/blob/master/LICENSE.txt 72 | *) 73 | -------------------------------------------------------------------------------- /src/FSharp.Azure.StorageTypeProvider/Configuration.fs: -------------------------------------------------------------------------------- 1 | module internal FSharp.Azure.StorageTypeProvider.Configuration 2 | 3 | open System.Collections.Generic 4 | open System.Configuration 5 | open System.IO 6 | open Microsoft.WindowsAzure.Storage 7 | 8 | let private doesFileExist folder file = 9 | let fullPath = Path.Combine(folder, file) 10 | if fullPath |> File.Exists then Some fullPath else None 11 | 12 | let private (|NoConfigRequested|RequestedConfigExists|RequestedConfigDoesNotExist|) (configFile, resolutionFolder) = 13 | let toOption s = if s = "" then None else Some s 14 | let configFile = configFile |> toOption |> Option.map (doesFileExist resolutionFolder) 15 | match configFile with 16 | | None -> NoConfigRequested 17 | | Some (Some configFile) -> RequestedConfigExists configFile 18 | | Some None -> RequestedConfigDoesNotExist 19 | 20 | let private (|DefaultConfigExists|NoDefaultConfigFound|) resolutionFolder = 21 | [ "app.config"; "web.config" ] 22 | |> List.tryPick (doesFileExist resolutionFolder) 23 | |> Option.map(fun config -> DefaultConfigExists config) 24 | |> defaultArg <| NoDefaultConfigFound 25 | 26 | let getConnectionString(connectionName: string, resolutionFolder, requestedConfig) = 27 | let configPath = 28 | match (requestedConfig, resolutionFolder), resolutionFolder with 29 | | RequestedConfigExists configPath, _ 30 | | NoConfigRequested, DefaultConfigExists configPath -> configPath 31 | | RequestedConfigDoesNotExist, _ -> raise <| FileNotFoundException(sprintf "Could not find config file '%s' in path '%s'." requestedConfig resolutionFolder) 32 | | NoConfigRequested, NoDefaultConfigFound -> failwithf "Cannot find either app.config or web.config in path '%s'." resolutionFolder 33 | let map = ExeConfigurationFileMap(ExeConfigFilename = configPath) 34 | let configSection = ConfigurationManager.OpenMappedExeConfiguration(map, ConfigurationUserLevel.None).ConnectionStrings.ConnectionStrings 35 | 36 | match configSection, lazy configSection.[connectionName] with 37 | | null, _ -> raise <| KeyNotFoundException(sprintf "Cannot find the section of %s file." configPath) 38 | | _, Lazy null -> raise <| KeyNotFoundException(sprintf "Cannot find name %s in section of %s file." connectionName configPath) 39 | | _, Lazy x -> x.ConnectionString 40 | 41 | 42 | [] 43 | module ConnectionValidation = 44 | let private memoize code = 45 | let cache = Dictionary() 46 | fun arg -> 47 | if not(cache.ContainsKey arg) 48 | then cache.[arg] <- code arg 49 | cache.[arg] 50 | let private checkConnectionString connectionString = 51 | try 52 | CloudStorageAccount 53 | .Parse(connectionString) 54 | .CreateCloudBlobClient() 55 | .GetContainerReference("abc") 56 | .ExistsAsync() 57 | |> Async.AwaitTask 58 | |> Async.RunSynchronously //throws an exception if attempted with an invalid connection string 59 | |> ignore 60 | Ok() 61 | with | ex -> Error ex 62 | let validateConnectionString = memoize checkConnectionString -------------------------------------------------------------------------------- /docs/content/hot-schema-loading.fsx: -------------------------------------------------------------------------------- 1 | (** 2 | Hot Schema Loading 3 | ================== 4 | 5 | As of 1.9.4, the type provider has support for "hot schema loading" of types. This is extremely useful when working 6 | in an exploratory mode on top of a storage account, particularly when adding and removing data at the same time. 7 | 8 | When Hot Schema Loading is activated, the type provider will on a regular basis "refresh" any types used in your scripts or 9 | F# files based on the latest schema in real time - that is, either any schema files or the live data source used. 10 | 11 | ## Activating Hot Schema Loading 12 | To turn on Hot Schema Loading, simply provide a value for the `autoRefresh` argument in the type provider. This argument 13 | represents the number of *seconds* between refreshes. 14 | 15 | Note that Hot Schema Loading is **not** turned on by default. 16 | *) 17 | 18 | /// A handle to local development storage whose schema refreshes every 5 seconds. 19 | type Azure = AzureTypeProvider<"UseDevelopmentStorage=true", autoRefresh = 5> 20 | 21 | (** 22 | ## Hot Schema Loading in practice 23 | This section outlines some of the uses of Hot Schema Loading across the different types of services in Azure Storage. 24 | 25 | ### Working with Blobs 26 | * As containers or blobs are renamed, added or deleted, the type system will automatically show compiler errors if your code references 27 | a blob that no longer exists. There's no need to reload your IDE; it will happen automatically. 28 | * From F# 4.1 onwards, the compiler will automatically suggest potential renames for closely matching blobs. 29 | 30 | ### Working with Tables 31 | * As Tables are created or deleted, they will update within your code. 32 | * As data changes within your tables, schema inference will automatically re-evaluate your code - new properties will show, removed properties 33 | will cause compiler errors and optional parameters will appear / remove as needed. 34 | 35 | ### Working with Queues 36 | * As Queues are created or deleted, they will update within your code. 37 | * The `Peek` member will automatically refresh with the latest contents on the queue. 38 | 39 | ## Costs with Hot Schema Loading 40 | Be aware that Hot Schema Loading makes repeated requests to your Azure Storage accounts to keep your schema up-to-date, and 41 | you should factor in costs associated with this. Whilst Azure Storage costs for storage, ingress and egress are very competitive, 42 | consider the following points to ensure that you do not end up with an unexpected bill. 43 | 44 | ### Use the local development emulator 45 | If you do not need to use a live account for development purposes, consider installing the Azure Storage Development emulator. 46 | This does not cost anything and allows you to seamlessly switch from local to live storage accounts. 47 | 48 | ### Set auto refresh at a high value 49 | Consider setting a relatively high schema refresh timeout e.g. 5 minutes to avoid repeated requests on a regular basis. 50 | 51 | ## Working with scripts 52 | When working with scripts, bear in mind that the **running FSI session** *does not auto-refresh*. It is *only* the IDE itself e.g. Visual Studio, 53 | Code or Rider etc. that benefits from hot schema loading. This can leave the IDE and FSI out of sync, and you may wish to occasionally 54 | reset / reload FSI if schema changes occur in order to re-load the latest schema into FSI. 55 | *) 56 | 57 | -------------------------------------------------------------------------------- /docs/tools/templates/template.cshtml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | @Title 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 22 | 23 | 24 |
25 |
26 | 30 |

@Properties["project-name"]

31 |
32 |
33 |
34 |
35 | @RenderBody() 36 |
37 |
38 | F# Project 39 | 58 |
59 |
60 |
61 | Fork me on GitHub 62 | 63 | 64 | -------------------------------------------------------------------------------- /docs/content/release-notes.md: -------------------------------------------------------------------------------- 1 | ### 0.9.0 - 8th May 2014 2 | * Fixed nuget package deployment 3 | * Sorted namespacing 4 | * Refactored lots of code 5 | * Added better error handling 6 | * Introduced F# scaffold project structure 7 | 8 | ### 0.9.1 - 16th May 2014 9 | * Fixed namespace clash between Type Provider and parent provider type class 10 | * Added ability to retrieve the name of a blob and table programmatically 11 | * Removed EntityId from the LightweightTableEntity and replaced with separate PartitionKey and RowKey properties 12 | * LightweightTableEntity made public and changed to a class rather than a record 13 | 14 | ### 0.9.2 - 12th June 2014 15 | * Fixed a bug whereby provided type constructors sometimes mixed up field names and their values. 16 | 17 | ### 1.0.0 - 14th June 2014 18 | * Forced Get to provide a Partition Key as well as Row Key to semantically reflect that it can never return more than 1 result (this removes the exception path). 19 | 20 | ### 1.1.0 - 13th August 2014 21 | * Perf improvement for batch inserts when building lightweight table entities. 22 | * Azure Storage Queue support. 23 | 24 | ### 1.2.0 - 20th April 2015 25 | * Upgrade to Azure Storage 4.3.0 SDK. 26 | * Added hooks into native Azure Storage SDK directly in the type provider. 27 | * Simplified some naming of methods on blobs. 28 | 29 | ### 1.3.0 - 27th August 2015 30 | * Lots of work on documentation. 31 | * Blobs now support connection string overrides. 32 | * Support for Page Blobs. 33 | * Go back to .NET 4.0 dependency - no need to push up to 4.5.1. 34 | * No longer add Azure SDK references to client projects. 35 | * Breaking change: downloadFolder() now requires the caller to explicitly start the Async workflow to being downloading. 36 | 37 | ### 1.4.0 - 28th October 2015 38 | * Configuration file support. 39 | 40 | ### 1.4.1 - 8th January 2016 41 | * Remove obsolete reference from StorageTypeProvider.fsx 42 | 43 | ### 1.5.0 - 24th June 2016 44 | * Better connection string management 45 | * Eager table type loading 46 | * Delete entire partitions 47 | * Better handling with large batches 48 | * Blob listing on folders 49 | 50 | ### 1.6.0 - 27th July 2016 51 | * Async support for Tables 52 | * BREAKING CHANGE: Lazy deserialization of AsString and AsBytes members on Queue messages 53 | * BREAKING CHANGE: Automatic inference of option types for fields that are missing for some rows 54 | * Name property on containers 55 | * Visibility Timeout control on queue messages 56 | * Minor bug fixes 57 | 58 | ### 1.6.1 - 3rd August 2016 59 | * Improve synchronous bulk table operation performance 60 | * Fix asynchronous bulk table operation error handling 61 | 62 | ### 1.7.0 - 6th October 2016 63 | * Support for Azure Metrics tables. 64 | * Support for simple humanization of table properties. 65 | 66 | ### 1.8.0 - 28th March 2017 67 | * Upgrade to F# 4.0 68 | * Upgrade to .NET452 69 | * Support for static blob schemas 70 | 71 | ### 1.9.0 - 6th May 2017 72 | * Better support for programmatic blob access 73 | * Static Table schema support 74 | * BREAKING CHANGE: Size property on Blobs changed to a method. 75 | * BREAKING CHANGE: Reworked static Blob schema 76 | 77 | ### 1.9.1 - 8th May 2017 78 | * Support for specifying blob type in blob schema definition. 79 | 80 | ### 1.9.2 - 31st May 2017 81 | * Support for MIME type assignment on blob upload. 82 | * Support for blob file and container properties 83 | * Upgrade to Azure Storage 7.2.1 84 | 85 | ### 1.9.3 - 25th August 2017 86 | * Blob and Queue SAS tokens are now configurable. 87 | * Ability to supply a prefix when listing blobs. 88 | 89 | ### 1.9.4 - 29th October 2017 90 | * Hot Schema Loading. 91 | 92 | ### 1.9.5 - 9th November 2017 93 | * Re-enable storage metrics support. -------------------------------------------------------------------------------- /src/FSharp.Azure.StorageTypeProvider/Shared.fs: -------------------------------------------------------------------------------- 1 | namespace global 2 | 3 | ///[omit] 4 | module Option = 5 | open System 6 | let ofString text = if String.IsNullOrWhiteSpace text then None else Some text 7 | 8 | ///[omit] 9 | module Async = 10 | open System 11 | 12 | let map mapper workflow = 13 | async { 14 | let! result = workflow 15 | return mapper result } 16 | 17 | 18 | let toAsyncResult workflow = 19 | workflow 20 | |> Async.Catch 21 | |> map (function 22 | | Choice1Of2 success -> Ok success 23 | | Choice2Of2 (:? AggregateException as agg) -> Error (agg.InnerExceptions |> Seq.toList) 24 | | Choice2Of2 err -> Error [ err ]) 25 | 26 | ///[omit] 27 | [] 28 | [] 29 | module Json = 30 | open Newtonsoft.Json.Linq 31 | type ObjectData = string * Json 32 | and Json = 33 | | String of string 34 | | Number of decimal 35 | | Boolean of bool 36 | | Object of ObjectData[] 37 | | Array of Json[] 38 | | Null 39 | member this.AsString = match this with Json.String s -> s | _ -> failwith "Expected a JSON string" 40 | member this.AsNumber = match this with Json.Number s -> s | _ -> failwith "Expected a JSON number" 41 | member this.AsBoolean = match this with Json.Boolean b -> b | _ -> failwith "Expected a JSON boolean" 42 | member this.AsObject = match this with Json.Object o -> o | _ -> failwith "Expected a JSON object" 43 | member this.AsArray = match this with Json.Array o -> o | _ -> failwith "Expected a JSON array" 44 | member this.TryGetProperty key = 45 | match this with 46 | | Json.Object data -> Some data 47 | | _ -> None 48 | |> Option.bind (Array.tryFind (fst >> (=) key)) 49 | |> Option.map snd 50 | member this.GetProperty key = 51 | match this.TryGetProperty key with 52 | | Some property -> property 53 | | None -> failwithf "Property '%s' does not exist." key 54 | 55 | let rec ofJToken (jToken:JToken) = 56 | match jToken with 57 | | :? JValue as v -> 58 | match v.Value with 59 | | :? string as s -> Json.String s 60 | | :? decimal as d -> Json.Number (decimal d) 61 | | :? int64 as i -> Json.Number (decimal i) 62 | | :? int32 as i -> Json.Number (decimal i) 63 | | :? float as f -> Json.Number (decimal f) 64 | | :? bool as b -> Json.Boolean b 65 | | null -> Json.Null 66 | | v -> failwithf "Cannot convert JSON value type %s" (v.GetType().FullName) 67 | | :? JObject as o -> o.Properties() |> Seq.map (fun p -> p.Name, ofJToken p.Value) |> Seq.toArray |> Json.Object 68 | | :? JArray as a -> a.Values() |> Seq.map ofJToken |> Seq.toArray |> Json.Array 69 | | v -> failwithf "Cannot convert JSON type %s" (v.GetType().FullName) 70 | 71 | /// Match an Object and treat Null as an empty Object 72 | let (|ObjectOrNull|_|) = function 73 | | Json.Object o -> Some o 74 | | Json.Null -> Some [||] 75 | | _ -> None 76 | 77 | namespace FSharp.Azure.StorageTypeProvider 78 | 79 | module internal Async = 80 | let segmentedAzureOperation getNextSegment = 81 | let rec doRecursiveSegmenting (getNextSegment:_ -> Async<_ * _>) (output:_ ResizeArray) token = async { 82 | match! getNextSegment token with 83 | | null, data -> 84 | output.AddRange data 85 | return output.ToArray() 86 | | token, data -> 87 | output.AddRange data 88 | return! doRecursiveSegmenting getNextSegment output token } 89 | 90 | doRecursiveSegmenting getNextSegment (ResizeArray()) null -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.sln.docstates 8 | 9 | # Xamarin Studio / monodevelop user-specific 10 | *.userprefs 11 | 12 | # Build results 13 | 14 | [Dd]ebug/ 15 | [Rr]elease/ 16 | x64/ 17 | build/ 18 | [Bb]in/ 19 | [Oo]bj/ 20 | 21 | # Enable "build/" folder in the NuGet Packages folder since NuGet packages use it for MSBuild targets 22 | !packages/*/build/ 23 | 24 | # MSTest test Results 25 | [Tt]est[Rr]esult*/ 26 | [Bb]uild[Ll]og.* 27 | 28 | *_i.c 29 | *_p.c 30 | *.ilk 31 | *.meta 32 | *.obj 33 | *.pch 34 | *.pdb 35 | *.pgc 36 | *.pgd 37 | *.rsp 38 | *.sbr 39 | *.tlb 40 | *.tli 41 | *.tlh 42 | *.tmp 43 | *.tmp_proj 44 | *.log 45 | *.vspscc 46 | *.vssscc 47 | .builds 48 | *.pidb 49 | *.log 50 | *.scc 51 | 52 | # Visual C++ cache files 53 | ipch/ 54 | *.aps 55 | *.ncb 56 | *.opensdf 57 | *.sdf 58 | *.cachefile 59 | 60 | # Visual Studio profiler 61 | *.psess 62 | *.vsp 63 | *.vspx 64 | 65 | # Guidance Automation Toolkit 66 | *.gpState 67 | 68 | # ReSharper is a .NET coding add-in 69 | _ReSharper*/ 70 | *.[Rr]e[Ss]harper 71 | 72 | # TeamCity is a build add-in 73 | _TeamCity* 74 | 75 | # DotCover is a Code Coverage Tool 76 | *.dotCover 77 | 78 | # NCrunch 79 | *.ncrunch* 80 | .*crunch*.local.xml 81 | 82 | # Installshield output folder 83 | [Ee]xpress/ 84 | 85 | # DocProject is a documentation generator add-in 86 | DocProject/buildhelp/ 87 | DocProject/Help/*.HxT 88 | DocProject/Help/*.HxC 89 | DocProject/Help/*.hhc 90 | DocProject/Help/*.hhk 91 | DocProject/Help/*.hhp 92 | DocProject/Help/Html2 93 | DocProject/Help/html 94 | 95 | # Click-Once directory 96 | publish/ 97 | 98 | # Publish Web Output 99 | *.Publish.xml 100 | 101 | # Enable nuget.exe in the .nuget folder (though normally executables are not tracked) 102 | !.nuget/NuGet.exe 103 | 104 | # Windows Azure Build Output 105 | csx 106 | *.build.csdef 107 | 108 | # Windows Store app package directory 109 | AppPackages/ 110 | 111 | # Others 112 | sql/ 113 | *.Cache 114 | ClientBin/ 115 | [Ss]tyle[Cc]op.* 116 | ~$* 117 | *~ 118 | *.dbmdl 119 | *.[Pp]ublish.xml 120 | *.pfx 121 | *.publishsettings 122 | 123 | # RIA/Silverlight projects 124 | Generated_Code/ 125 | 126 | # Backup & report files from converting an old project file to a newer 127 | # Visual Studio version. Backup files are not needed, because we have git ;-) 128 | _UpgradeReport_Files/ 129 | Backup*/ 130 | UpgradeLog*.XML 131 | UpgradeLog*.htm 132 | 133 | # SQL Server files 134 | App_Data/*.mdf 135 | App_Data/*.ldf 136 | 137 | 138 | #LightSwitch generated files 139 | GeneratedArtifacts/ 140 | _Pvt_Extensions/ 141 | ModelManifest.xml 142 | 143 | # ========================= 144 | # Windows detritus 145 | # ========================= 146 | 147 | # Windows image file caches 148 | Thumbs.db 149 | ehthumbs.db 150 | 151 | # Folder config file 152 | Desktop.ini 153 | 154 | # Recycle Bin used on file shares 155 | $RECYCLE.BIN/ 156 | 157 | # Mac desktop service store files 158 | .DS_Store 159 | 160 | # =================================================== 161 | # Exclude F# project specific directories and files 162 | # =================================================== 163 | 164 | # NuGet Packages Directory 165 | packages/ 166 | 167 | # Generated documentation folder 168 | docs/output/ 169 | 170 | # Temp folder used for publishing docs 171 | temp/ 172 | 173 | # Test results produced by build 174 | TestResults.xml 175 | 176 | # Nuget outputs 177 | nuget/*.nupkg 178 | 179 | .paket/paket.exe 180 | paket-files/ 181 | 182 | #Fake build files 183 | .fake 184 | testoutput/ 185 | .paket/load/ 186 | /.ionide/symbolCache.db 187 | /.vs/FSharp.Azure.StorageTypeProvider/DesignTimeBuild/.dtbcache 188 | /.vs/UnitTests/DesignTimeBuild/.dtbcache 189 | /tests/IntegrationTests/.ionide/symbolCache.db 190 | /src/FSharp.Azure.StorageTypeProvider/.ionide/symbolCache.db 191 | /.ionide/symbolCache.db-journal 192 | -------------------------------------------------------------------------------- /RELEASE_NOTES.md: -------------------------------------------------------------------------------- 1 | ### 0.9.0 - 8th May 2014 2 | * Fixed nuget package deployment 3 | * Sorted namespacing 4 | * Refactored lots of code 5 | * Added better error handling 6 | * Introduced F# scaffold project structure 7 | 8 | ### 0.9.1 - 16th May 2014 9 | * Fixed namespace clash between Type Provider and parent provider type class 10 | * Added ability to retrieve the name of a blob and table programmatically 11 | * Removed EntityId from the LightweightTableEntity and replaced with separate PartitionKey and RowKey properties 12 | * LightweightTableEntity made public and changed to a class rather than a record 13 | 14 | ### 0.9.2 - 12th June 2014 15 | * Fixed a bug whereby provided type constructors sometimes mixed up field names and their values. 16 | 17 | ### 1.0.0 - 14th June 2014 18 | * Forced Get to provide a Partition Key as well as Row Key to semantically reflect that it can never return more than 1 result (this removes the exception path). 19 | 20 | ### 1.1.0 - 13th August 2014 21 | * Perf improvement for batch inserts when building lightweight table entities. 22 | * Azure Storage Queue support. 23 | 24 | ### 1.2.0 - 20th April 2015 25 | * Upgrade to Azure Storage 4.3.0 SDK. 26 | * Added hooks into native Azure Storage SDK directly in the type provider. 27 | * Simplified some naming of methods on blobs. 28 | 29 | ### 1.3.0 - 27th August 2015 30 | * Lots of work on documentation. 31 | * Blobs now support connection string overrides. 32 | * Support for Page Blobs. 33 | * Go back to .NET 4.0 dependency - no need to push up to 4.5.1. 34 | * No longer add Azure SDK references to client projects. 35 | * Breaking change: downloadFolder() now requires the caller to explicitly start the Async workflow to being downloading. 36 | 37 | ### 1.4.0 - 28th October 2015 38 | * Configuration file support. 39 | 40 | ### 1.4.1 - 8th January 2016 41 | * Remove obsolete reference from StorageTypeProvider.fsx 42 | 43 | ### 1.5.0 - 24th June 2016 44 | * Better connection string management 45 | * Eager table type loading 46 | * Delete entire partitions 47 | * Better handling with large batches 48 | * Blob listing on folders 49 | 50 | ### 1.6.0 - 27th July 2016 51 | * Async support for Tables 52 | * BREAKING CHANGE: Lazy deserialization of AsString and AsBytes members on Queue messages 53 | * BREAKING CHANGE: Automatic inference of option types for fields that are missing for some rows 54 | * Name property on containers 55 | * Visibility Timeout control on queue messages 56 | * Minor bug fixes 57 | 58 | ### 1.6.1 - 3rd August 2016 59 | * Improve synchronous bulk table operation performance 60 | * Fix asynchronous bulk table operation error handling 61 | 62 | ### 1.7.0 - 6th October 2016 63 | * Support for Azure Metrics tables. 64 | * Support for simple humanization of table properties. 65 | 66 | ### 1.8.0 - 28th March 2017 67 | * Upgrade to F# 4.0 68 | * Upgrade to .NET452 69 | * Support for static blob schemas 70 | 71 | ### 1.9.0 - 6th May 2017 72 | * Better support for programmatic blob access 73 | * Static Table schema support 74 | * BREAKING CHANGE: Size property on Blobs changed to a method. 75 | * BREAKING CHANGE: Reworked static Blob schema 76 | 77 | ### 1.9.1 - 8th May 2017 78 | * Support for specifying blob type in blob schema definition. 79 | 80 | ### 1.9.2 - 31st May 2017 81 | * Support for MIME type assignment on blob upload. 82 | * Support for blob file and container properties 83 | * Upgrade to Azure Storage 7.2.1 84 | 85 | ### 1.9.3 - 25th August 2017 86 | * Blob and Queue SAS tokens are now configurable. 87 | * Ability to supply a prefix when listing blobs. 88 | 89 | ### 1.9.4 - 29th October 2017 90 | * Hot Schema Loading. 91 | 92 | ### 1.9.5 - 9th November 2017 93 | * Re-enable storage metrics support. 94 | 95 | ### 2.0.0b - 12th November 2018 96 | * Upgrade to NET Standard 2.0 97 | * Upgrade Azure Storage to v9 98 | * Upgrade many other dependencies 99 | * Rationalise many API implementations 100 | * Minor changes to API to expose Async methods on provided types 101 | 102 | ### 2.0.0 - 28th December 2018 103 | * Support for blob-only accounts 104 | 105 | ### 2.0.1 - 05th February 2018 106 | * Fix #119 (Even when JSON tableSchema parameter is used Emulator required to be running) -------------------------------------------------------------------------------- /docs/content/quickstart.fsx: -------------------------------------------------------------------------------- 1 | (*** hide ***) 2 | #load @"..\tools\references.fsx" 3 | open FSharp.Azure.StorageTypeProvider.Table 4 | 5 | (** 6 | Introducing the Azure Storage Type Provider 7 | =========================================== 8 | 9 | The Azure Storage Type Provider is designed to let you rapidly access your 10 | storage assets in Azure from F# without the need to resort to third party tools 11 | or the server explorer in Visual Studio. 12 | 13 | The provider is designed to be as easy as possible to consume, whilst allowing 14 | several benefits over using the standard .NET SDK, such as removing the need for 15 | magic strings when access tables / queues / blobs, as well as strengthening the 16 | typing of queries over tables. 17 | 18 | Connecting to Azure 19 | =================== 20 | 21 | Connecting to your storage account is simple. 22 | 23 | From within a script, first ``#load "StorageTypeProvider.fsx"`` to reference all dependencies. 24 | Then, you can generate a type for a storage account simply by providing your Azure 25 | account credentials via a number of ways. 26 | 27 | *) 28 | 29 | open FSharp.Azure.StorageTypeProvider 30 | 31 | // Connect to a live account using a standard Azure Storage connection string 32 | type Live = 33 | AzureTypeProvider<"DefaultEndpointsProtocol=https;AccountName=name;AccountKey=key;"> 34 | 35 | // Connect to a live account using a two-part name and key. 36 | type LiveTwoPart = AzureTypeProvider<"name", "key"> 37 | 38 | // Connect to local storage emulator 39 | type Local = AzureTypeProvider<"UseDevelopmentStorage=true"> 40 | 41 | // Connect via configuration file with named connection string. 42 | type FromConfig = AzureTypeProvider 43 | 44 | (** 45 | 46 | An example ``sample.config`` file can be found at ``AzureStorageTypeProvider/docs/content/sample.config``. 47 | 48 | Common Themes 49 | ============= 50 | 51 | The API is split into three areas: **Tables**, **Queues** and **Containers**. 52 | 53 | All three share some common themes: - 54 | 55 | - Where possible, entities such as queues, tables, containers, folders or blobs are typed based on 56 | live queries to the appropriate underlying storage SDK. Thus there's no requirement to enter magic 57 | strings for blobs, tables or queues. 58 | 59 | *) 60 | 61 | // Reference the "employee" table. No magic strings needed. 62 | let employee = Local.Tables.employee.Get(Row "1", Partition "somepartition") 63 | 64 | // Navigation through the "samples" container to the "folder/childFile.txt" blob. 65 | let blobContents = Local.Containers.samples.``folder/``.``childFile.txt``.Read() 66 | 67 | // Reference the "sample-queue" queue. 68 | let queueMessage = Local.Queues.``sample-queue``.Dequeue() 69 | 70 | (** 71 | 72 | - Common methods and functionality are made available directly on the appropriate assets, but 73 | there's always access to the raw Azure SDK if you need it to do anything more. 74 | 75 | *) 76 | 77 | // Get the raw Azure Cloud Blob Client and use it to get a ref to the same blob. 78 | let rawBlobRef = Local.Containers.CloudBlobClient.GetContainerReference("samples") 79 | .GetBlockBlobReference("folder/childFile.txt") 80 | 81 | // Or for the Cloud Table Client 82 | let rawTableRef = Local.Tables.CloudTableClient.GetTableReference("employee") 83 | 84 | // Or use a hybrid approach - use the TP to find the asset you want and then switch. 85 | let rawBlobRefHybrid = Local.Containers.samples.``folder/``.``childFile.txt``.AsCloudBlockBlob() 86 | 87 | (** 88 | 89 | - You can override the connection to the destination asset at runtime. This is useful if not using 90 | configuration files so that you can use your local (free) storage emulator for development, and then 91 | move to a live account when testing or moving to production etc.. In addition it's also useful for 92 | data migration scenarios. 93 | 94 | *) 95 | 96 | // Redirect to a blob using a different connection string. 97 | let liveBlob = Local.Containers.samples.``file1.txt``.AsCloudBlockBlob("myOtherConnectionString") 98 | 99 | // Get row 1A from the "employee" table using a different connection string. 100 | let row1A = Local.Tables.employee.Get(Row "1", Partition "A", "myOtherConnectionString") 101 | -------------------------------------------------------------------------------- /src/FSharp.Azure.StorageTypeProvider/Blob/BlobMemberFactory.fs: -------------------------------------------------------------------------------- 1 | /// Generates top-level blob containers folders. 2 | module internal FSharp.Azure.StorageTypeProvider.Blob.BlobMemberFactory 3 | 4 | open FSharp.Azure.StorageTypeProvider.Blob.BlobRepository 5 | open ProviderImplementation.ProvidedTypes 6 | open System 7 | open Microsoft.WindowsAzure.Storage.Blob 8 | open FSharp.Control.Tasks 9 | 10 | let rec private createBlobItem (domainType : ProvidedTypeDefinition) connectionString containerName fileItem = 11 | match fileItem with 12 | | Folder(path, name, contents) -> 13 | let folderProp = ProvidedTypeDefinition((sprintf "%s.%s" containerName path), Some typeof, hideObjectMethods = true) 14 | domainType.AddMember(folderProp) 15 | folderProp.AddMembersDelayed(fun _ -> 16 | contents 17 | |> Async.RunSynchronously 18 | |> Array.choose (createBlobItem domainType connectionString containerName) 19 | |> Array.toList) 20 | Some <| ProvidedProperty(name, folderProp, getterCode = fun _ -> <@@ ContainerBuilder.createBlobFolder connectionString containerName path @@>) 21 | | Blob(path, name, blobType, length) -> 22 | match blobType, length with 23 | | _, Some 0L -> None 24 | | BlobType.PageBlob, _ -> Some <| ProvidedProperty(name, typeof, getterCode = fun _ -> <@@ BlobBuilder.createPageBlobFile connectionString containerName path @@>) 25 | | BlobType.BlockBlob, _ -> 26 | let blobType = 27 | match path with 28 | | BlobBuilder.XML -> typeof 29 | | BlobBuilder.Binary | BlobBuilder.Text -> typeof 30 | Some <| ProvidedProperty(name, blobType, getterCode = fun _ -> <@@ BlobBuilder.createBlockBlobFile connectionString containerName path @@>) 31 | | _ -> None 32 | 33 | let private createContainerType (domainType : ProvidedTypeDefinition) connectionString (container : LightweightContainer) = 34 | let individualContainerType = ProvidedTypeDefinition(container.Name + "Container", Some typeof, hideObjectMethods = true) 35 | individualContainerType.AddXmlDoc <| sprintf "Provides access to the '%s' container." container.Name 36 | individualContainerType.AddMembersDelayed(fun _ -> 37 | container.Contents 38 | |> Async.RunSynchronously 39 | |> Array.choose (createBlobItem domainType connectionString container.Name) 40 | |> Array.toList) 41 | domainType.AddMember individualContainerType 42 | // this local binding is required for the quotation. 43 | let containerName = container.Name 44 | let containerProp = 45 | ProvidedProperty(container.Name, individualContainerType, getterCode = fun _ -> <@@ ContainerBuilder.createContainer connectionString containerName @@>) 46 | containerProp.AddXmlDocDelayed(fun () -> sprintf "Provides access to the '%s' container." containerName) 47 | containerProp 48 | 49 | /// Builds up the Blob Storage container members 50 | let getBlobStorageMembers staticSchema (connectionString, domainType : ProvidedTypeDefinition) = 51 | let containerListingType = ProvidedTypeDefinition("Containers", Some typeof, hideObjectMethods = true) 52 | let createContainerType = createContainerType domainType connectionString 53 | 54 | match staticSchema with 55 | | [] -> containerListingType.AddMembersDelayed(fun _ -> 56 | getBlobStorageAccountManifest connectionString 57 | |> Async.RunSynchronously 58 | |> Array.map createContainerType 59 | |> Array.toList) 60 | | staticSchema -> 61 | staticSchema 62 | |> List.map createContainerType 63 | |> containerListingType.AddMembers 64 | 65 | domainType.AddMember containerListingType 66 | 67 | let cbcProp = ProvidedProperty("CloudBlobClient", typeof, getterCode = (fun _ -> <@@ ContainerBuilder.createBlobClient connectionString @@>)) 68 | cbcProp.AddXmlDoc "Gets a handle to the Blob Azure SDK client for this storage account." 69 | containerListingType.AddMember(cbcProp) 70 | 71 | let containerListingProp = ProvidedProperty("Containers", containerListingType, isStatic = true, getterCode = (fun _ -> <@@ () @@>)) 72 | containerListingProp.AddXmlDoc "Gets the list of all containers in this storage account." 73 | Some containerListingProp -------------------------------------------------------------------------------- /tests/IntegrationTests/TableHelpers.fs: -------------------------------------------------------------------------------- 1 | module FSharp.Azure.StorageTypeProvider.TableHelpers 2 | 3 | open Microsoft.WindowsAzure.Storage 4 | open Microsoft.WindowsAzure.Storage.Table 5 | open System 6 | 7 | let bArr = [| for i in 1 .. 255 do yield i |> byte |] 8 | 9 | type LargeEntity() = 10 | //20 byte array properties with a max size of 64kb each give this entity type a maz size of c. 1.3Mb 11 | inherit TableEntity() 12 | member val ByteArr1 = bArr with get, set 13 | member val ByteArr2 = bArr with get, set 14 | member val ByteArr3 = bArr with get, set 15 | member val ByteArr4 = bArr with get, set 16 | member val ByteArr5 = bArr with get, set 17 | member val ByteArr6 = bArr with get, set 18 | member val ByteArr7 = bArr with get, set 19 | member val ByteArr9 = bArr with get, set 20 | member val ByteArr10 = bArr with get, set 21 | member val ByteArr11= bArr with get, set 22 | member val ByteArr12 = bArr with get, set 23 | member val ByteArr13 = bArr with get, set 24 | member val ByteArr14 = bArr with get, set 25 | member val ByteArr15 = bArr with get, set 26 | member val ByteArr16 = bArr with get, set 27 | member val ByteArr17 = bArr with get, set 28 | member val ByteArr18 = bArr with get, set 29 | member val ByteArr19 = bArr with get, set 30 | member val ByteArr20 = bArr with get, set 31 | 32 | type RandomEntity() = 33 | inherit TableEntity() 34 | member val Name = String.Empty with get, set 35 | member val YearsWorking = 0 with get, set 36 | member val Dob = DateTime.MinValue with get, set 37 | member val Salary = 0.0 with get, set 38 | member val IsManager = false with get, set 39 | 40 | type AlternativeEntity() = 41 | inherit TableEntity() 42 | member val Name = String.Empty with get, set 43 | member val Dob = DateTime.MinValue with get, set 44 | member val Salary = 0.0 with get, set 45 | member val IsManager = false with get, set 46 | member val IsAnimal = true with get, set 47 | 48 | let saveRow (table:CloudTable) = TableOperation.Insert >> table.ExecuteAsync >> Async.AwaitTask >> Async.RunSynchronously >> ignore 49 | 50 | let insertRow (pk, rk, name, yearsWorking, dob, salary, isManager) table = 51 | RandomEntity(Name = name, YearsWorking = yearsWorking, Salary = salary, Dob = dob, PartitionKey = pk, RowKey = rk.ToString(), IsManager = isManager) 52 | |> saveRow table 53 | let insertLargeRow (pk, rk) table = LargeEntity(PartitionKey = pk, RowKey = rk) |> saveRow table 54 | let insertSampleRow (pk, rk, name, dob, salary, isManager, isAnimal) table = 55 | AlternativeEntity(PartitionKey = pk, RowKey = rk.ToString(), Name = name, Dob = dob, Salary = salary, IsManager = isManager, IsAnimal = isAnimal) 56 | |> saveRow table 57 | 58 | let resetData() = 59 | let recreateTable table = 60 | let table = 61 | CloudStorageAccount 62 | .DevelopmentStorageAccount 63 | .CreateCloudTableClient() 64 | .GetTableReference table 65 | if table.ExistsAsync().Result then table.DeleteAsync() |> Async.AwaitTask |> Async.RunSynchronously 66 | table.CreateAsync() |> Async.AwaitTask |> Async.RunSynchronously 67 | table 68 | 69 | let largeTable = recreateTable "large" 70 | largeTable |> insertLargeRow("1","1") 71 | largeTable |> insertLargeRow("1","2") 72 | largeTable |> insertLargeRow("2","1") 73 | 74 | let employeeTable = recreateTable "employee" 75 | recreateTable "emptytable" |> ignore 76 | 77 | employeeTable |> insertRow("men", 1, "fred", 10, DateTime(1990, 5, 1, 0, 0, 0, DateTimeKind.Utc), 0., true) 78 | employeeTable |> insertRow("men", 2, "fred", 35, DateTime(1980, 4, 4, 0, 0, 0, DateTimeKind.Utc), 1.5, false) 79 | employeeTable |> insertRow("men", 3, "tim", 99, DateTime(2001, 10, 5, 0, 0, 0, DateTimeKind.Utc), 10., false) 80 | employeeTable |> insertRow("women", 1, "sara", 35, DateTime(2005, 4, 30, 0, 0, 0, DateTimeKind.Utc), 3.5, true) 81 | employeeTable |> insertRow("women", 2, "rachel", 20, DateTime(1965, 8, 20, 0, 0, 0, DateTimeKind.Utc), 5.5, false) 82 | 83 | let optionalTable = recreateTable "optionals" 84 | optionalTable |> insertRow("partition", 1, "fred", 10, DateTime(1990, 5, 1, 0, 0, 0, DateTimeKind.Utc), 0., true) 85 | optionalTable |> insertSampleRow("partition", 2, "fred", DateTime(1990, 5, 1, 0, 0, 0, DateTimeKind.Utc), 0., true, false) -------------------------------------------------------------------------------- /src/FSharp.Azure.StorageTypeProvider/Queue/QueueMemberFactory.fs: -------------------------------------------------------------------------------- 1 | module internal FSharp.Azure.StorageTypeProvider.Queue.QueueMemberFactory 2 | 3 | open FSharp.Azure.StorageTypeProvider.Queue 4 | open FSharp.Azure.StorageTypeProvider.Queue.QueueRepository 5 | open Microsoft.WindowsAzure.Storage.Queue 6 | open ProviderImplementation.ProvidedTypes 7 | open System 8 | 9 | let private createPeekForQueue (connectionString, domainType:ProvidedTypeDefinition, queueName) = 10 | let peekType = ProvidedTypeDefinition(sprintf "%s.Queues.%s.Peek" connectionString queueName, None, hideObjectMethods = true) 11 | domainType.AddMember peekType 12 | peekType.AddMembersDelayed(fun () -> 13 | let messages = peekMessages connectionString queueName 32 14 | messages 15 | |> Seq.map(fun msg -> 16 | let messageType = ProvidedTypeDefinition(sprintf "%s.Queues.%s.Peek.%s" connectionString queueName (string msg.Id), None, hideObjectMethods = true) 17 | domainType.AddMember messageType 18 | messageType.AddMembersDelayed(fun () -> 19 | let contents = msg.AsString 20 | let contentsProp = 21 | let out = String(contents.ToCharArray() |> Seq.truncate 32 |> Seq.toArray) 22 | if contents.Length <= 32 then out 23 | else out + "..." 24 | 25 | let dequeueCount = msg.DequeueCount 26 | let inserted = msg.InsertionTime.ToString() 27 | let expires = msg.ExpirationTime.ToString() 28 | let id = msg.Id 29 | 30 | [ ProvidedProperty(sprintf "Id: %s" (string msg.Id), typeof, getterCode = (fun _ -> <@@ id @@>)) 31 | ProvidedProperty(sprintf "Contents: '%s'" contentsProp, typeof, getterCode = (fun _ -> <@@ contents @@>)) 32 | ProvidedProperty(sprintf "Dequeued %d times" msg.DequeueCount, typeof, getterCode = (fun _ -> <@@ dequeueCount @@>)) 33 | ProvidedProperty(sprintf "Inserted on %A" msg.InsertionTime, typeof, getterCode = (fun _ -> <@@ inserted @@>)) 34 | ProvidedProperty(sprintf "Expires at %A" msg.ExpirationTime, typeof, getterCode = (fun _ -> <@@ expires @@>)) 35 | ]) 36 | ProvidedProperty(sprintf "%s (%s)" (String(msg.AsString.ToCharArray() |> Seq.truncate 32 |> Seq.toArray)) (string msg.Id), messageType, getterCode = (fun _ -> <@@ () @@>))) 37 | |> Seq.toList) 38 | peekType 39 | 40 | let createQueueMemberType connectionString (domainType:ProvidedTypeDefinition) queueName = 41 | let queueType = ProvidedTypeDefinition(sprintf "%s.queue.%s" connectionString queueName, Some typeof, hideObjectMethods = true) 42 | domainType.AddMember queueType 43 | queueType.AddMemberDelayed(fun () -> 44 | let p = ProvidedProperty("Peek", createPeekForQueue(connectionString, domainType, queueName), getterCode = (fun _ -> <@@ () @@>)) 45 | p.AddXmlDoc <| sprintf "Allows you to peek at the top 32 items on the queue (as of %A)." DateTime.UtcNow 46 | p) 47 | queueName, queueType 48 | 49 | open Microsoft.WindowsAzure.Storage.RetryPolicies 50 | 51 | let private doesQueueEndpointExists connectionString = async { 52 | let client = getQueueClient connectionString 53 | client.DefaultRequestOptions.RetryPolicy <- NoRetry() 54 | client.DefaultRequestOptions.MaximumExecutionTime <- Nullable(TimeSpan(0, 0, 5)) 55 | match! client.GetQueueReference("test").ExistsAsync() |> Async.AwaitTask |> Async.toAsyncResult with 56 | | Ok _ -> return true 57 | | Error _ -> return false } 58 | 59 | /// Builds up the Queue Storage member 60 | let getQueueStorageMembers (connectionString, domainType : ProvidedTypeDefinition) = 61 | if not (doesQueueEndpointExists connectionString |> Async.RunSynchronously) then None 62 | else 63 | let queueListingType = ProvidedTypeDefinition("Queues", Some typeof, hideObjectMethods = true) 64 | let createQueueMember = createQueueMemberType connectionString domainType 65 | queueListingType.AddMembersDelayed(fun () -> 66 | connectionString 67 | |> getQueues 68 | |> Async.RunSynchronously 69 | |> Array.map (createQueueMember >> fun (name, queueType) -> 70 | ProvidedProperty(name, queueType, getterCode = fun _ -> <@@ ProvidedQueue(connectionString, name) @@> )) 71 | |> Array.toList) 72 | 73 | domainType.AddMember queueListingType 74 | queueListingType.AddMember(ProvidedProperty("CloudQueueClient", typeof, getterCode = (fun _ -> <@@ QueueBuilder.getQueueClient connectionString @@>))) 75 | let queueListingProp = ProvidedProperty("Queues", queueListingType, isStatic = true, getterCode = (fun _ -> <@@ () @@>)) 76 | queueListingProp.AddXmlDoc "Gets the list of all queues in this storage account." 77 | Some queueListingProp -------------------------------------------------------------------------------- /docs/content/queues.fsx: -------------------------------------------------------------------------------- 1 | (*** hide ***) 2 | #load @"..\tools\references.fsx" 3 | open FSharp.Azure.StorageTypeProvider 4 | open FSharp.Azure.StorageTypeProvider.Queue 5 | open System.Xml.Linq 6 | open System 7 | type Azure = AzureTypeProvider<"UseDevelopmentStorage=true"> 8 | (** 9 | Working with Queues 10 | =================== 11 | 12 | For more information on Queues in general, please see some of the many articles on 13 | [MSDN](https://msdn.microsoft.com/en-us/library/microsoft.windowsazure.storage.queue.aspx) or the [Azure](http://azure.microsoft.com/en-us/documentation/services/storage/) [documentation](http://azure.microsoft.com/en-us/documentation/articles/storage-dotnet-how-to-use-queues/). Some of the core features of the Queue provider are: - 14 | 15 | ## Rapid navigation 16 | 17 | You can easily move between queues and view key information on that queue. Simply dotting into 18 | a queue will automatically request the latest details on the queue. This allows easy exploration 19 | of your queue assets, directly from within the REPL. 20 | *) 21 | 22 | (*** define-output: blobStats ***) 23 | let queue = Azure.Queues.``sample-queue`` 24 | printfn "Queue '%s' has %d items on it." queue.Name (queue.GetCurrentLength()) 25 | (*** include-output: blobStats ***) 26 | 27 | (** 28 | ## Processing messages 29 | It is easy to push and pop messages onto / off a queue - simply call the Enqueue() and Dequeue() 30 | methods on the appropriate queue. Enqueue will return an option message, in case there is nothing 31 | on the queue. Once you have finished processing the message, simply call Delete(). *) 32 | 33 | (*** define-output: queue1 ***) 34 | async { 35 | printfn "Queue length is %d." (queue.GetCurrentLength()) 36 | 37 | // Put a message on the queue 38 | printfn "Enqueuing a message!" 39 | do! queue.Enqueue("Hello from Azure Type Provider") 40 | printfn "Queue length is %d." (queue.GetCurrentLength()) 41 | 42 | // Get the message back off the queue 43 | let dequeuedMessage = (queue.Dequeue() |> Async.RunSynchronously).Value // don't try this at home :) 44 | printfn "%A" dequeuedMessage 45 | 46 | // Delete it off the queue to tell Azure we're done with it. 47 | printfn "Deleting the message." 48 | do! queue.DeleteMessage dequeuedMessage.Id 49 | printfn "Queue length is %d." (queue.GetCurrentLength()) 50 | } |> Async.RunSynchronously 51 | (*** include-output: queue1 ***) 52 | 53 | (** 54 | ## Modifying Queues 55 | You can easily modify the contents of an existing message and push it back onto the queue, or clear 56 | the queue entirely. Note that the properties to access the payload (AsString and AsBytes) are lazily 57 | evaluated and as such are exposed as Lazy. 58 | *) 59 | (*** define-output: queue2 ***) 60 | let printMessage msg = 61 | printfn "Message %A with body '%s' has been dequeued %d times." msg.Id msg.AsString.Value msg.DequeueCount 62 | 63 | async { 64 | printfn "Enqueuing a message!" 65 | do! queue.Enqueue("Hello from Azure Type Provider") 66 | 67 | // Get the message, then put it back on the queue with a new payload immediately. 68 | printfn "Dequeuing it." 69 | let! message = queue.Dequeue() 70 | match message with 71 | | Some message -> 72 | printMessage message 73 | printfn "Updating it and dequeuing it again." 74 | do! queue.UpdateMessage(message.Id, "Goodbye from Azure Type Provider") 75 | 76 | // Now dequeue the message again and interrogate it 77 | let! message = queue.Dequeue() 78 | match message with 79 | | Some message -> 80 | printMessage message 81 | do! queue.DeleteMessage message.Id 82 | | None -> () 83 | | None -> () 84 | } |> Async.RunSynchronously 85 | (*** include-output: queue2 ***) 86 | 87 | (** 88 | ##Shared Access Signature generation 89 | 90 | The type provider exposes a simple method for generating time-dependant SAS codes for 91 | queues. Omit permissions parameter to get full-access SAS token. 92 | *) 93 | 94 | (*** define-output: sas ***) 95 | let duration = TimeSpan.FromMinutes 37. 96 | printfn "Current time: %O" DateTime.UtcNow 97 | printfn "SAS expiry: %O" (DateTime.UtcNow.Add duration) 98 | let sasCode = queue.GenerateSharedAccessSignature(duration, permissions = (QueuePermission.Peek ||| QueuePermission.Enqueue ||| QueuePermission.DequeueAndDeleteMessageAndClear)) 99 | printfn "SAS URI: %O" sasCode 100 | (*** include-output: sas ***) 101 | 102 | (** 103 | ## Peeking the queue 104 | The Queue Provider allows you to preview messages on the queue directly in intellisense. Simply 105 | dot into the "Peek" property on the queue, and the first 32 messages on the queue will appear. 106 | Properties on them can be bound with their values. This is particularly useful when using the 107 | [hot schema loading](hot-schema-loading.html#Working-with-Queues) feature of the type provider. 108 | *) 109 | 110 | -------------------------------------------------------------------------------- /src/FSharp.Azure.StorageTypeProvider/Table/TableMemberFactory.fs: -------------------------------------------------------------------------------- 1 | module internal FSharp.Azure.StorageTypeProvider.Table.TableMemberFactory 2 | 3 | open FSharp.Azure.StorageTypeProvider.Table 4 | open FSharp.Azure.StorageTypeProvider.Table.TableRepository 5 | open Microsoft.WindowsAzure.Storage 6 | open Microsoft.WindowsAzure.Storage.Table 7 | open ProviderImplementation.ProvidedTypes 8 | 9 | let (|StorageExn|_|) (ex:exn) = 10 | match ex with 11 | | :? StorageException as ex -> Some(StorageExn (ex.RequestInformation.HttpStatusMessage, ex.RequestInformation.HttpStatusCode, ex :> exn)) 12 | | _ -> None 13 | 14 | let (|BlobOnlyAccount|FullAccount|) = function 15 | | Error [ StorageExn("Not Implemented", 501, _) ] -> BlobOnlyAccount 16 | | Error (ex :: _) -> raise ex 17 | | Error [] -> failwith "Unknown account type" 18 | | Ok _ -> FullAccount 19 | 20 | /// Builds up the Table Storage member 21 | let getTableStorageMembers optionalStaticSchema schemaInferenceRowCount humanize (connectionString, domainType : ProvidedTypeDefinition) = 22 | async { 23 | let tableListingType = ProvidedTypeDefinition("Tables", Some typeof, hideObjectMethods = true) 24 | domainType.AddMember tableListingType 25 | 26 | /// Creates an individual Table member 27 | let createTableType columnDefinitions connectionString tableName propertyName = 28 | let tableEntityType = ProvidedTypeDefinition(tableName + "Entity", Some typeof, hideObjectMethods = true) 29 | let tableType = ProvidedTypeDefinition(tableName + "Table", Some typeof, hideObjectMethods = true) 30 | domainType.AddMembers [ tableEntityType; tableType ] 31 | 32 | TableEntityMemberFactory.buildTableEntityMembers columnDefinitions humanize (tableType, tableEntityType, domainType, connectionString, tableName) 33 | let tableProp = ProvidedProperty(propertyName, tableType, getterCode = (fun _ -> <@@ TableBuilder.createAzureTable connectionString tableName @@>)) 34 | tableProp.AddXmlDoc <| sprintf "Provides access to the '%s' table." tableName 35 | tableProp 36 | 37 | match optionalStaticSchema with 38 | | Some (optionalStaticSchema:StaticSchema.Parsed.TableSchema) -> 39 | optionalStaticSchema.Tables 40 | |> Array.map(fun table -> createTableType table.Columns connectionString table.Table table.Table) 41 | |> Array.toList 42 | |> tableListingType.AddMembers 43 | | None -> 44 | match! (getTableClient connectionString).GetTableReference("1").ExistsAsync() |> Async.AwaitTask |> Async.toAsyncResult with 45 | | BlobOnlyAccount -> 46 | () 47 | | FullAccount -> 48 | tableListingType.AddMembersDelayed(fun _ -> 49 | async { 50 | let! tables = getTables connectionString 51 | return! 52 | tables 53 | |> Array.map (fun table -> async { 54 | let! schema = TableEntityMemberFactory.generateSchema table schemaInferenceRowCount connectionString 55 | let tableProp = createTableType schema connectionString table table 56 | return tableProp }) 57 | |> Async.Parallel 58 | |> Async.map Array.toList } 59 | |> Async.RunSynchronously) 60 | 61 | // Get any metrics tables that are available 62 | let metrics = getMetricsTables connectionString 63 | if metrics <> Seq.empty then 64 | tableListingType.AddMembersDelayed(fun _ -> 65 | let metricsTablesType = ProvidedTypeDefinition("$Azure_Metrics", Some typeof, hideObjectMethods = true) 66 | domainType.AddMember metricsTablesType 67 | 68 | for (period, theLocation, service, tableName) in metrics do 69 | let schema = TableEntityMemberFactory.generateSchema tableName schemaInferenceRowCount connectionString |> Async.RunSynchronously 70 | createTableType schema connectionString tableName (sprintf "%s %s metrics (%s)" period service theLocation) 71 | |> metricsTablesType.AddMember 72 | 73 | let metricsTablesProp = ProvidedProperty("Azure Metrics", metricsTablesType, getterCode = (fun _ -> <@@ () @@>)) 74 | metricsTablesProp.AddXmlDoc "Provides access to metrics tables populated by Azure that are available on this storage account." 75 | [ metricsTablesProp ]) 76 | 77 | let ctcProp = ProvidedProperty("CloudTableClient", typeof, getterCode = (fun _ -> <@@ TableBuilder.createAzureTableRoot connectionString @@>)) 78 | ctcProp.AddXmlDoc "Gets a handle to the Table Azure SDK client for this storage account." 79 | tableListingType.AddMember ctcProp 80 | 81 | let tableListingProp = ProvidedProperty("Tables", tableListingType, isStatic = true, getterCode = (fun _ -> <@@ () @@>)) 82 | tableListingProp.AddXmlDoc "Gets the list of all tables in this storage account." 83 | return Some tableListingProp } 84 | |> Async.RunSynchronously 85 | -------------------------------------------------------------------------------- /src/FSharp.Azure.StorageTypeProvider/Blob/BlobRepository.fs: -------------------------------------------------------------------------------- 1 | ///Contains reusable helper functions for accessing blobs 2 | module FSharp.Azure.StorageTypeProvider.Blob.BlobRepository 3 | 4 | open FSharp.Azure.StorageTypeProvider 5 | open Microsoft.WindowsAzure.Storage 6 | open Microsoft.WindowsAzure.Storage.Blob 7 | open System 8 | open System.IO 9 | 10 | type ContainerItem = 11 | | Folder of path : string * name : string * contents : ContainerItem array Async 12 | | Blob of path : string * name : string * blobType : BlobType * length : int64 option 13 | 14 | type LightweightContainer = 15 | { Name : string 16 | Contents : ContainerItem array Async } 17 | 18 | let getBlobClient connection = CloudStorageAccount.Parse(connection).CreateCloudBlobClient() 19 | let getContainerRef(connection, container) = (getBlobClient connection).GetContainerReference(container) 20 | let getBlockBlobRef (connection, container, file) = getContainerRef(connection, container).GetBlockBlobReference(file) 21 | let getPageBlobRef (connection, container, file) = getContainerRef(connection, container).GetPageBlobReference(file) 22 | 23 | let private getItemName (item : string) (parent : CloudBlobDirectory) = 24 | item, 25 | match parent with 26 | | null -> item 27 | | parent -> item.Substring(parent.Prefix.Length) 28 | 29 | [] 30 | module private SdkExtensions = 31 | type CloudBlobClient with 32 | member blobClient.ListContainersAsync() = 33 | let listContainers token = async { 34 | let! results = blobClient.ListContainersSegmentedAsync token |> Async.AwaitTask 35 | return results.ContinuationToken, results.Results } 36 | Async.segmentedAzureOperation listContainers 37 | 38 | type CloudBlobContainer with 39 | member container.ListBlobsAsync incSubDirs prefix = 40 | let listBlobs token = async { 41 | let! results = container.ListBlobsSegmentedAsync(prefix = prefix, useFlatBlobListing = incSubDirs, blobListingDetails = BlobListingDetails.None, maxResults = Nullable(), currentToken = token, options = BlobRequestOptions(), operationContext = null) |> Async.AwaitTask 42 | return results.ContinuationToken, results.Results } 43 | Async.segmentedAzureOperation listBlobs 44 | 45 | let listBlobs incSubDirs (container:CloudBlobContainer) prefix = async { 46 | let! results = container.ListBlobsAsync incSubDirs prefix 47 | 48 | //can safely ignore folder types as we have a flat structure if & only if we want to include items from sub directories 49 | return 50 | [| for result in results do 51 | match result with 52 | | :? ICloudBlob as blob -> 53 | let path, name = getItemName blob.Name blob.Parent 54 | yield Blob(path, name, blob.Properties.BlobType, Some blob.Properties.Length) 55 | | _ -> () |] } 56 | 57 | let getBlobStorageAccountManifest (connectionString:string) = 58 | let rec getContainerStructureAsync prefix (container:CloudBlobContainer) = async { 59 | let! blobs = container.ListBlobsAsync false prefix 60 | let blobs = blobs |> Array.distinctBy (fun b -> b.Uri.AbsoluteUri) 61 | return 62 | [| for blob in blobs do 63 | match blob with 64 | | :? CloudBlobDirectory as directory -> 65 | let path, name = getItemName directory.Prefix directory.Parent 66 | yield Folder(path, name, container |> getContainerStructureAsync directory.Prefix) 67 | | :? ICloudBlob as blob -> 68 | let path, name = getItemName blob.Name blob.Parent 69 | yield Blob(path, name, blob.Properties.BlobType, Some blob.Properties.Length) 70 | | _ -> () |] } 71 | 72 | async { 73 | let client = (CloudStorageAccount.Parse connectionString).CreateCloudBlobClient() 74 | let! containers = client.ListContainersAsync() 75 | return! 76 | containers 77 | |> Array.map (fun container -> async { 78 | let structure = container |> getContainerStructureAsync null 79 | return { Name = container.Name; Contents = structure } }) 80 | |> Async.Parallel } 81 | 82 | let downloadFolder (connectionDetails, path) = async { 83 | let downloadFile (blobRef:ICloudBlob) destination = 84 | let targetDirectory = Path.GetDirectoryName(destination) 85 | if not (Directory.Exists targetDirectory) then Directory.CreateDirectory targetDirectory |> ignore 86 | blobRef.DownloadToFileAsync(destination, FileMode.Create) |> Async.AwaitTask 87 | 88 | let connection, container, folderPath = connectionDetails 89 | let containerRef = (getBlobClient connection).GetContainerReference(container) 90 | let! blobs = containerRef.ListBlobsAsync true folderPath 91 | 92 | return! 93 | blobs 94 | |> Array.choose (function 95 | | :? ICloudBlob as b -> Some b 96 | | _ -> None) 97 | |> Array.map (fun blob -> 98 | let targetName = 99 | match folderPath with 100 | | folderPath when String.IsNullOrEmpty folderPath -> blob.Name 101 | | _ -> blob.Name.Replace(folderPath, String.Empty) 102 | downloadFile blob (Path.Combine(path, targetName))) 103 | |> Async.Parallel 104 | |> Async.Ignore } -------------------------------------------------------------------------------- /src/FSharp.Azure.StorageTypeProvider/Blob/MimeTypes.fs: -------------------------------------------------------------------------------- 1 | module internal MimeTypes 2 | 3 | let tryFindMimeType = 4 | [ ".3gp", "video/3gpp" 5 | ".3gpp", "video/3gpp" 6 | ".7z", "application/x-7z-compressed" 7 | ".aac", "audio/mp4" 8 | ".ai", "application/postscript" 9 | ".appcache", "text/cache-manifest" 10 | ".asf", "video/x-ms-asf" 11 | ".asx", "video/x-ms-asf" 12 | ".atom", "application/atom+xml" 13 | ".avi", "video/x-msvideo" 14 | ".bbaw", "application/x-bb-appworld" 15 | ".bin", "application/octet-stream" 16 | ".bmp", "image/bmp" 17 | ".cco", "application/x-cocoa" 18 | ".crt", "application/x-x509-ca-cert" 19 | ".crx", "application/x-chrome-extension" 20 | ".css", "text/css" 21 | ".csv", "text/csv" 22 | ".cur", "image/x-icon" 23 | ".deb", "application/octet-stream" 24 | ".der", "application/x-x509-ca-cert" 25 | ".dll", "application/octet-stream" 26 | ".dmg", "application/octet-stream" 27 | ".doc", "application/msword" 28 | ".docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document" 29 | ".ear", "application/java-archive" 30 | ".eot", "application/vnd.ms-fontobject" 31 | ".eps", "application/postscript" 32 | ".exe", "application/octet-stream" 33 | ".f4a", "audio/mp4" 34 | ".f4b", "audio/mp4" 35 | ".f4p", "video/mp4" 36 | ".f4v", "video/mp4" 37 | ".flv", "video/x-flv" 38 | ".geojson", "application/vnd.geo+json" 39 | ".gif", "image/gif" 40 | ".hdp", "image/jxr" 41 | ".hqx", "application/mac-binhex40" 42 | ".htc", "text/x-component" 43 | ".htm", "text/html" 44 | ".html", "text/html" 45 | ".ico", "image/x-icon" 46 | ".img", "application/octet-stream" 47 | ".iso", "application/octet-stream" 48 | ".jad", "text/vnd.sun.j2me.app-descriptor" 49 | ".jar", "application/java-archive" 50 | ".jardiff", "application/x-java-archive-diff" 51 | ".jng", "image/x-jng" 52 | ".jnlp", "application/x-java-jnlp-file" 53 | ".jpeg", "image/jpeg" 54 | ".jpg", "image/jpeg" 55 | ".js", "application/javascript" 56 | ".json", "application/json" 57 | ".jsonld", "application/ld+json" 58 | ".jxr", "image/jxr" 59 | ".kar", "audio/midi" 60 | ".kml", "application/vnd.google-earth.kml+xml" 61 | ".kmz", "application/vnd.google-earth.kmz" 62 | ".m4a", "audio/mp4" 63 | ".m4v", "video/mp4" 64 | ".map", "application/json" 65 | ".md", "text/markdown" 66 | ".mid", "audio/midi" 67 | ".midi", "audio/midi" 68 | ".mml", "text/mathml" 69 | ".mng", "video/x-mng" 70 | ".mov", "video/quicktime" 71 | ".mp3", "audio/mpeg" 72 | ".mp4", "video/mp4" 73 | ".mpeg", "video/mpeg" 74 | ".mpg", "video/mpeg" 75 | ".msi", "application/octet-stream" 76 | ".msm", "application/octet-stream" 77 | ".msp", "application/octet-stream" 78 | ".oex", "application/x-opera-extension" 79 | ".oga", "audio/ogg" 80 | ".ogg", "audio/ogg" 81 | ".ogv", "video/ogg" 82 | ".opus", "audio/ogg" 83 | ".otf", "font/opentype" 84 | ".pdb", "application/x-pilot" 85 | ".pdf", "application/pdf" 86 | ".pem", "application/x-x509-ca-cert" 87 | ".pl", "application/x-perl" 88 | ".pm", "application/x-perl" 89 | ".png", "image/png" 90 | ".ppt", "application/vnd.ms-powerpoint" 91 | ".pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation" 92 | ".prc", "application/x-pilot" 93 | ".ps", "application/postscript" 94 | ".ra", "audio/x-realaudio" 95 | ".rar", "application/x-rar-compressed" 96 | ".rdf", "application/xml" 97 | ".rpm", "application/x-redhat-package-manager" 98 | ".rss", "application/rss+xml" 99 | ".rtf", "application/rtf" 100 | ".run", "application/x-makeself" 101 | ".safariextz", "application/octet-stream" 102 | ".sea", "application/x-sea" 103 | ".shtml", "text/html" 104 | ".sit", "application/x-stuffit" 105 | ".svg", "image/svg+xml" 106 | ".svgz", "image/svg+xml" 107 | ".swf", "application/x-shockwave-flash" 108 | ".tcl", "application/x-tcl" 109 | ".tif", "image/tiff" 110 | ".tiff", "image/tiff" 111 | ".tk", "application/x-tcl" 112 | ".topojson", "application/json" 113 | ".torrent", "application/x-bittorrent" 114 | ".ttc", "application/x-font-ttf" 115 | ".ttf", "application/x-font-ttf" 116 | ".txt", "text/plain" 117 | ".vcard", "text/vcard" 118 | ".vcf", "text/vcard" 119 | ".vtt", "text/vtt" 120 | ".war", "application/java-archive" 121 | ".wav", "audio/x-wav" 122 | ".wbmp", "image/vnd.wap.wbmp" 123 | ".wdp", "image/jxr" 124 | ".webapp", "application/x-web-app-manifest+json" 125 | ".webm", "video/webm" 126 | ".webmanifest", "application/manifest+json" 127 | ".webp", "image/webp" 128 | ".wml", "text/vnd.wap.wml" 129 | ".wmlc", "application/vnd.wap.wmlc" 130 | ".wmv", "video/x-ms-wmv" 131 | ".woff", "application/font-woff" 132 | ".woff2", "application/font-woff2" 133 | ".xhtml", "application/xhtml+xml" 134 | ".xloc", "text/vnd.rim.location.xloc" 135 | ".xls", "application/vnd.ms-excel" 136 | ".xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" 137 | ".xml", "application/xml" 138 | ".xpi", "application/x-xpinstall" 139 | ".xsl", "application/xslt+xml" 140 | ".zip", "application/zip" ] 141 | |> Map.ofList 142 | |> fun m -> m.TryFind -------------------------------------------------------------------------------- /src/FSharp.Azure.StorageTypeProvider/Table/TableQueryBuilder.fs: -------------------------------------------------------------------------------- 1 | module internal FSharp.Azure.StorageTypeProvider.Table.TableQueryBuilder 2 | 3 | open FSharp.Azure.StorageTypeProvider.Table.TableRepository 4 | open Microsoft.FSharp.Core.CompilerServices 5 | open Microsoft.FSharp.Quotations 6 | open Microsoft.WindowsAzure.Storage.Table 7 | open ProviderImplementation.ProvidedTypes 8 | open System 9 | open System.Reflection 10 | open System.Text.RegularExpressions 11 | 12 | let private queryComparisons = 13 | typeof.GetFields() 14 | |> Seq.map(fun field -> 15 | let name = field.Name 16 | (if name.EndsWith("Equal") then name + "To" else name), field.GetValue(null) :?> string) 17 | |> Seq.cache 18 | 19 | let private splitOnCaps text = 20 | Regex.Replace(text, "((?<=[a-z])[A-Z]|[A-Z](?=[a-z]))", " $1").Trim() 21 | 22 | let private buildGenericProp<'a> parentQueryType propertyName = 23 | [ for compName, compValue in queryComparisons -> 24 | let invokeCode = fun (args: Expr list) -> <@@ buildFilter(propertyName, compValue, (%%args.[1]: 'a)) :: ((%%args.[0]: obj) :?> string list) @@> 25 | let providedMethod = ProvidedMethod(compName |> splitOnCaps, [ ProvidedParameter(propertyName.ToLower(), typeof<'a>) ], parentQueryType, invokeCode = invokeCode) 26 | providedMethod.AddXmlDocDelayed <| fun _ -> sprintf "Compares the %s property against the supplied value using the '%s' operator" propertyName compValue 27 | providedMethod ] 28 | 29 | let private buildCustomProp parentQueryType propertyName methodName documentation exectedResult = 30 | let invoker = fun (args: Expr list) -> <@@ buildFilter(propertyName, QueryComparisons.Equal, exectedResult) :: ((%%args.[0]: obj) :?> string list) @@> 31 | let providedMethod = ProvidedMethod(methodName, [], parentQueryType, invokeCode = invoker) 32 | providedMethod.AddXmlDocDelayed <| fun _ -> documentation 33 | providedMethod 34 | 35 | /// Generates strongly-type query provided properties for an entity property e.g. Equal, GreaterThan etc. etc. 36 | let private buildPropertyOperatorsType tableName propertyName propertyType parentQueryType = 37 | let propertyOperatorsType = ProvidedTypeDefinition(sprintf "%s.%sQueryOperators" tableName propertyName, Some typeof, hideObjectMethods = true) 38 | propertyOperatorsType.AddMembersDelayed(fun () -> 39 | match propertyType with 40 | | EdmType.String -> buildGenericProp parentQueryType propertyName 41 | | EdmType.Boolean -> 42 | let buildDescription = sprintf "Tests whether %s is %s." propertyName 43 | [ buildCustomProp parentQueryType propertyName "True" (buildDescription "true") true 44 | buildCustomProp parentQueryType propertyName "False" (buildDescription "false") false ] 45 | | EdmType.DateTime -> buildGenericProp parentQueryType propertyName 46 | | EdmType.Double -> buildGenericProp parentQueryType propertyName 47 | | EdmType.Int32 -> buildGenericProp parentQueryType propertyName 48 | | EdmType.Int64 -> buildGenericProp parentQueryType propertyName 49 | | EdmType.Guid -> buildGenericProp parentQueryType propertyName 50 | | _ -> []) 51 | propertyOperatorsType 52 | 53 | /// Creates a query property (and child methods etc.) for a given entity 54 | let createTableQueryType (tableEntityType: ProvidedTypeDefinition) connection tableName (columnDefinitions: ColumnDefinition seq) = 55 | let tableQueryType = ProvidedTypeDefinition(tableName + "QueryBuilder", Some typeof, hideObjectMethods = true) 56 | let operatorTypes = [ "PartitionKey", buildPropertyOperatorsType tableName "PartitionKey" EdmType.String tableQueryType 57 | "RowKey", buildPropertyOperatorsType tableName "RowKey" EdmType.String tableQueryType 58 | "Timestamp", buildPropertyOperatorsType tableName "Timestamp" EdmType.DateTime tableQueryType ] @ 59 | [ for cd in columnDefinitions -> cd.Name, buildPropertyOperatorsType tableName cd.Name cd.ColumnType tableQueryType ] 60 | 61 | tableQueryType.AddMembersDelayed(fun () -> 62 | let executeQueryMethodAsync = 63 | ProvidedMethod 64 | ("ExecuteAsync", [ ProvidedParameter("maxResults", typeof, optionalValue = 0) 65 | ProvidedParameter("connectionString", typeof, optionalValue = connection) ], 66 | typeof>.GetGenericTypeDefinition().MakeGenericType(tableEntityType.MakeArrayType()), 67 | invokeCode = (fun args -> <@@ executeQueryAsync (%%args.[2] : string) tableName %%args.[1] (composeAllFilters((%%args.[0]: obj) :?> string list)) @@>)) 68 | executeQueryMethodAsync.AddXmlDocDelayed <| fun _ -> "Executes the current query asynchronously." 69 | 70 | let executeQueryMethod = 71 | ProvidedMethod 72 | ("Execute", [ ProvidedParameter("maxResults", typeof, optionalValue = 0) 73 | ProvidedParameter("connectionString", typeof, optionalValue = connection) ], 74 | tableEntityType.MakeArrayType(), 75 | invokeCode = (fun args -> <@@ executeQueryAsync (%%args.[2] : string) tableName %%args.[1] (composeAllFilters((%%args.[0]: obj) :?> string list)) |> Async.RunSynchronously @@>)) 76 | executeQueryMethod.AddXmlDocDelayed <| fun _ -> "Executes the current query." 77 | 78 | let customQueryProperties = 79 | [ for (name, operatorType) in operatorTypes -> 80 | let queryProperty = ProvidedProperty("Where" + name + "Is" |> splitOnCaps, operatorType, getterCode = (fun args -> <@@ (%%args.[0]: obj) :?> string list @@>)) 81 | queryProperty.AddXmlDocDelayed <| fun _ -> sprintf "Creates a query part for the %s property." name 82 | queryProperty :> MemberInfo ] 83 | 84 | (executeQueryMethodAsync :> MemberInfo) :: 85 | (executeQueryMethod :> MemberInfo) :: 86 | customQueryProperties) 87 | 88 | tableQueryType, operatorTypes |> List.unzip |> snd -------------------------------------------------------------------------------- /src/FSharp.Azure.StorageTypeProvider/Blob/BlobRepositoryAsync.fs: -------------------------------------------------------------------------------- 1 | ///Contains reusable helper functions for accessing blobs 2 | module internal FSharp.Azure.StorageTypeProvider.Blob.BlobRepositoryAsync 3 | 4 | open Microsoft.WindowsAzure.Storage 5 | open Microsoft.WindowsAzure.Storage.Blob 6 | open System 7 | open System.IO 8 | open FSharp.Control.Tasks.ContextInsensitive 9 | open System.Threading.Tasks 10 | 11 | type ContainerItem = 12 | | Folder of path : string * name : string * contents : ContainerItem array Lazy 13 | | Blob of path : string * name : string * blobType : BlobType * length : int64 option 14 | 15 | type LightweightContainer = 16 | { Name : string 17 | Contents : ContainerItem seq Lazy } 18 | 19 | let getBlobClient connection = CloudStorageAccount.Parse(connection).CreateCloudBlobClient() 20 | let getContainerRef(connection, container) = (getBlobClient connection).GetContainerReference(container) 21 | let getBlockBlobRef (connection, container, file) = getContainerRef(connection, container).GetBlockBlobReference(file) 22 | let getPageBlobRef (connection, container, file) = getContainerRef(connection, container).GetPageBlobReference(file) 23 | 24 | let private getItemName (item : string) (parent : CloudBlobDirectory) = 25 | item, 26 | match parent with 27 | | null -> item 28 | | parent -> item.Substring(parent.Prefix.Length) 29 | 30 | let rec private getContainerStructure wildcard (container : CloudBlobContainer) = task { 31 | let rec getResults token = task { 32 | let! blobsSeqmented = container.ListBlobsSegmentedAsync(prefix = wildcard, currentToken = token) 33 | let token = blobsSeqmented.ContinuationToken 34 | let result = blobsSeqmented.Results |> Seq.toList 35 | if isNull token then 36 | return result 37 | else 38 | let! others = getResults token 39 | return result @ others } 40 | let! results = getResults null 41 | let containerStructur = 42 | results 43 | |> Seq.distinctBy (fun b -> b.Uri.AbsoluteUri) 44 | |> Seq.choose (function 45 | | :? CloudBlobDirectory as directory -> 46 | let path, name = getItemName directory.Prefix directory.Parent 47 | Some(Folder(path, name, lazy(container |> getContainerStructure directory.Prefix))) 48 | | :? ICloudBlob as blob -> 49 | let path, name = getItemName blob.Name blob.Parent 50 | Some(Blob(path, name, blob.BlobType, Some blob.Properties.Length)) 51 | | _ -> None) 52 | |> Seq.toArray 53 | return containerStructur 54 | } 55 | let listBlobs incSubDirs (container:CloudBlobContainer) prefix = 56 | container.ListBlobsSegmentedAsync(prefix = prefix, useFlatBlobListing = incSubDirs, blobListingDetails = BlobListingDetails.All, maxResults = Nullable(), currentToken = null, options = BlobRequestOptions(), operationContext = null).Result 57 | |> fun s -> s.Results 58 | |> Seq.choose(function 59 | | :? ICloudBlob as blob -> 60 | let path, name = getItemName blob.Name blob.Parent 61 | Some(Blob(path, name, blob.Properties.BlobType, Some blob.Properties.Length)) 62 | | _ -> None) //can safely ignore folder types as we have a flat structure if & only if we want to include items from sub directories 63 | 64 | // let getBlobStorageAccountManifest connection = task { 65 | // let rec getResults token = task { 66 | // let! containerResultSeqment = (getBlobClient connection).ListContainersSegmentedAsync(currentToken = token) 67 | // let token = containerResultSeqment.ContinuationToken 68 | // let result = containerResultSeqment.Results |> Seq.toList 69 | // if isNull token then 70 | // return result 71 | // else 72 | // let! others = getResults token 73 | // return result @ others } 74 | // let! results = getResults null 75 | // let blobStorageAccountManifest = task { 76 | // let blobContainerList = results |> Seq.toList 77 | // let manifestList = 78 | // blobContainerList 79 | // |> List.map (fun blobContainer -> 80 | // let! content = getContainerStructure null blobContainer 81 | // { Name = blobContainer.Name 82 | // Contents = 83 | // lazy 84 | // content |> Seq.cache}) 85 | // return manifestList 86 | // } 87 | // let! results = blobStorageAccountManifest 88 | // } 89 | 90 | let getBlobStorageAccountManifest connection = 91 | (getBlobClient connection).ListContainersSegmentedAsync(null).Result 92 | |> fun s -> s.Results 93 | |> Seq.toList 94 | |> List.map (fun container -> 95 | { Name = container.Name 96 | Contents = 97 | lazy 98 | let! stuctur = 99 | container 100 | |> getContainerStructure null 101 | stuctur 102 | |> Seq.cache }) 103 | let downloadFolder (connectionDetails, path) = 104 | let downloadFile (blobRef:ICloudBlob) destination = 105 | let targetDirectory = Path.GetDirectoryName(destination) 106 | if not (Directory.Exists targetDirectory) then Directory.CreateDirectory targetDirectory |> ignore 107 | blobRef.DownloadToFileAsync(destination, FileMode.Create) |> Async.AwaitTask 108 | 109 | let connection, container, folderPath = connectionDetails 110 | let containerRef = (getBlobClient connection).GetContainerReference(container) 111 | containerRef.ListBlobsSegmentedAsync(prefix = folderPath, useFlatBlobListing = true, blobListingDetails = BlobListingDetails.All, maxResults = Nullable(), currentToken = null, options = BlobRequestOptions(), operationContext = null).Result 112 | |> fun s -> s.Results 113 | |> Seq.choose (function 114 | | :? ICloudBlob as b -> Some b 115 | | _ -> None) 116 | |> Seq.map (fun blob -> 117 | let targetName = 118 | match folderPath with 119 | | folderPath when String.IsNullOrEmpty folderPath -> blob.Name 120 | | _ -> blob.Name.Replace(folderPath, String.Empty) 121 | downloadFile blob (Path.Combine(path, targetName))) 122 | |> Async.Parallel 123 | |> Async.Ignore -------------------------------------------------------------------------------- /tests/IntegrationTests/QueueUnitTests.fs: -------------------------------------------------------------------------------- 1 | module QueueTests 2 | 3 | open FSharp.Azure.StorageTypeProvider 4 | open FSharp.Azure.StorageTypeProvider.Queue 5 | open Swensen.Unquote 6 | open Swensen.Unquote.Operators 7 | open Microsoft.WindowsAzure.Storage.Queue 8 | open System 9 | open System.Linq 10 | open System.Net 11 | open System.Text 12 | open Expecto 13 | open FSharp.Control.Tasks.ContextSensitive 14 | 15 | type Local = AzureTypeProvider<"DevStorageAccount"> 16 | 17 | [] 18 | let compilationTests = 19 | testList "Queue Compilation Tests" [ 20 | testCase "Correctly identifies queues" (fun _ -> 21 | Local.Queues.``sample-queue`` |> ignore 22 | Local.Queues.``second-sample`` |> ignore 23 | Local.Queues.``third-sample`` |> ignore) 24 | ] 25 | 26 | let queue = Local.Queues.``sample-queue`` 27 | 28 | let queueSafeAsync = beforeAfterAsync QueueHelpers.resetData QueueHelpers.resetData 29 | 30 | [] 31 | let readOnlyQueueTests = 32 | testList "Queue Tests Read Only" [ 33 | testCase "Cloud Queue Client gives same results as the Type Provider" (fun _ -> 34 | let queues = Local.Queues 35 | let queueNames = queues.CloudQueueClient.ListQueuesSegmentedAsync(null) |> Async.AwaitTask |> Async.RunSynchronously |> fun r -> r.Results |> Seq.map(fun q -> q.Name) |> Set.ofSeq 36 | Expect.containsAll queueNames [ queues.``sample-queue``.Name; queues.``second-sample``.Name; queues.``third-sample``.Name ] "") 37 | 38 | testCase "Cloud Queue is the same queue as the Type Provider" (fun _ -> (queue.AsCloudQueue().Name) |> shouldEqual queue.Name) 39 | testCaseAsync "Dequeue with nothing on the queue returns None" <| async { 40 | let! msg = queue.Dequeue() 41 | Expect.isNone msg "" } 42 | ] 43 | 44 | [] 45 | let detailedQueueTests = 46 | testSequenced <| testList "Queue Tests" [ 47 | testCaseAsync "Enqueues a message" <| queueSafeAsync (async { 48 | do! queue.Enqueue "Foo" 49 | queue.GetCurrentLength() |> shouldEqual 1 }) 50 | testCaseAsync "Dequeues a message" <| queueSafeAsync (async { 51 | do! queue.Enqueue "Foo" 52 | let! message = queue.Dequeue() 53 | message.Value.Contents.Value |> shouldEqual "Foo" }) 54 | 55 | testCaseAsync "Safely supports lazy evaluation of 'bad data'" <| queueSafeAsync (async { 56 | let uri = 57 | let sas = queue.GenerateSharedAccessSignature(TimeSpan.FromDays 7.) 58 | sprintf "http://127.0.0.1:10001/devstoreaccount1/%s/messages%s" queue.Name sas 59 | 60 | //Create a broken message to send to the queue. The queue system expects "Test" to be a Base64 encoded string. 61 | let request = Encoding.UTF8.GetBytes @"Test" 62 | 63 | // Send the request 64 | use wc = new WebClient() 65 | wc.UploadData(uri, request) |> ignore 66 | 67 | let! msg = queue.Dequeue() 68 | Expect.isSome msg "No message was returned" 69 | Expect.throws (fun _ -> msg.Value.Contents.Value |> ignore) "Value shouldn't have been string parseable" }) 70 | testCaseAsync "Deletes a message" <| queueSafeAsync (async { 71 | do! queue.Enqueue "Foo" 72 | let! message = queue.Dequeue() 73 | do! queue.DeleteMessage message.Value.Id 74 | 0 |> shouldEqual <| queue.GetCurrentLength() }) 75 | testCaseAsync "Update Message affects the text message body" <| queueSafeAsync (async { 76 | do! queue.Enqueue "Foo" 77 | do! Async.Sleep 100 78 | let! message = queue.Dequeue() 79 | do! queue.UpdateMessage(message.Value.Id, "Bar", TimeSpan.FromSeconds 0.) 80 | do! Async.Sleep 100 81 | let! message = queue.Dequeue() 82 | message.Value.Contents.Value |> shouldEqual "Bar" }) 83 | testCaseAsync "Dequeue Count is correctly emitted" <| queueSafeAsync (async { 84 | do! queue.Enqueue("Foo") 85 | do! Async.Sleep 100 86 | let! message = queue.Dequeue() 87 | do! queue.UpdateMessage(message.Value.Id, TimeSpan.FromSeconds 0.) 88 | do! Async.Sleep 100 89 | let! message = queue.Dequeue() 90 | do! queue.UpdateMessage(message.Value.Id, TimeSpan.FromSeconds 0.) 91 | do! Async.Sleep 100 92 | let! message = queue.Dequeue() 93 | 3 |> shouldEqual message.Value.DequeueCount }) 94 | testCaseAsync "Clear correctly empties the queue" <| queueSafeAsync (async { 95 | do! queue.Enqueue "Foo" 96 | do! queue.Enqueue "Bar" 97 | do! queue.Enqueue "Test" 98 | do! queue.Clear() 99 | queue.GetCurrentLength() |> shouldEqual 0 }) 100 | testCaseAsync "Queue message with visibility timeout reappears correctly" <| queueSafeAsync (async { 101 | do! queue.Enqueue "test" 102 | do! queue.Dequeue(TimeSpan.FromSeconds 1.) |> Async.Ignore 103 | do! Async.Sleep 2000 104 | let! message = queue.Dequeue() 105 | Expect.isSome message "Should be a message" 106 | message.Value.DequeueCount |> shouldEqual 2 }) 107 | testCaseAsync "Queue message with visibility timeout is correctly applied" <| queueSafeAsync (async { 108 | do! queue.Enqueue "test" 109 | let! message = queue.Dequeue(TimeSpan.FromSeconds 3.) 110 | do! Async.Sleep 2000 111 | let! message = queue.Dequeue() 112 | Expect.isNone message "Should be no message" }) 113 | ] 114 | 115 | [] 116 | let sasTokenTests = 117 | testList "SAS Token Tests" [ 118 | testCase "Generates token with default (full-access) queue permissions" (fun _ -> 119 | let sas = queue.GenerateSharedAccessSignature(TimeSpan.FromDays 7.) 120 | Expect.stringContains sas "sp=raup" "Invalid permissions" 121 | ) 122 | testCase "Generates token with specific queue permissions" (fun _ -> 123 | let sas = queue.GenerateSharedAccessSignature(TimeSpan.FromDays 7., permissions = (QueuePermission.Enqueue ||| QueuePermission.UpdateMessage)) 124 | Expect.stringContains sas "sp=au" "Invalid permissions" 125 | ) 126 | ] -------------------------------------------------------------------------------- /docs/tools/formatters.fsx: -------------------------------------------------------------------------------- 1 | module Formatters 2 | #I "../../packages/FSharp.Formatting/lib/net40" 3 | #r "FSharp.Markdown.dll" 4 | #r "FSharp.Literate.dll" 5 | #r "../../packages/Deedle/lib/net40/Deedle.dll" 6 | #r "../../packages/FAKE/tools/FakeLib.dll" 7 | #load "../../packages/FSharp.Charting/FSharp.Charting.fsx" 8 | 9 | open Fake 10 | open System.IO 11 | open Deedle 12 | open Deedle.Internal 13 | open FSharp.Literate 14 | open FSharp.Markdown 15 | open FSharp.Charting 16 | 17 | // -------------------------------------------------------------------------------------- 18 | // Implements Markdown formatters for common FsLab things - including Deedle series 19 | // and frames, F# Charting charts and System.Image values 20 | // -------------------------------------------------------------------------------------- 21 | 22 | // How many columns and rows from frame should be rendered 23 | let startColumnCount = 3 24 | let endColumnCount = 3 25 | 26 | let startRowCount = 8 27 | let endRowCount = 4 28 | 29 | // How many items from a series should be rendered 30 | let startItemCount = 5 31 | let endItemCount = 3 32 | 33 | // -------------------------------------------------------------------------------------- 34 | // Helper functions etc. 35 | // -------------------------------------------------------------------------------------- 36 | 37 | open System.Windows.Forms 38 | open FSharp.Charting.ChartTypes 39 | 40 | /// Extract values from any series using reflection 41 | let (|SeriesValues|_|) (value:obj) = 42 | let iser = value.GetType().GetInterface("ISeries`1") 43 | if iser <> null then 44 | let keys = value.GetType().GetProperty("Keys").GetValue(value) :?> System.Collections.IEnumerable 45 | let vector = value.GetType().GetProperty("Vector").GetValue(value) :?> IVector 46 | Some(Seq.zip (Seq.cast keys) vector.ObjectSequence) 47 | else None 48 | 49 | /// Format value as a single-literal paragraph 50 | let formatValue def = function 51 | | Some v -> [ Paragraph [Literal (v.ToString()) ]] 52 | | _ -> [ Paragraph [Literal def] ] 53 | 54 | /// Format body of a single table cell 55 | let td v = [ Paragraph [Literal v] ] 56 | 57 | /// Use 'f' to transform all values, then call 'g' with Some for 58 | /// values to show and None for "..." in the middle 59 | let mapSteps (startCount, endCount) f g input = 60 | input 61 | |> Seq.map f |> Seq.startAndEnd startCount endCount 62 | |> Seq.map (function Choice1Of3 v | Choice3Of3 v -> g (Some v) | _ -> g None) 63 | |> List.ofSeq 64 | 65 | // Tuples with the counts, for easy use later on 66 | let fcols = startColumnCount, endColumnCount 67 | let frows = startRowCount, endRowCount 68 | let sitms = startItemCount, endItemCount 69 | 70 | /// Reasonably nice default style for charts 71 | let chartStyle ch = 72 | let grid = ChartTypes.Grid(LineColor=System.Drawing.Color.LightGray) 73 | ch 74 | |> Chart.WithYAxis(MajorGrid=grid) 75 | |> Chart.WithXAxis(MajorGrid=grid) 76 | 77 | // -------------------------------------------------------------------------------------- 78 | // Build FSI evaluator 79 | // -------------------------------------------------------------------------------------- 80 | 81 | /// Builds FSI evaluator that can render System.Image, F# Charts, series & frames 82 | let createFsiEvaluator root output = 83 | 84 | /// Counter for saving files 85 | let imageCounter = 86 | let count = ref 0 87 | (fun () -> incr count; !count) 88 | 89 | let transformation (value:obj, typ:System.Type) = 90 | match value with 91 | | :? System.Drawing.Image as img -> 92 | // Pretty print image - save the image to the "images" directory 93 | // and return a DirectImage reference to the appropriate location 94 | let id = imageCounter().ToString() 95 | let file = "chart" + id + ".png" 96 | ensureDirectory (output @@ "images") 97 | img.Save(output @@ "images" @@ file, System.Drawing.Imaging.ImageFormat.Png) 98 | Some [ Paragraph [DirectImage ("Chart", (root + "/images/" + file, None))] ] 99 | 100 | | :? ChartTypes.GenericChart as ch -> 101 | // Pretty print F# Chart - save the chart to the "images" directory 102 | // and return a DirectImage reference to the appropriate location 103 | let id = imageCounter().ToString() 104 | let file = "chart" + id + ".png" 105 | ensureDirectory (output @@ "images") 106 | 107 | // We need to reate host control, but it does not have to be visible 108 | ( use ctl = new ChartControl(chartStyle ch, Dock = DockStyle.Fill, Width=500, Height=300) 109 | ch.CopyAsBitmap().Save(output @@ "images" @@ file, System.Drawing.Imaging.ImageFormat.Png) ) 110 | Some [ Paragraph [DirectImage ("Chart", (root + "/images/" + file, None))] ] 111 | 112 | | SeriesValues s -> 113 | // Pretty print series! 114 | let heads = s |> mapSteps sitms fst (function Some k -> td (k.ToString()) | _ -> td " ... ") 115 | let row = s |> mapSteps sitms snd (function Some v -> formatValue "N/A" (OptionalValue.asOption v) | _ -> td " ... ") 116 | let aligns = s |> mapSteps sitms id (fun _ -> AlignDefault) 117 | [ InlineBlock "
" 118 | TableBlock(Some ((td "Keys")::heads), AlignDefault::aligns, [ (td "Values")::row ]) 119 | InlineBlock "
" ] |> Some 120 | 121 | | :? IFrame as f -> 122 | // Pretty print frame! 123 | {new IFrameOperation<_> with 124 | member x.Invoke(f) = 125 | let heads = f.ColumnKeys |> mapSteps fcols id (function Some k -> td (k.ToString()) | _ -> td " ... ") 126 | let aligns = f.ColumnKeys |> mapSteps fcols id (fun _ -> AlignDefault) 127 | let rows = 128 | f.Rows |> Series.observationsAll |> mapSteps frows id (fun item -> 129 | let def, k, data = 130 | match item with 131 | | Some(k, Some d) -> "N/A", k.ToString(), Series.observationsAll d |> Seq.map snd 132 | | Some(k, _) -> "N/A", k.ToString(), f.ColumnKeys |> Seq.map (fun _ -> None) 133 | | None -> " ... ", " ... ", f.ColumnKeys |> Seq.map (fun _ -> None) 134 | let row = data |> mapSteps fcols id (function Some v -> formatValue def v | _ -> td " ... ") 135 | (td k)::row ) 136 | Some [ 137 | InlineBlock "
" 138 | TableBlock(Some ([]::heads), AlignDefault::aligns, rows) 139 | InlineBlock "
" 140 | ] } 141 | |> f.Apply 142 | | _ -> None 143 | 144 | // Create FSI evaluator, register transformations & return 145 | let fsiEvaluator = FsiEvaluator() 146 | fsiEvaluator.RegisterTransformation(transformation) 147 | fsiEvaluator -------------------------------------------------------------------------------- /docs/tools/generate.fsx: -------------------------------------------------------------------------------- 1 | // -------------------------------------------------------------------------------------- 2 | // Builds the documentation from `.fsx` and `.md` files in the 'docs/content' directory 3 | // (the generated documentation is stored in the 'docs/output' directory) 4 | // -------------------------------------------------------------------------------------- 5 | 6 | #load "Formatters.fsx" 7 | 8 | // Binaries that have XML documentation (in a corresponding generated XML file) 9 | // Any binary output / copied to bin/projectName/projectName.dll will 10 | // automatically be added as a binary to generate API docs for. 11 | // for binaries output to root bin folder please add the filename only to the 12 | // referenceBinaries list below in order to generate documentation for the binaries. 13 | // (This is the original behaviour of ProjectScaffold prior to multi project support) 14 | let referenceBinaries = [ @"..\..\bin\FSharp.Azure.StorageTypeProvider.dll" ] 15 | 16 | let githubLink = "http://github.com/fsprojects/AzureStorageTypeProvider" 17 | 18 | // Specify more information about your project 19 | let info = 20 | [ "project-name", "Azure Storage Type Provider" 21 | "project-author", "Isaac Abraham" 22 | "project-summary", "Stuff" 23 | "project-github", githubLink 24 | "project-nuget", "http://www.nuget.org/packages/FSharp.Azure.StorageTypeProvider" ] 25 | 26 | // -------------------------------------------------------------------------------------- 27 | // For typical project, no changes are needed below 28 | // -------------------------------------------------------------------------------------- 29 | 30 | #I "../../packages/FAKE/tools/" 31 | #load "../../packages/FSharp.Formatting/FSharp.Formatting.fsx" 32 | #r "NuGet.Core.dll" 33 | #r "FakeLib.dll" 34 | open Fake 35 | open System.IO 36 | open Fake.FileHelper 37 | open FSharp.Literate 38 | open FSharp.MetadataFormat 39 | 40 | // When called from 'build.fsx', use the public project URL as 41 | // otherwise, use the current 'output' directory. 42 | #if RELEASE 43 | let root = "." 44 | #else 45 | let root = "file://" + (__SOURCE_DIRECTORY__ @@ "../output") 46 | #endif 47 | 48 | // Paths with template/source/output locations 49 | let bin = __SOURCE_DIRECTORY__ @@ "../../bin" 50 | let content = __SOURCE_DIRECTORY__ @@ "../content" 51 | let output = __SOURCE_DIRECTORY__ @@ "../output" 52 | let files = __SOURCE_DIRECTORY__ @@ "../files" 53 | let templates = __SOURCE_DIRECTORY__ @@ "templates" 54 | let formatting = __SOURCE_DIRECTORY__ @@ "../../packages/FSharp.Formatting/" 55 | let docTemplate = "docpage.cshtml" 56 | 57 | // Where to look for *.csproj templates (in this order) 58 | let layoutRootsAll = new System.Collections.Generic.Dictionary() 59 | layoutRootsAll.Add("en",[ templates; formatting @@ "templates" 60 | formatting @@ "templates/reference" ]) 61 | subDirectories (directoryInfo templates) 62 | |> Seq.iter (fun d -> 63 | let name = d.Name 64 | if name.Length = 2 || name.Length = 3 then 65 | layoutRootsAll.Add( 66 | name, [templates @@ name 67 | formatting @@ "templates" 68 | formatting @@ "templates/reference" ])) 69 | 70 | // Copy static files and CSS + JS from F# Formatting 71 | let copyFiles () = 72 | CopyRecursive files output true |> Log "Copying file: " 73 | ensureDirectory (output @@ "content") 74 | CopyRecursive (formatting @@ "styles") (output @@ "content") true 75 | |> Log "Copying styles and scripts: " 76 | 77 | let references = 78 | if isMono then 79 | // Workaround compiler errors in Razor-ViewEngine 80 | let d = RazorEngine.Compilation.ReferenceResolver.UseCurrentAssembliesReferenceResolver() 81 | let loadedList = d.GetReferences () |> Seq.map (fun r -> r.GetFile()) |> Seq.cache 82 | // We replace the list and add required items manually as mcs doesn't like duplicates... 83 | let getItem name = loadedList |> Seq.find (fun l -> l.Contains name) 84 | [ (getItem "FSharp.Core").Replace("4.3.0.0", "4.3.1.0") 85 | Path.GetFullPath "./../../packages/FSharp.Compiler.Service/lib/net40/FSharp.Compiler.Service.dll" 86 | Path.GetFullPath "./../../packages/FSharp.Formatting/lib/net40/System.Web.Razor.dll" 87 | Path.GetFullPath "./../../packages/FSharp.Formatting/lib/net40/RazorEngine.dll" 88 | Path.GetFullPath "./../../packages/FSharp.Formatting/lib/net40/FSharp.Literate.dll" 89 | Path.GetFullPath "./../../packages/FSharp.Formatting/lib/net40/FSharp.CodeFormat.dll" 90 | Path.GetFullPath "./../../packages/FSharp.Formatting/lib/net40/FSharp.MetadataFormat.dll" ] 91 | |> Some 92 | else None 93 | 94 | let binaries = referenceBinaries 95 | let libDirs = [ bin ] 96 | 97 | // Build API reference from XML comments 98 | let buildReference () = 99 | CleanDir (output @@ "reference") 100 | MetadataFormat.Generate 101 | ( binaries, output @@ "reference", layoutRootsAll.["en"], 102 | parameters = ("root", root)::info, 103 | sourceRepo = githubLink @@ "tree/master", 104 | sourceFolder = __SOURCE_DIRECTORY__ @@ ".." @@ "..", 105 | ?assemblyReferences = references, 106 | publicOnly = true, libDirs = libDirs) 107 | 108 | // Build documentation from `fsx` and `md` files in `docs/content` 109 | let buildDocumentation () = 110 | 111 | // First, process files which are placed in the content root directory. 112 | let fsiEvaluator = Formatters.createFsiEvaluator root output 113 | Literate.ProcessDirectory 114 | ( content, docTemplate, output, replacements = ("root", root)::info, 115 | layoutRoots = layoutRootsAll.["en"], 116 | ?assemblyReferences = references, 117 | generateAnchors = true, 118 | processRecursive = false, 119 | fsiEvaluator = fsiEvaluator) 120 | 121 | // And then process files which are placed in the sub directories 122 | // (some sub directories might be for specific language). 123 | 124 | let subdirs = Directory.EnumerateDirectories(content, "*", SearchOption.TopDirectoryOnly) 125 | for dir in subdirs do 126 | let dirname = (new DirectoryInfo(dir)).Name 127 | let layoutRoots = 128 | // Check whether this directory name is for specific language 129 | let key = layoutRootsAll.Keys 130 | |> Seq.tryFind (fun i -> i = dirname) 131 | match key with 132 | | Some lang -> layoutRootsAll.[lang] 133 | | None -> layoutRootsAll.["en"] // "en" is the default language 134 | 135 | Literate.ProcessDirectory 136 | ( dir, docTemplate, output @@ dirname, replacements = ("root", root) :: info, 137 | layoutRoots = layoutRoots, 138 | ?assemblyReferences = references, 139 | generateAnchors = true ) 140 | 141 | // Generate 142 | copyFiles() 143 | #if HELP 144 | buildDocumentation() 145 | #endif 146 | #if REFERENCE 147 | buildReference() 148 | #endif 149 | -------------------------------------------------------------------------------- /src/FSharp.Azure.StorageTypeProvider/AzureTypeProvider.fs: -------------------------------------------------------------------------------- 1 | namespace ProviderImplementation 2 | 3 | open FSharp.Azure.StorageTypeProvider 4 | open FSharp.Azure.StorageTypeProvider.Blob 5 | open FSharp.Azure.StorageTypeProvider.Queue 6 | open FSharp.Azure.StorageTypeProvider.Table 7 | open FSharp.Azure.StorageTypeProvider.Configuration 8 | open Microsoft.FSharp.Core.CompilerServices 9 | open ProviderImplementation.ProvidedTypes 10 | open System 11 | open System.Collections.Generic 12 | open System.Reflection 13 | 14 | [] 15 | /// [omit] 16 | type public AzureTypeProvider(config : TypeProviderConfig) as this = 17 | inherit TypeProviderForNamespaces(config) 18 | 19 | let namespaceName = "FSharp.Azure.StorageTypeProvider" 20 | let thisAssembly = Assembly.GetExecutingAssembly() 21 | let azureAccountType = ProvidedTypeDefinition(thisAssembly, namespaceName, "AzureTypeProvider", baseType = Some typeof) 22 | 23 | let buildConnectionString (args : obj []) = 24 | let (|ConnectionString|TwoPart|DevelopmentStorage|) (args:obj []) = 25 | let getArg i = args.[i] :?> string 26 | let accountNameOrConnectionString, accountKey = getArg 0, getArg 1 27 | let configFileKey, configFileName = getArg 2, getArg 3 28 | 29 | match accountNameOrConnectionString, accountKey, configFileKey with 30 | | _ when (not << String.IsNullOrWhiteSpace) configFileKey -> 31 | let connectionFromConfig = getConnectionString(configFileKey, config.ResolutionFolder, configFileName) 32 | ConnectionString connectionFromConfig 33 | | _ when accountNameOrConnectionString.StartsWith "DefaultEndpointsProtocol" -> ConnectionString accountNameOrConnectionString 34 | | _ when [ accountNameOrConnectionString; accountKey ] |> List.exists String.IsNullOrWhiteSpace -> DevelopmentStorage 35 | | _ -> TwoPart (accountNameOrConnectionString, accountKey) 36 | 37 | match args with 38 | | DevelopmentStorage -> "UseDevelopmentStorage=true" 39 | | ConnectionString conn -> conn 40 | | TwoPart (name, key) -> sprintf "DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s;" name key 41 | 42 | let startLiveRefresh : obj -> _ = 43 | function 44 | | :? int as seconds when seconds >= 1 -> 45 | let seconds = seconds * 1000 46 | async { 47 | while true do 48 | do! Async.Sleep seconds 49 | this.Invalidate() 50 | } |> Async.Start 51 | | _ -> () 52 | 53 | let buildTypes (typeName : string) (args : obj []) = 54 | // Create the top level property 55 | let typeProviderForAccount = ProvidedTypeDefinition(thisAssembly, namespaceName, typeName, baseType = Some typeof) 56 | typeProviderForAccount.AddMember(ProvidedConstructor([], fun _ -> <@@ null @@>)) 57 | 58 | startLiveRefresh args.[8] 59 | 60 | let connectionString = buildConnectionString args 61 | let staticBlobSchema = args.[6] :?> string |> Option.ofString 62 | let staticTableSchema = args.[7] :?> string |> Option.ofString 63 | let connectionStringValidation = 64 | match staticBlobSchema, staticTableSchema with 65 | | Some _, _ | _, Some _ -> None 66 | | _ -> Some (validateConnectionString connectionString) 67 | 68 | let parsedBlobSchema = Blob.StaticSchema.createSchema config.ResolutionFolder staticBlobSchema 69 | let parsedTableSchema = Table.StaticSchema.createSchema config.ResolutionFolder staticTableSchema 70 | 71 | match connectionStringValidation, parsedBlobSchema, parsedTableSchema with 72 | | Some (Ok ()), Ok blobSchema, Ok tableSchema 73 | | None, Ok blobSchema, Ok tableSchema -> 74 | let domainTypes = ProvidedTypeDefinition("Domain", Some typeof) 75 | typeProviderForAccount.AddMember(domainTypes) 76 | 77 | let schemaInferenceRowCount = args.[4] :?> int 78 | let humanizeColumns = args.[5] :?> bool 79 | 80 | // Now create child members e.g. containers, tables etc. 81 | typeProviderForAccount.AddMembers 82 | ([ (BlobMemberFactory.getBlobStorageMembers blobSchema, "blobs") 83 | (TableMemberFactory.getTableStorageMembers tableSchema schemaInferenceRowCount humanizeColumns, "tables") 84 | (QueueMemberFactory.getQueueStorageMembers, "queues") ] 85 | |> List.choose (fun (builder, name) -> 86 | try builder(connectionString, domainTypes) 87 | with ex -> failwithf "An error occurred during initial type generation for %s: %O" name ex)) 88 | typeProviderForAccount 89 | | Some (Error ex), _, _ -> failwithf "Unable to validate connection string (%O)" ex 90 | | _, Error ex, _ -> failwithf "Unable to parse blob schema file (%O)" ex 91 | | _, _, Error ex -> failwithf "Unable to parse table schema file (%O)" ex 92 | 93 | let createParam (name, defaultValue:'a, help) = 94 | let providedParameter = ProvidedStaticParameter(name, typeof<'a>, defaultValue) 95 | providedParameter.AddXmlDoc help 96 | providedParameter 97 | 98 | // Parameterising the provider 99 | let parameters = 100 | [ createParam("accountName", String.Empty, "The Storage Account name, or full connection string in the format 'DefaultEndpointsProtocol=protocol;AccountName=account;AccountKey=key;'.") 101 | createParam("accountKey", String.Empty, "The Storage Account key. Ignored if the accountName argument is the full connection string.") 102 | createParam("connectionStringName", String.Empty, "The Connection String key from the configuration file to use to retrieve the connection string. If set, accountName and accountKey are ignored.") 103 | createParam("configFileName", "app.config", "The name of the configuration file to look for. Defaults to 'app.config'") 104 | createParam("schemaSize", 10, "The maximum number of rows to read per table, from which to infer schema. Defaults to 10.") 105 | createParam("humanize", false, "Whether to humanize table column names. Defaults to false.") 106 | createParam("blobSchema", String.Empty, "Provide a path to a local file containing a fixed schema to eagerly use, instead of lazily generating the blob schema from a live storage account.") 107 | createParam("tableSchema", String.Empty, "Provide a path to a local file containing a fixed schema to eagerly use, instead of lazily generating the table schema from a live storage account.") 108 | createParam("autoRefresh", 0, "Optionally provide the number of seconds to wait before refreshing the schema. Defaults to 0 (never).") ] 109 | 110 | let memoize func = 111 | let cache = Dictionary() 112 | fun argsAsString args -> 113 | if not (cache.ContainsKey argsAsString) then 114 | cache.Add(argsAsString, func argsAsString args) 115 | cache.[argsAsString] 116 | 117 | do 118 | azureAccountType.DefineStaticParameters(parameters, memoize buildTypes) 119 | this.AddNamespace(namespaceName, [ azureAccountType ]) 120 | azureAccountType.AddXmlDoc("The entry type to connect to Azure Storage assets.") 121 | 122 | [] 123 | do () -------------------------------------------------------------------------------- /src/FSharp.Azure.StorageTypeProvider/Table/ProvidedTableTypes.fs: -------------------------------------------------------------------------------- 1 | namespace FSharp.Azure.StorageTypeProvider.Table 2 | 3 | open FSharp.Azure.StorageTypeProvider 4 | open FSharp.Azure.StorageTypeProvider.Table.TableRepository 5 | open Microsoft.WindowsAzure.Storage 6 | open Microsoft.WindowsAzure.Storage.Table 7 | open System 8 | open System.Reflection 9 | 10 | /// Represents a Table in Azure. 11 | type AzureTable internal (defaultConnection, tableName) = 12 | let getConnectionDetails (insertMode, connectionString) = 13 | defaultArg insertMode TableInsertMode.Insert, defaultArg connectionString defaultConnection 14 | let getTableForConnection = getTable tableName 15 | 16 | let safeGetOption name = function | Some value -> Some(name, box value) | _ -> None 17 | 18 | let buildInsertParams insertMode connectionString (entities : seq) = 19 | let insertMode, connectionString = getConnectionDetails (insertMode, connectionString) 20 | let table = getTableForConnection connectionString 21 | let insertOp = createInsertOperation insertMode 22 | 23 | let propBuilders = 24 | typeof<'b>.GetProperties(BindingFlags.Public ||| BindingFlags.Instance) 25 | |> Seq.map (fun prop entity -> prop.Name, prop.GetValue(entity, null)) 26 | |> Seq.toArray 27 | 28 | let tblEntities = 29 | entities 30 | |> Seq.map (fun (partitionKey, rowKey, entity) -> 31 | let values = 32 | propBuilders 33 | |> Seq.map (fun builder -> builder entity) 34 | |> Seq.choose(fun (name, value) -> 35 | match value with 36 | | :? Option<(byte [])> as value -> safeGetOption name value 37 | | :? Option as value -> safeGetOption name value 38 | | :? Option as value -> safeGetOption name value 39 | | :? Option as value -> safeGetOption name value 40 | | :? Option as value -> safeGetOption name value 41 | | :? Option as value -> safeGetOption name value 42 | | :? Option as value -> safeGetOption name value 43 | | :? Option as value -> safeGetOption name value 44 | | _ -> Some (name, value)) 45 | |> Map.ofSeq 46 | LightweightTableEntity(partitionKey, rowKey, DateTimeOffset.MinValue, values) |> buildDynamicTableEntity) 47 | tblEntities, insertOp, table 48 | 49 | let getSinglePartitionResult partitionKey = function 50 | | [| partition |] -> partition 51 | | _ -> partitionKey, [||] 52 | 53 | /// Gets a handle to the Azure SDK client for this table. 54 | member __.AsCloudTable(?connectionString) = getTableForConnection (defaultArg connectionString defaultConnection) 55 | 56 | /// Inserts a batch of entities into the table, using all public properties on the object as fields. 57 | member this.Insert(entities : seq, ?insertMode, ?connectionString) = 58 | match insertMode, connectionString with 59 | | Some insertMode, Some connectionString -> this.InsertAsync(entities, insertMode, connectionString) 60 | | Some insertMode, None -> this.InsertAsync(entities, insertMode) 61 | | None, Some connectionString -> this.InsertAsync(entities, connectionString = connectionString) 62 | | None, None -> this.InsertAsync(entities) 63 | |> Async.RunSynchronously 64 | 65 | /// Inserts a batch of entities into the table, using all public properties on the object as fields. 66 | member __.InsertAsync(entities : seq, ?insertMode, ?connectionString) = async { 67 | let tblEntities, insertOp, table = buildInsertParams insertMode connectionString entities 68 | return! tblEntities |> executeBatchOperationAsync insertOp table } 69 | 70 | /// Inserts a single entity into the table, using public properties on the object as fields. 71 | member this.Insert(partitionKey, rowKey, entity, ?insertMode, ?connectionString) = 72 | match insertMode, connectionString with 73 | | Some insertMode, Some connectionString -> this.InsertAsync(partitionKey, rowKey, entity, insertMode, connectionString) 74 | | Some insertMode, None -> this.InsertAsync(partitionKey, rowKey, entity, insertMode) 75 | | None, Some connectionString -> this.InsertAsync(partitionKey, rowKey, entity, connectionString = connectionString) 76 | | None, None -> this.InsertAsync(partitionKey, rowKey, entity) 77 | |> Async.RunSynchronously 78 | 79 | /// Inserts a single entity into the table asynchronously, using public properties on the object as fields. 80 | member this.InsertAsync(partitionKey, rowKey, entity, ?insertMode, ?connectionString) = async { 81 | let insertMode, connectionString = getConnectionDetails (insertMode, connectionString) 82 | let! insertRes = this.InsertAsync([ partitionKey, rowKey, entity ], insertMode, connectionString) 83 | return 84 | insertRes 85 | |> Seq.head 86 | |> snd 87 | |> Seq.head } 88 | 89 | /// Deletes a batch of entities from the table using the supplied pairs of Partition and Row keys. 90 | member this.Delete(entities, ?connectionString) = 91 | match connectionString with 92 | | Some connectionString -> this.DeleteAsync(entities, connectionString) 93 | | None -> this.DeleteAsync entities 94 | |> Async.RunSynchronously 95 | 96 | /// Asynchronously deletes a batch of entities from the table using the supplied pairs of Partition and Row keys. 97 | member __.DeleteAsync(entities, ?connectionString) = async { 98 | let table = getTableForConnection (defaultArg connectionString defaultConnection) 99 | return! entities 100 | |> Seq.map (fun entityId -> 101 | let Partition(partitionKey), Row(rowKey) = entityId 102 | DynamicTableEntity(partitionKey, rowKey, ETag = "*")) 103 | |> executeBatchOperationAsync TableOperation.Delete table } 104 | 105 | /// Deletes an entire partition from the table 106 | member this.DeletePartition(partitionKey, ?connectionString) = 107 | match connectionString with 108 | | Some connectionString -> this.DeletePartitionAsync(partitionKey, connectionString) 109 | | None -> this.DeletePartitionAsync(partitionKey) 110 | |> Async.RunSynchronously 111 | 112 | /// Asynchronously deletes an entire partition from the table 113 | member __.DeletePartitionAsync(partitionKey, ?connectionString) = async { 114 | let table = getTableForConnection (defaultArg connectionString defaultConnection) 115 | let connectionString = defaultArg connectionString defaultConnection 116 | let filter = Table.TableQuery.GenerateFilterCondition ("PartitionKey", Table.QueryComparisons.Equal, partitionKey) 117 | let! queryResponse = executeGenericQueryAsync connectionString table.Name Int32.MaxValue filter (fun e -> (Partition e.PartitionKey, Row e.RowKey)) 118 | let! deleteResponse = queryResponse |> __.DeleteAsync 119 | return deleteResponse |> getSinglePartitionResult partitionKey } 120 | 121 | /// Gets the name of the table. 122 | member __.Name = tableName 123 | 124 | /// [omit] 125 | module TableBuilder = 126 | /// Creates an Azure Table object. 127 | let createAzureTable connectionString tableName = AzureTable(connectionString, tableName) 128 | let createAzureTableRoot connectionString = TableRepository.getTableClient connectionString -------------------------------------------------------------------------------- /src/FSharp.Azure.StorageTypeProvider/Queue/ProvidedQueueTypes.fs: -------------------------------------------------------------------------------- 1 | namespace FSharp.Azure.StorageTypeProvider.Queue 2 | 3 | open FSharp.Azure.StorageTypeProvider.Queue.QueueRepository 4 | open Microsoft.WindowsAzure.Storage.Queue 5 | open System 6 | 7 | /// The unique identifier for this Azure queue message. 8 | type MessageId = MessageId of string 9 | /// The unique identifier for this request of this Azure queue message. 10 | type PopReceipt = PopReceipt of string 11 | /// The composite identifier of this Azure queue message. 12 | type ProvidedMessageId = ProvidedMessageId of MessageId : MessageId * PopReceipt : PopReceipt 13 | 14 | /// Represents a single message that has been dequeued. 15 | type ProvidedQueueMessage = 16 | { /// The composite key of this message, containing both the message id and the pop receipt. 17 | Id : ProvidedMessageId 18 | /// The number of times this message has been dequeued. 19 | DequeueCount : int 20 | /// The time that this message was inserted. 21 | InsertionTime : DateTimeOffset option 22 | /// The time that this message will expire. 23 | ExpirationTime : DateTimeOffset option 24 | /// The time that this message will next become visible. 25 | NextVisibleTime : DateTimeOffset option 26 | /// Gets the contest of the message as a string. 27 | Contents : Lazy } 28 | 29 | /// Represents the set of possible permissions for a shared access queue policy. 30 | [] 31 | type QueuePermission = 32 | | Peek = 1 33 | | Enqueue = 2 34 | | UpdateMessage = 4 35 | | DequeueAndDeleteMessageAndClear = 8 36 | 37 | module internal Factory = 38 | let unpackId messageId = 39 | let (ProvidedMessageId(MessageId messageId, PopReceipt popReceipt)) = messageId 40 | messageId, popReceipt 41 | 42 | let toProvidedQueueMessage (message : CloudQueueMessage) = 43 | { Id = ProvidedMessageId(MessageId message.Id, PopReceipt message.PopReceipt) 44 | DequeueCount = message.DequeueCount 45 | InsertionTime = message.InsertionTime |> Option.ofNullable 46 | ExpirationTime = message.ExpirationTime |> Option.ofNullable 47 | NextVisibleTime = message.NextVisibleTime |> Option.ofNullable 48 | Contents = lazy message.AsString } 49 | 50 | let toAzureQueueMessage providedMessageId = 51 | let messageId, popReceipt = providedMessageId |> unpackId 52 | CloudQueueMessage(messageId, popReceipt) 53 | 54 | /// Represents an Azure Storage Queue. 55 | type ProvidedQueue(defaultConnectionString, name) = 56 | let getConnectionString connection = defaultArg connection defaultConnectionString 57 | let getQueue = getConnectionString >> getQueueRef name 58 | let enqueue message = getQueue >> (fun q -> q.AddMessageAsync message |> Async.AwaitTask) 59 | let updateMessage fields connectionString newTimeout message = 60 | let newTimeout = defaultArg newTimeout TimeSpan.Zero 61 | connectionString 62 | |> getQueue 63 | |> (fun queue -> queue.UpdateMessageAsync(message, newTimeout, fields) |> Async.AwaitTask) 64 | 65 | /// Gets a handle to the Azure SDK client for this queue. 66 | member __.AsCloudQueue(?connectionString) = getQueue connectionString 67 | 68 | /// Gets the queue length. 69 | member __.GetCurrentLength(?connectionString) = 70 | let queueRef = getQueue connectionString 71 | queueRef.FetchAttributesAsync() |> Async.AwaitTask |> Async.RunSynchronously 72 | queueRef.ApproximateMessageCount 73 | |> Option.ofNullable 74 | |> defaultArg <| 0 75 | 76 | /// Dequeues the next message and optionally sets the visibility timeout (i.e. how long you can work with the message before it reappears in the queue) 77 | member __.Dequeue(?connectionString, ?visibilityTimeout) = async { 78 | let! message = (getQueue connectionString).GetMessageAsync(visibilityTimeout |> Option.toNullable, null, null) |> Async.AwaitTask 79 | return 80 | match message with 81 | | null -> None 82 | | _ -> Some(message |> Factory.toProvidedQueueMessage) } 83 | 84 | /// Dequeues the next message using the default connection string and sets the visibility timeout (i.e. how long you can work with the message before it reappears in the queue) 85 | member __.Dequeue(visibilityTimeout) = __.Dequeue(defaultConnectionString, visibilityTimeout) 86 | 87 | ///Generates a shared access signature, defaulting to start from now. Do not pass 'permissions' for full-access. 88 | member __.GenerateSharedAccessSignature(duration, ?start, ?connectionString, ?permissions) = 89 | let permissions = defaultArg permissions (QueuePermission.Peek ||| QueuePermission.Enqueue ||| QueuePermission.UpdateMessage ||| QueuePermission.DequeueAndDeleteMessageAndClear) 90 | let typeMap = 91 | [ QueuePermission.Peek, SharedAccessQueuePermissions.Read; 92 | QueuePermission.Enqueue, SharedAccessQueuePermissions.Add; 93 | QueuePermission.UpdateMessage, SharedAccessQueuePermissions.Update; 94 | QueuePermission.DequeueAndDeleteMessageAndClear, SharedAccessQueuePermissions.ProcessMessages; 95 | ] |> Map.ofList 96 | 97 | let sharedAccessQueuePermissions = 98 | typeMap 99 | |> Map.fold (fun s k v -> if permissions.HasFlag k then s ||| v else s) SharedAccessQueuePermissions.None 100 | 101 | getQueue connectionString |> generateSas start duration sharedAccessQueuePermissions 102 | 103 | /// Enqueues a new message. 104 | member __.Enqueue(content, ?connectionString) = 105 | connectionString |> enqueue (CloudQueueMessage content) 106 | 107 | /// Deletes an existing message. 108 | member __.DeleteMessage(providedMessageId, ?connectionString) = 109 | let messageId, popReceipt = providedMessageId |> Factory.unpackId 110 | (connectionString |> getQueue).DeleteMessageAsync(messageId, popReceipt) |> Async.AwaitTask 111 | 112 | /// Updates the visibility of an existing message. 113 | member __.UpdateMessage(messageId, newTimeout, ?connectionString) = 114 | let message = messageId |> Factory.toAzureQueueMessage 115 | message |> updateMessage MessageUpdateFields.Visibility connectionString (Some newTimeout) 116 | 117 | /// Updates the visibility and the string contents of an existing message. If no timeout is provided, the update is immediately visible. 118 | member __.UpdateMessage(messageId, contents:string, ?newTimeout, ?connectionString) = 119 | let message = messageId |> Factory.toAzureQueueMessage 120 | message.SetMessageContent contents 121 | message |> updateMessage (MessageUpdateFields.Visibility ||| MessageUpdateFields.Content) connectionString newTimeout 122 | 123 | /// Updates the visibility and the binary contents of an existing message. If no timeout is provided, the update is immediately visible. 124 | member __.UpdateMessage(messageId, contents:byte array, ?newTimeout, ?connectionString) = 125 | let message = messageId |> Factory.toAzureQueueMessage 126 | message.SetMessageContent contents 127 | message |> updateMessage (MessageUpdateFields.Visibility ||| MessageUpdateFields.Content) connectionString newTimeout 128 | 129 | /// Clears the queue of all messages. 130 | member __.Clear(?connectionString) = 131 | connectionString 132 | |> getQueue 133 | |> (fun q -> q.ClearAsync()) 134 | |> Async.AwaitTask 135 | 136 | /// Gets the name of the queue. 137 | member __.Name = (None |> getQueue).Name 138 | 139 | /// [omit] 140 | /// Allows creation of queue entities. 141 | module QueueBuilder = 142 | /// Gets a queue client. 143 | let getQueueClient connectionString = QueueRepository.getQueueClient connectionString -------------------------------------------------------------------------------- /docs/content/blobs.fsx: -------------------------------------------------------------------------------- 1 | (*** hide ***) 2 | #load @"..\tools\references.fsx" 3 | open FSharp.Azure.StorageTypeProvider 4 | open FSharp.Azure.StorageTypeProvider.Blob 5 | open System.Xml.Linq 6 | open System 7 | type Azure = AzureTypeProvider<"UseDevelopmentStorage=true"> 8 | 9 | (** 10 | Working with Blobs 11 | ================== 12 | 13 | For more information on Blobs in general, please see some of the many articles on 14 | [MSDN](https://msdn.microsoft.com/en-us/library/microsoft.windowsazure.storage.blob.aspx) or the [Azure](http://azure.microsoft.com/en-us/documentation/services/storage/) [documentation](http://azure.microsoft.com/en-us/documentation/articles/storage-dotnet-how-to-use-blobs/). Some of the core features of the Blob provider are: - 15 | 16 | ## Rapid navigation 17 | 18 | You can easily move between containers, folders and blobs. Simply dotting into a container 19 | or folder will automatically request the children of that node from Azure. This allows 20 | easy exploration of your blob assets, directly from within the REPL. Support exists for both 21 | page and block blobs. 22 | *) 23 | 24 | (*** define-output: blobStats ***) 25 | let container = Azure.Containers.samples 26 | let theBlob = container.``folder/``.``childFile.txt`` 27 | printfn "Blob '%s' is %d bytes big." theBlob.Name (theBlob.Size()) 28 | (*** include-output: blobStats ***) 29 | 30 | (** 31 | You can also perform useful helper actions on folders, such as pulling back all blobs in a folder. 32 | *) 33 | 34 | (*** define-output: folders ***) 35 | let folder = container.``folder2/`` 36 | let blobs = folder.ListBlobs(true) 37 | printfn "Folder '%s' has the following blobs: %A" folder.Path blobs 38 | (*** include-output: folders ***) 39 | 40 | (** 41 | Also note that blobs support [hot schema loading](hot-schema-loading.html#Working-with-Blobs) to allow schema updates to occur as your storage account contents change. 42 | 43 | ## Shared types 44 | Individual files, folders and containers share a common base type so list operations are possible e.g. 45 | *) 46 | 47 | (*** define-output: sumOfSizes ***) 48 | let totalSize = 49 | [ container.``file1.txt`` 50 | container.``file2.txt`` 51 | container.``file3.txt`` 52 | container.``sample.txt`` ] 53 | |> List.sumBy(fun blob -> blob.Size()) 54 | 55 | printfn "These files take up %d bytes." totalSize 56 | (*** include-output: sumOfSizes ***) 57 | 58 | (** 59 | ## Flexible API for read operations 60 | You can quickly read the contents of a blob synchronously or asynchronously. 61 | *) 62 | 63 | (*** define-output: blobRead ***) 64 | // sync read 65 | let contents = theBlob.Read() 66 | printfn "sync contents = '%s'" contents 67 | 68 | // async read 69 | async { 70 | let! contentsAsync = theBlob.ReadAsync() 71 | printfn "async contents = '%s'" contentsAsync 72 | } |> Async.RunSynchronously 73 | (*** include-output: blobRead ***) 74 | 75 | (** 76 | In addition, the provider has support for custom methods for different document types e.g. XML. 77 | *) 78 | 79 | (*** define-output: xmlBlobs ***) 80 | let (contentsAsText:string) = container.``data.xml``.Read() 81 | // only available on XML documents 82 | let (contentsAsXml:XDocument) = container.``data.xml``.ReadAsXDocument() 83 | 84 | printfn "text output = '%O'" contentsAsText 85 | printfn "xml output = '%O'" contentsAsXml 86 | (*** include-output: xmlBlobs ***) 87 | 88 | (** 89 | ## Streaming support 90 | The provider exposes the ability to easily open a stream to a document for sequential reading. 91 | This is extremely useful for previewing large files etc. 92 | *) 93 | (*** define-output: streaming ***) 94 | let streamReader = container.``sample.txt``.OpenStreamAsText() 95 | while (not streamReader.EndOfStream) do 96 | printfn "LINE: '%s'" (streamReader.ReadLine()) 97 | (*** include-output: streaming ***) 98 | 99 | (** 100 | 101 | Again, since files share a common type, you can easily merge multiple sequential streams into one: - 102 | 103 | *) 104 | 105 | (*** define-output: streaming-fancy ***) 106 | let lines = 107 | [ container.``file1.txt`` 108 | container.``file2.txt`` 109 | container.``file3.txt`` 110 | container.``sample.txt`` ] 111 | |> Seq.collect(fun file -> file.ReadLines()) // could also use yield! syntax within a seq { } 112 | 113 | printfn "starting to read all lines" 114 | for line in lines do 115 | printfn "%s" line 116 | printfn "finished reading all lines" 117 | (*** include-output: streaming-fancy ***) 118 | 119 | (** 120 | 121 | ## Offline development 122 | In addition to using the Azure Storage Emulator, you can also simply provide the type provider 123 | with a JSON file containing the list of blob containers, folders and files. This is particularly 124 | useful within the context of a CI process, or when you know a specific "known good" structure of 125 | blobs within a storage account. 126 | 127 | You can still access blobs using the compile-time storage connection string if provided, or 128 | override as normal at runtime. 129 | 130 | *) 131 | 132 | type BlobSchema = AzureTypeProvider 133 | let fileFromSchema = BlobSchema.Containers.samples.``file3.txt`` 134 | 135 | (** 136 | The contents of `BlobSchema.json` looks as follows: - 137 | 138 | *) 139 | 140 | 141 | (*** hide ***) 142 | let blobSchemaValue = IO.File.ReadAllText "BlobSchema.json" 143 | 144 | (*** include-value: blobSchemaValue ***) 145 | 146 | (** 147 | 148 | Note that folder names must end with a forward slash e.g. `myfolder/`. Also observe that you can 149 | specify the `Type` of blob as either `pageblob` or `blockblob`. If not specified, this defaults 150 | to `blockblob`. You can leave "empty" values as either `null` or `{ }`. 151 | 152 | ## Programmatic access 153 | There are times when working with blobs (particularly when working with an offline schema) that you 154 | need to access blobs using "stringly typed" access. There are three ways you can do this within the 155 | type provider. 156 | 157 | ### Safe support 158 | For read access to blobs, you can use the Try... methods that are available on containers and 159 | folders. These asynchronously check if the blob exists, before returning an optional handle to it. 160 | 161 | *) 162 | 163 | (*** define-output: programmatic-access ***) 164 | let fileAsBlockBlob = container.TryGetBlockBlob "file1.txt" |> Async.RunSynchronously 165 | printfn "Does file1.txt exist as a block blob? %b" (Option.isSome fileAsBlockBlob) 166 | let fileAsPageBlob = container.TryGetPageBlob "file1.txt" |> Async.RunSynchronously 167 | printfn "Does file1.txt exist as a page blob? %b" (Option.isSome fileAsPageBlob) 168 | let doesNotExist = container.TryGetBlockBlob "doesNotExist" |> Async.RunSynchronously 169 | printfn "Does doesNotExist exist as a block blob? %b" (Option.isSome doesNotExist) 170 | 171 | (*** include-output: programmatic-access ***) 172 | 173 | (** 174 | 175 | ### Unsafe support for block blob access 176 | You can also "unsafely" access a block blob using indexers. This returns a blob reference which may or 177 | may not exist but can be used quickly and easily - especially useful if you want to create a blob that 178 | does not yet exist. However, be aware that any attempts to access a blob that does not exist will throw 179 | an Azure SDK exception. 180 | 181 | *) 182 | 183 | (*** define-output: unsafe-blob ***) 184 | let newBlob = container.["doesNotExist"] 185 | newBlob.AsCloudBlockBlob().UploadText "hello" 186 | printfn "Contents of blob: %s" (newBlob.Read()) 187 | newBlob.AsCloudBlockBlob().Delete() 188 | 189 | (*** include-output: unsafe-blob ***) 190 | 191 | (** 192 | 193 | ### Fallback to basic Azure SDK 194 | Lastly, you can always fall back to the raw .NET Azure SDK (which the type provider sits on top of). 195 | 196 | *) 197 | 198 | // Access the 'samples' container using the raw SDK. 199 | let rawContainer = Azure.Containers.samples.AsCloudBlobContainer() 200 | 201 | // All blobs can be referred to as an ICloudBlob 202 | let iCloudBlob = Azure.Containers.samples.``file1.txt``.AsICloudBlob() 203 | 204 | // Only available to CloudBlockBlobs. 205 | let blockBlob = Azure.Containers.samples.``file1.txt``.AsCloudBlockBlob() 206 | 207 | // Only available to PageBlockBlobs. 208 | let pageBlob = Azure.Containers.samples.``pageData.bin``.AsCloudPageBlob() 209 | 210 | (** 211 | 212 | ## Download assets 213 | You can quickly and easily download files, folders or entire containers to local disk. 214 | *) 215 | 216 | // download file1.txt asynchronously into "C:\temp\files" 217 | let asyncFileDownload = container.``file1.txt``.Download(@"C:\temp\files\") 218 | 219 | (** 220 | ##Shared Access Signature generation 221 | 222 | The type provider exposes a simple method for generating time-dependant SAS codes for 223 | single files. 224 | *) 225 | 226 | (*** define-output: sas ***) 227 | let duration = TimeSpan.FromMinutes 37. 228 | printfn "Current time: %O" DateTime.UtcNow 229 | printfn "SAS expiry: %O" (DateTime.UtcNow.Add duration) 230 | let sasCode = container.``file1.txt``.GenerateSharedAccessSignature duration 231 | printfn "SAS URI: %O" sasCode 232 | (*** include-output: sas ***) 233 | -------------------------------------------------------------------------------- /tests/IntegrationTests/BlobUnitTests.fs: -------------------------------------------------------------------------------- 1 | module BlobTests 2 | 3 | open Expecto 4 | open FSharp.Azure.StorageTypeProvider 5 | open FSharp.Azure.StorageTypeProvider.Blob 6 | open Microsoft.WindowsAzure.Storage.Blob 7 | open System 8 | open System.Linq 9 | open System.IO 10 | open FSharp.Control.Tasks.ContextInsensitive 11 | 12 | type Local = AzureTypeProvider<"UseDevelopmentStorage=true", ""> 13 | type BlobSchema = AzureTypeProvider 14 | 15 | let container = Local.Containers.samples 16 | 17 | [] 18 | let blobCompilationTests = 19 | testList "Blob Compilation Tests" [ 20 | testCase "Correctly identifies blob containers" (fun _ -> Local.Containers.samples |> ignore) 21 | 22 | testCase "Correctly identifies blobs in a container" (fun _ -> 23 | [ container.``file1.txt`` 24 | container.``file2.txt`` 25 | container.``file3.txt`` ] |> ignore) 26 | 27 | testCase "Correctly identifies blobs in a subfolder" (fun _ -> container .``folder/``.``childFile.txt`` |> ignore) 28 | testCase "Page Blobs are listed" (fun _ -> container.``pageData.bin`` |> ignore) ] 29 | 30 | let testFileDownload (blobFile:BlobFile) = 31 | let filename = Path.GetTempFileName() 32 | File.Delete filename 33 | blobFile.Download filename |> Async.RunSynchronously 34 | let predicates = 35 | [ File.Exists 36 | FileInfo >> fun fi -> fi.Length = blobFile.Size() ] 37 | |> List.map(fun pred -> pred filename) 38 | File.Delete filename 39 | predicates |> List.iter(fun item -> Expect.isTrue item "") 40 | 41 | let testFolderDownload download expectedFiles expectedFolders = 42 | let tempFolder = Path.Combine(Path.GetTempPath(), sprintf "tpTestFolder_%O" (System.Guid.NewGuid())) 43 | if Directory.Exists tempFolder then Directory.Delete(tempFolder, true) 44 | download tempFolder |> Async.RunSynchronously 45 | let files = Directory.GetFiles(tempFolder, "*", SearchOption.AllDirectories) |> Seq.length 46 | let folders = Directory.GetDirectories(tempFolder, "*", SearchOption.AllDirectories) |> Seq.length 47 | Directory.Delete(tempFolder, true) 48 | files |> shouldEqual expectedFiles 49 | folders |> shouldEqual expectedFolders 50 | 51 | [] 52 | let blobMainTests = 53 | testList "Blob Main Tests" [ 54 | testCase "Correctly gets size of a blob" (fun _ -> container .``sample.txt``.Size() |> shouldEqual 190L) 55 | testCase "Correctly gets metadata for a blob" (fun _ -> 56 | let metadata = container .``sample.txt``.GetProperties() |> Async.RunSynchronously 57 | metadata.Size |> shouldEqual 190L) 58 | testCase "Reads a text file as text" (fun _ -> 59 | let text = container .``sample.txt``.Read() 60 | text |> shouldEqual "the quick brown fox jumps over the lazy dog\nLorem ipsum dolor sit amet, consectetur adipiscing elit. Cras malesuada.\nLorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla porttitor." ) 61 | 62 | testCase "Streams a text file line-by-line" (fun _ -> 63 | let text = container .``sample.txt``.ReadLines() |> Seq.toArray 64 | 65 | text.[0] |> shouldEqual "the quick brown fox jumps over the lazy dog" 66 | text.[1] |> shouldEqual "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras malesuada." 67 | text.[2] |> shouldEqual "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla porttitor." 68 | text.Length |> shouldEqual 3) 69 | 70 | testCase "Opens a file with xml extension as an XML document" (fun _ -> 71 | let document = container.``data.xml``.ReadAsXDocument() 72 | let value = document.Elements().First() 73 | .Elements().First() 74 | .Value 75 | value |> shouldEqual "thing") 76 | 77 | testCase "Cloud Blob Client relates to the same data as the type provider" (fun _ -> 78 | Expect.contains (Local.Containers.CloudBlobClient.ListContainersSegmentedAsync(null) 79 | |> Async.AwaitTask 80 | |> Async.RunSynchronously 81 | |> fun r -> r.Results 82 | |> Seq.map(fun c -> c.Name)) "samples" "") 83 | 84 | testCase "Cloud Blob Container relates to the same data as the type provider" (fun _ -> 85 | let client = container.AsCloudBlobContainer() 86 | let blobs = client.ListBlobsSegmentedAsync(null) |> Async.AwaitTask |> Async.RunSynchronously |> fun x -> x.Results |> Seq.choose(function | :? CloudBlockBlob as b -> Some b | _ -> None) |> Seq.map(fun c -> c.Name) |> Seq.toList 87 | blobs |> shouldEqual [ "data.xml"; "file1.txt"; "file2.txt"; "file3.txt"; "sample.txt" ]) 88 | 89 | testCase "CloudBlockBlob relates to the same data as the type provider" (fun _ -> 90 | let blob = container.``data.xml``.AsCloudBlockBlob() 91 | blob.Name |> shouldEqual "data.xml") 92 | 93 | testCase "Page Blobs support streams" (fun _ -> Expect.stringStarts (container.``pageData.bin``.OpenStreamAsText().ReadToEnd()) "hello from page blob" "") 94 | 95 | testCase "CloudPageBlob relates to the same data as the type provider" (fun _ -> 96 | let blob = container.``pageData.bin``.AsCloudPageBlob() 97 | blob.Name |> shouldEqual "pageData.bin") 98 | 99 | testCase "Correctly transforms metadata for a blob container" (fun _ -> 100 | let underlyingContainer = container.AsCloudBlobContainer() 101 | underlyingContainer.FetchAttributesAsync() |> Async.AwaitTask |> Async.RunSynchronously 102 | 103 | let metadata = container.GetProperties() |> Async.RunSynchronously 104 | metadata.LastModified |> shouldEqual (underlyingContainer.Properties.LastModified |> Option.ofNullable) 105 | ) 106 | 107 | testCase "Page Blobs calculate size correctly" (fun _ -> container.``pageData.bin``.Size() |> shouldEqual 512L) 108 | testCase "Can correctly download a block blob" (fun _ -> testFileDownload container.``file1.txt``) 109 | testCase "Can correctly download a page blob" (fun _ -> testFileDownload container.``pageData.bin``) 110 | testCase "Can correctly download a folder" (fun _ -> testFolderDownload container.``folder/``.Download 2 0) 111 | testCase "Can correctly download a container" (fun _ -> testFolderDownload container.Download 12 5) 112 | 113 | testCase "Can access Path property on a folder" (fun _ -> 114 | let childFolder = Local.Containers.samples.``folder2/``.``child/`` 115 | childFolder.Path |> shouldEqual "folder2/child/") 116 | 117 | testCase "ListBlobs method returns correct number of blobs" (fun _ -> 118 | let childFolder = Local.Containers.samples.``folder2/``.``child/`` 119 | let allBlobs = childFolder.ListBlobs() |> Async.RunSynchronously 120 | Seq.length allBlobs |> shouldEqual 1) 121 | 122 | testCase "Can access List blobs method on a folder" (fun _ -> 123 | let childFolder = Local.Containers.samples.``folder2/``.``child/`` 124 | let allBlobs = childFolder.ListBlobs true |> Async.RunSynchronously 125 | let count = allBlobs |> Seq.length 126 | count |> shouldEqual 4) 127 | 128 | testCase "Container name is correct" (fun _ -> Local.Containers.samples.Name |> shouldEqual "samples") 129 | 130 | testCase "Sets Content Type on upload" (fun _ -> 131 | let testContent extension mimeType = 132 | let filename = sprintf "test.%s" extension 133 | File.WriteAllText(filename, "foo") 134 | Local.Containers.samples.Upload filename |> Async.RunSynchronously 135 | File.Delete filename 136 | 137 | let blob = Local.Containers.samples.[filename].AsCloudBlockBlob() 138 | blob.FetchAttributesAsync() |> Async.AwaitTask |> Async.RunSynchronously 139 | blob.DeleteAsync() |> Async.AwaitTask |> Async.RunSynchronously 140 | 141 | blob.Properties.ContentType |> shouldEqual mimeType 142 | testContent "txt" "text/plain" 143 | testContent "swf" "application/x-shockwave-flash" 144 | testContent "jpg" "image/jpeg") 145 | 146 | testCase "Retrieves blobs with prefix" (fun _ -> 147 | let blobs = Local.Containers.samples.``folder2/``.ListBlobs(prefix = "child/grandchild2/") |> Async.RunSynchronously |> Seq.map(fun b -> b.Name) |> Seq.toArray 148 | blobs |> shouldEqual [| "folder2/child/grandchild2/descedant3.txt" |]) 149 | 150 | testCase "Retrieves blobs with prefix and subfolders" (fun _ -> 151 | let blobs = Local.Containers.samples.``folder2/``.ListBlobs(includeSubfolders = true, prefix = "child/") |> Async.RunSynchronously|> Seq.map(fun b -> b.Name) |> Seq.sort |> Seq.toArray 152 | blobs.Length |> shouldEqual 4) 153 | ] 154 | 155 | [] 156 | let blobContainerTests = 157 | testList "Blob Container Tests" [ 158 | testCase "Can list all blobs in a container" (fun _ -> 159 | let blobs = Local.Containers.samples.ListBlobs true |> Async.RunSynchronously |> Seq.length 160 | blobs |> shouldEqual 12) 161 | testCase "Container supports unsafe blob access" (fun _ -> 162 | let b = Local.Containers.samples.["file1.txt"] 163 | b.Size() |> shouldEqual 5L) 164 | testCase "Container supports safe blob access" (fun _ -> 165 | let b = Local.Containers.samples.TryGetBlockBlob "file1.txt" |> Async.RunSynchronously 166 | Expect.isSome b "Should have returned a blob") 167 | ] 168 | 169 | [] 170 | let blobStaticSchemaTests = 171 | testList "Blob Static Schema Tests" [ 172 | testCase "Correct container name from a static schema" (fun _ -> 173 | let container = BlobSchema.Containers.samples 174 | container.Name |> shouldEqual "samples") 175 | 176 | testCase "Correct folder path from a static schema" (fun _ -> 177 | let folder = BlobSchema.Containers.samples.``folder2/``.``child/`` 178 | folder.Path |> shouldEqual "folder2/child/") 179 | 180 | testCase "Correct blob name from a static schema" (fun _ -> 181 | let blob = BlobSchema.Containers.samples.``folder/``.``childFile.txt`` 182 | blob.Name |> shouldEqual "folder/childFile.txt") 183 | 184 | testCase "Can access a real file using static schema" (fun _ -> 185 | let blob = BlobSchema.Containers.samples.``file1.txt`` 186 | blob.Size() |> shouldEqual 5L) 187 | 188 | testCase "Compiles with a non-existant file" (fun _ -> 189 | BlobSchema.Containers.random.``file.txt`` 190 | |> ignore) // compiles! 191 | 192 | testCase "Compiles with folder-only paths" (fun _ -> 193 | BlobSchema.Containers.random.``folder/``.``emptyFolder/`` 194 | |> ignore) //compiles! 195 | 196 | testCase "Compiles with empty container" (fun _ -> 197 | BlobSchema.Containers.emptyContainer 198 | |> ignore) //compiles! 199 | 200 | testCase "Default to block blob if not specified" (fun _ -> 201 | BlobSchema.Containers.samples.``file2.txt``.AsICloudBlob().BlobType 202 | |> shouldEqual BlobType.BlockBlob) 203 | 204 | testCase "Sets as block blob if specified" (fun _ -> 205 | BlobSchema.Containers.samples.``file1.txt``.AsICloudBlob().BlobType 206 | |> shouldEqual BlobType.BlockBlob) 207 | 208 | testCase "Sets as page blob if specified" (fun _ -> 209 | BlobSchema.Containers.samples.``file3.txt``.AsICloudBlob().BlobType 210 | |> shouldEqual BlobType.PageBlob) 211 | ] 212 | 213 | [] 214 | let blobProgrammaticTests = 215 | testList "Blob Folder Tests" [ 216 | testCase "Can return an unsafe handle to a blob" <| fun _ -> 217 | let blob = Local.Containers.samples.``folder/``.["childFile.txt"] 218 | blob.Name |> shouldEqual "folder/childFile.txt" 219 | blob.Size() |> shouldEqual 16L 220 | testCase "Safe handle to an existing block blob returns Some" <| fun _ -> 221 | let blob = Local.Containers.samples.``folder/``.TryGetBlockBlob "childFile.txt" |> Async.RunSynchronously 222 | Expect.isSome blob "" 223 | testCase "Safe handle to a non-existant block blob returns None" <| fun _ -> 224 | let blob = Local.Containers.samples.``folder/``.TryGetBlockBlob "childFilexxx.txt" |> Async.RunSynchronously 225 | Expect.isNone blob "" 226 | testCase "Safe handle to a non-existant page blob returns None" <| fun _ -> 227 | let blob = Local.Containers.samples.``folder/``.TryGetBlockBlob "childFilexxx.txt" |> Async.RunSynchronously 228 | Expect.isNone blob "" 229 | testCase "Safe handle to the wrong blob type returns None" <| fun _ -> 230 | let blob = Local.Containers.samples.``folder/``.TryGetPageBlob "childFile.txt" |> Async.RunSynchronously 231 | Expect.isNone blob "" 232 | ] 233 | 234 | [] 235 | let sasTokenTests = 236 | testList "SAS Token Tests" [ 237 | testCase "Generates token with default (full-access) blob permissions" (fun _ -> 238 | let sas = container.``file1.txt``.GenerateSharedAccessSignature (TimeSpan.FromDays 7.) 239 | Expect.stringContains (sas.ToString()) "sp=rwdl" "Invalid permissions" 240 | ) 241 | 242 | testCase "Generates token with specific blob permissions" (fun _ -> 243 | let sas = container.``file1.txt``.GenerateSharedAccessSignature(TimeSpan.FromDays 7., permissions = (SharedAccessBlobPermissions.Read ||| SharedAccessBlobPermissions.List)) 244 | Expect.stringContains (sas.ToString()) "sp=rl" "Invalid permissions" 245 | ) 246 | ] -------------------------------------------------------------------------------- /src/FSharp.Azure.StorageTypeProvider/Blob/ProvidedBlobTypes.fs: -------------------------------------------------------------------------------- 1 | namespace FSharp.Azure.StorageTypeProvider.Blob 2 | 3 | open FSharp.Azure.StorageTypeProvider.Blob.BlobRepository 4 | open Microsoft.WindowsAzure.Storage 5 | open Microsoft.WindowsAzure.Storage.Blob 6 | open System 7 | open System.IO 8 | open System.Xml.Linq 9 | 10 | type BlobMetadata internal (properties:Blob.BlobProperties) = 11 | let lastModified = properties.LastModified |> Option.ofNullable 12 | let appendBlobCommittedBlockCount = properties.AppendBlobCommittedBlockCount |> Option.ofNullable 13 | let pageBlobSequenceNumber = properties.PageBlobSequenceNumber |> Option.ofNullable 14 | member __.AppendBlobCommittedBlockCount = appendBlobCommittedBlockCount 15 | member __.BlobType = properties.BlobType 16 | member __.CacheControl = properties.CacheControl 17 | member __.ContentDisposition = properties.ContentDisposition 18 | member __.ContentEncoding = properties.ContentEncoding 19 | member __.ContentLanguage = properties.ContentLanguage 20 | member __.ContentMD5 = properties.ContentMD5 21 | member __.ContentType = properties.ContentType 22 | member __.ETag = properties.ETag 23 | member __.IsServerEncrypted = properties.IsServerEncrypted 24 | member __.LastModified = lastModified 25 | member __.LeaseDuration = properties.LeaseDuration 26 | member __.LeaseState = properties.LeaseState 27 | member __.LeaseStatus = properties.LeaseStatus 28 | member __.Size = properties.Length 29 | member __.PageBlobSequenceNumber = pageBlobSequenceNumber 30 | 31 | type BlobContainerMetadata internal (properties:Blob.BlobContainerProperties) = 32 | let lastModified = properties.LastModified |> Option.ofNullable 33 | member __.ETag = properties.ETag 34 | member __.LastModified = lastModified 35 | member __.LeaseStatus = properties.LeaseStatus 36 | member __.LeaseState = properties.LeaseState 37 | member __.LeaseDuration = properties.LeaseDuration 38 | 39 | /// Represents a file in blob storage. 40 | [] 41 | type BlobFile internal (defaultConnectionString, container, file, getBlobRef : _ -> ICloudBlob) = 42 | let getBlobRef connectionString = getBlobRef (defaultArg connectionString defaultConnectionString, container, file) 43 | member private __.BlobRef connectionString = getBlobRef connectionString 44 | 45 | /// Gets a handle to the Azure SDK client for this blob. 46 | member this.AsICloudBlob(?connectionString) = this.BlobRef(connectionString) 47 | 48 | /// Generates a shared access signature for the supplied duration and permissions. Do not pass permissions for full-access. 49 | member this.GenerateSharedAccessSignature(duration, ?connectionString, ?permissions) = 50 | let expiry = Nullable(DateTimeOffset.UtcNow.Add(duration)) 51 | let permissions = defaultArg permissions (SharedAccessBlobPermissions.Read ||| SharedAccessBlobPermissions.Write ||| 52 | SharedAccessBlobPermissions.Delete ||| SharedAccessBlobPermissions.List) 53 | let policy = SharedAccessBlobPolicy (SharedAccessExpiryTime = expiry, Permissions = permissions) 54 | let blobRef = this.BlobRef connectionString 55 | let sas = blobRef.GetSharedAccessSignature policy 56 | Uri(sprintf "%s%s" (blobRef.Uri.ToString()) sas) 57 | 58 | /// Downloads this file to the specified path. 59 | member this.Download(path, ?connectionString) = 60 | let targetDirectory = Path.GetDirectoryName(path) 61 | if not (Directory.Exists targetDirectory) then Directory.CreateDirectory targetDirectory |> ignore 62 | this.BlobRef(connectionString).DownloadToFileAsync(path, FileMode.Create) |> Async.AwaitTask 63 | 64 | /// Opens this file as a stream for reading. 65 | member this.OpenStream(?connectionString:string) = this.BlobRef(connectionString).OpenReadAsync(AccessCondition(), BlobRequestOptions(), null).Result 66 | 67 | /// Opens this file as a text stream for reading. 68 | member this.OpenStreamAsText(?connectionString) = 69 | match connectionString with 70 | | Some connectionString -> new StreamReader(this.OpenStream(connectionString)) 71 | | None -> new StreamReader(this.OpenStream()) 72 | 73 | /// Lazily read the contents of this blob a line at a time. 74 | member this.ReadLines(?connectionString) = seq { 75 | use stream = 76 | match connectionString with 77 | | Some connectionString -> this.OpenStreamAsText(connectionString) 78 | | None -> this.OpenStreamAsText() 79 | while not stream.EndOfStream do 80 | yield stream.ReadLine() } 81 | 82 | /// Fetches the latest metadata for the blob. 83 | member __.GetProperties(?connectionString) = async { 84 | let blobRef = getBlobRef connectionString 85 | do! blobRef.FetchAttributesAsync() |> Async.AwaitTask 86 | return BlobMetadata blobRef.Properties } 87 | 88 | /// Gets the blob size in bytes. 89 | member this.Size(?connectionString) = 90 | async { 91 | let! metadata = 92 | match connectionString with 93 | | Some connectionString -> this.GetProperties connectionString 94 | | None -> this.GetProperties() 95 | return metadata.Size } 96 | |> Async.RunSynchronously 97 | 98 | /// Gets the name of the blob 99 | member __.Name with get() = (getBlobRef None).Name 100 | 101 | override this.ToString() = this.Name 102 | 103 | type BlockBlobFile internal (defaultConnectionString, container, file) = 104 | inherit BlobFile(defaultConnectionString, container, file, (fun blob -> getBlockBlobRef blob :> ICloudBlob)) 105 | let getBlobRef connectionString = getBlockBlobRef(defaultArg connectionString defaultConnectionString, container, file) 106 | 107 | /// Gets a handle to the Azure SDK client for this blob. 108 | member __.AsCloudBlockBlob(?connectionString) = getBlobRef connectionString 109 | 110 | /// Reads this file as a string. 111 | member __.Read(?connectionString) = (getBlobRef connectionString).DownloadTextAsync().Result 112 | 113 | /// Reads this file as a string asynchronously. 114 | member __.ReadAsync(?connectionString) = getBlobRef(connectionString).DownloadTextAsync() |> Async.AwaitTask 115 | 116 | type PageBlobFile internal (defaultConnectionString, container, file) = 117 | inherit BlobFile(defaultConnectionString, container, file, (fun blob -> getPageBlobRef blob :> ICloudBlob)) 118 | 119 | /// Gets a handle to the Azure SDK client for this blob. 120 | member __.AsCloudPageBlob(?connectionString) = getPageBlobRef(defaultArg connectionString defaultConnectionString, container, file) 121 | 122 | /// Represents an XML file stored in blob storage. 123 | type XmlFile internal (defaultConnectionString, container, file) = 124 | inherit BlockBlobFile(defaultConnectionString, container, file) 125 | 126 | /// Reads this file as an XDocument. 127 | member this.ReadAsXDocument(?connectionString) = this.Read(defaultArg connectionString defaultConnectionString) |> XDocument.Parse 128 | 129 | /// Reads this file as an XDocument asynchronously. 130 | member this.ReadAsXDocumentAsync(?connectionString) = async { 131 | let! text = this.ReadAsync(defaultArg connectionString defaultConnectionString) 132 | return XDocument.Parse text } 133 | 134 | module BlobBuilder = 135 | let internal (|Text|Binary|XML|) (name : string) = 136 | let endsWith extension = name.EndsWith(extension, StringComparison.InvariantCultureIgnoreCase) 137 | match name with 138 | | _ when [ ".txt"; ".csv" ] |> Seq.exists endsWith -> Text 139 | | _ when endsWith ".xml" -> XML 140 | | _ -> Binary 141 | 142 | /// Creates a block blob file object. 143 | let createBlockBlobFile connectionString containerName path = 144 | let details = connectionString, containerName, path 145 | match path with 146 | | XML -> XmlFile(details) :> BlockBlobFile 147 | | Text | Binary -> BlockBlobFile(details) 148 | 149 | /// Creates a page blob file object. 150 | let createPageBlobFile connectionString containerName path = 151 | PageBlobFile(connectionString, containerName, path) 152 | 153 | let getSafe defaultConnectionString container getBlobFile connectionString path = 154 | let connectionString = connectionString |> defaultArg <| defaultConnectionString 155 | let blob : #BlobFile = getBlobFile connectionString container path 156 | async { 157 | try 158 | let! exists = blob.AsICloudBlob().ExistsAsync() |> Async.AwaitTask 159 | return if exists then Some blob else None 160 | with 161 | | :? AggregateException as ex when 162 | match ex.InnerException with 163 | | :? StorageException as ex when ex.Message = "Blob type of the blob reference doesn't match blob type of the blob." -> true 164 | | _ -> false 165 | -> return None } 166 | 167 | let listBlobs defaultConnectionString container file includeSubfolders prefix connectionString = async { 168 | let connectionString = connectionString |> defaultArg <| defaultConnectionString 169 | let includeSubfolders = includeSubfolders |> defaultArg <| false 170 | let container = getContainerRef (connectionString, container) 171 | let prefix = file + (prefix |> Option.toObj) 172 | let! blobs = listBlobs includeSubfolders container prefix 173 | 174 | return 175 | blobs 176 | |> Array.choose (function 177 | | Blob(path, _, blobType, _) -> 178 | match blobType with 179 | | BlobType.PageBlob -> (createPageBlobFile connectionString container.Name path) :> BlobFile 180 | | _ -> (createBlockBlobFile connectionString container.Name path) :> BlobFile 181 | |> Some 182 | | _ -> None) } 183 | 184 | /// Represents a pseudo-folder in blob storage. 185 | type BlobFolder internal (defaultConnectionString, container, file) = 186 | let getSafe getBlob connectionString path = 187 | let path = Path.Combine(file, path) 188 | BlobBuilder.getSafe defaultConnectionString container getBlob connectionString path 189 | let listBlobs = BlobBuilder.listBlobs defaultConnectionString container file 190 | 191 | /// Downloads the entire folder contents to the local file system asynchronously. 192 | member __.Download(path, ?connectionString) = 193 | let connectionDetails = defaultArg connectionString defaultConnectionString, container, file 194 | downloadFolder (connectionDetails, path) 195 | 196 | /// The Path of the current folder 197 | member __.Path with get() = file 198 | 199 | /// Lists all blobs contained in this folder 200 | member __.ListBlobs(?includeSubfolders, ?prefix, ?connectionString) = listBlobs includeSubfolders prefix connectionString 201 | /// Allows unsafe navigation to a blob by name. 202 | member __.Item with get(path) = BlobBuilder.createBlockBlobFile defaultConnectionString container (Path.Combine(file, path)) 203 | /// Safely retrieves a reference to a block blob asynchronously. 204 | member __.TryGetBlockBlob(path, ?connectionString) = getSafe BlobBuilder.createBlockBlobFile connectionString path 205 | /// Safely retrieves a reference to a page blob asynchronously. 206 | member __.TryGetPageBlob(path, ?connectionString) = getSafe BlobBuilder.createPageBlobFile connectionString path 207 | 208 | /// Represents a container in blob storage. 209 | type BlobContainer internal (defaultConnectionString, container) = 210 | let getSafe getBlob connectionString path = BlobBuilder.getSafe defaultConnectionString container getBlob connectionString path 211 | let listBlobs = BlobBuilder.listBlobs defaultConnectionString container "" 212 | let getBlobContainerRef connectionString = getContainerRef(defaultArg connectionString defaultConnectionString, container) 213 | 214 | /// Gets a handle to the Azure SDK client for this container. 215 | member __.AsCloudBlobContainer(?connectionString) = getBlobContainerRef connectionString 216 | 217 | /// Downloads the entire container contents to the local file system asynchronously. 218 | member __.Download(path, ?connectionString) = 219 | let connectionDetails = (defaultArg connectionString defaultConnectionString), container, String.Empty 220 | downloadFolder (connectionDetails, path) 221 | 222 | /// Uploads a file to this container. 223 | member __.Upload(path, ?connectionString) = 224 | let filename = path |> Path.GetFileName 225 | let blobRef = getBlockBlobRef ((defaultArg connectionString defaultConnectionString), container, filename) 226 | 227 | // Set the MIME type of the file if we can 228 | Path.GetExtension filename 229 | |> MimeTypes.tryFindMimeType 230 | |> Option.iter(fun mimeType -> blobRef.Properties.ContentType <- mimeType) 231 | 232 | blobRef.UploadFromFileAsync path |> Async.AwaitTask 233 | 234 | /// Gets the name of this container. 235 | member __.Name with get() = container 236 | /// Allows unsafe navigation to a blob by name. 237 | member __.Item with get(path) = BlobBuilder.createBlockBlobFile defaultConnectionString container path 238 | /// Safely retrieves a reference to a block blob asynchronously. 239 | member __.TryGetBlockBlob(path, ?connectionString) = getSafe BlobBuilder.createBlockBlobFile connectionString path 240 | /// Safely retrieves a reference to a page blob asynchronously. 241 | member __.TryGetPageBlob(path, ?connectionString) = getSafe BlobBuilder.createPageBlobFile connectionString path 242 | /// Lists all blobs contained in this container. 243 | member __.ListBlobs(?includeSubfolders, ?prefix, ?connectionString) = listBlobs includeSubfolders prefix connectionString 244 | /// Fetches the latest metadata for the blob container. 245 | member __.GetProperties(?connectionString) = async { 246 | let containerRef = getBlobContainerRef connectionString 247 | do! containerRef.FetchAttributesAsync() |> Async.AwaitTask 248 | return BlobContainerMetadata containerRef.Properties } 249 | 250 | /// Builder methods to construct blobs etc.. 251 | /// [omit] 252 | module ContainerBuilder = 253 | /// Creates a blob container object. 254 | let createContainer connectionString containerName = BlobContainer(connectionString, containerName) 255 | 256 | /// Creates a blob folder object. 257 | let createBlobFolder connectionString containerName path = BlobFolder(connectionString, containerName, path) 258 | 259 | /// Creates a blob client. 260 | let createBlobClient connectionString = BlobRepository.getBlobClient connectionString -------------------------------------------------------------------------------- /src/FSharp.Azure.StorageTypeProvider/Table/TableRepository.fs: -------------------------------------------------------------------------------- 1 | ///[omit] 2 | ///Contains helper functions for accessing tables 3 | module FSharp.Azure.StorageTypeProvider.Table.TableRepository 4 | 5 | open FSharp.Azure.StorageTypeProvider.Table 6 | open FSharp.Azure.StorageTypeProvider 7 | open Microsoft.WindowsAzure.Storage 8 | open Microsoft.WindowsAzure.Storage.Table 9 | open System 10 | 11 | /// Suggests batch sizes based on a given entity type and published EDM property type sizes (source: https://msdn.microsoft.com/en-us/library/dd179338.aspx) 12 | module private BatchCalculator = 13 | /// The basic size of a single row with no custom properties. 14 | let private basicRowSize = 15 | let partitionKey = 1 16 | let rowKey = 1 17 | let timestamp = 2 18 | partitionKey + rowKey + timestamp 19 | 20 | /// Gets the maximum size, in KB, of a single property. 21 | let private getMaxPropertySize (property:EntityProperty) = 22 | match property.PropertyType with 23 | | EdmType.DateTime -> 2 24 | | EdmType.Binary -> 64 25 | | EdmType.Boolean -> 1 26 | | EdmType.Double -> 2 27 | | EdmType.Guid -> 4 28 | | EdmType.Int32 -> 1 29 | | EdmType.Int64 -> 1 30 | | EdmType.String -> 64 31 | | unknown -> failwith (sprintf "Unknown EdmType %A" unknown) 32 | 33 | /// Calculates the maximum size of a given entity. 34 | let private getMaxEntitySize (entity:DynamicTableEntity) = 35 | let entityRowSize = entity.Properties.Values |> Seq.sumBy getMaxPropertySize 36 | basicRowSize + entityRowSize 37 | 38 | let private maximumBatchSizeKb = 4000 39 | 40 | /// Calculates the maximum number of entities of a given type that can be inserted in a single batch. 41 | let getBatchSize entity = maximumBatchSizeKb / (getMaxEntitySize entity) 42 | 43 | let internal getTableClient connection = CloudStorageAccount.Parse(connection).CreateCloudTableClient() 44 | 45 | let private boxedNone = box None 46 | let buildTableEntity partitionKey rowKey names (values: obj []) = 47 | let properties = 48 | Seq.zip names values 49 | |> Seq.choose(fun (name, value) -> 50 | match value with 51 | | value when value = boxedNone -> None 52 | | :? Option as option -> option |> Option.map(fun value -> name, box value) 53 | | :? Option as option -> option |> Option.map(fun value -> name, box value) 54 | | :? Option as option -> option |> Option.map(fun value -> name, box value) 55 | | :? Option as option -> option |> Option.map(fun value -> name, box value) 56 | | :? Option as option -> option |> Option.map(fun value -> name, box value) 57 | | :? Option as option -> option |> Option.map(fun value -> name, box value) 58 | | :? Option as option -> option |> Option.map(fun value -> name, box value) 59 | | :? Option as option -> option |> Option.map(fun value -> name, box value) 60 | | value -> Some(name, value)) 61 | |> Map.ofSeq 62 | 63 | LightweightTableEntity(partitionKey, rowKey, DateTimeOffset.MinValue, properties) 64 | 65 | let internal getTable tableName connection = 66 | let client = getTableClient connection 67 | client.GetTableReference tableName 68 | 69 | type private DynamicQuery = DynamicTableEntity TableQuery 70 | 71 | [] 72 | module private SdkExtensions = 73 | type CloudTableClient with 74 | member cloudTableClient.ListTablesAsync() = 75 | let getTables token = async { 76 | let! result = cloudTableClient.ListTablesSegmentedAsync token |> Async.AwaitTask 77 | return result.ContinuationToken, result.Results } 78 | Async.segmentedAzureOperation getTables 79 | 80 | type CloudTable with 81 | member cloudTable.ExecuteQueryAsync(query:DynamicQuery) = 82 | let mutable remainingRows = query.TakeCount |> Option.ofNullable 83 | let doQuery token = async { 84 | let! query = cloudTable.ExecuteQuerySegmentedAsync(query, token) |> Async.AwaitTask 85 | remainingRows <- remainingRows |> Option.map(fun remainingRows -> remainingRows - query.Results.Count) 86 | let token = match remainingRows with Some x when x <= 0 -> null | None | Some _ -> query.ContinuationToken 87 | return token, query.Results } 88 | Async.segmentedAzureOperation doQuery 89 | 90 | /// Gets all tables 91 | let internal getTables connection = async { 92 | let client = getTableClient connection 93 | let! results = client.ListTablesAsync() 94 | return results |> Array.map(fun table -> table.Name) } 95 | 96 | let internal getMetricsTables connection = 97 | let services = [ "Blob"; "Queue"; "Table"; "File" ] 98 | let locations = [ "Primary"; "Secondary" ] 99 | let periods = [ "Hourly", "Hour"; "Per Minute", "Minute" ] 100 | 101 | let client = getTableClient connection 102 | seq { 103 | for (description, period) in periods do 104 | for location in locations do 105 | for service in services do 106 | let tableName = sprintf "$Metrics%s%sTransactions%s" period location service 107 | if (client.GetTableReference(tableName).ExistsAsync().Result) then 108 | yield description, location, service, tableName } 109 | 110 | let internal getRowsForSchema (rowCount: int) connection tableName = async { 111 | let table = getTable tableName connection 112 | let! results = table.ExecuteQueryAsync(DynamicQuery().Take(Nullable rowCount)) 113 | return results |> Array.truncate rowCount } 114 | 115 | let toLightweightTableEntity (dte:DynamicTableEntity) = 116 | LightweightTableEntity( 117 | Partition dte.PartitionKey, 118 | Row dte.RowKey, 119 | dte.Timestamp, 120 | dte.Properties 121 | |> Seq.map(fun p -> p.Key, p.Value.PropertyAsObject) 122 | |> Map.ofSeq) 123 | 124 | let executeGenericQueryAsync connection tableName maxResults filterString mapToReturnEntity = async { 125 | let query = 126 | let query = DynamicQuery().Where(filterString) 127 | if maxResults > 0 then query.Take(Nullable maxResults) else query 128 | let table = getTable tableName connection 129 | let! output = table.ExecuteQueryAsync query 130 | return output |> Array.map mapToReturnEntity } 131 | 132 | let executeQueryAsync connection tableName maxResults filterString = 133 | executeGenericQueryAsync connection tableName maxResults filterString toLightweightTableEntity 134 | 135 | let internal buildDynamicTableEntity(entity:LightweightTableEntity) = 136 | let tableEntity = DynamicTableEntity(entity.PartitionKey, entity.RowKey, ETag = "*") 137 | for (key, value) in entity.Values |> Map.toArray do 138 | tableEntity.Properties.[key] <- 139 | match value with 140 | | :? (byte []) as value -> EntityProperty.GeneratePropertyForByteArray(value) 141 | | :? string as value -> EntityProperty.GeneratePropertyForString(value) 142 | | :? int as value -> EntityProperty.GeneratePropertyForInt(Nullable value) 143 | | :? bool as value -> EntityProperty.GeneratePropertyForBool(Nullable value) 144 | | :? DateTime as value -> EntityProperty.GeneratePropertyForDateTimeOffset(Nullable(DateTimeOffset value)) 145 | | :? double as value -> EntityProperty.GeneratePropertyForDouble(Nullable value) 146 | | :? System.Guid as value -> EntityProperty.GeneratePropertyForGuid(Nullable value) 147 | | :? int64 as value -> EntityProperty.GeneratePropertyForLong(Nullable value) 148 | | _ -> EntityProperty.CreateEntityPropertyFromObject(value) 149 | tableEntity 150 | 151 | let internal createInsertOperation(insertMode) = 152 | match insertMode with 153 | | TableInsertMode.Insert -> TableOperation.Insert 154 | | TableInsertMode.Upsert -> TableOperation.InsertOrReplace 155 | | _ -> failwith "unknown insertion mode" 156 | 157 | let private batch size source = 158 | let rec doBatch output currentBatch counter remainder = 159 | match remainder with 160 | | [] -> if List.isEmpty currentBatch then output else currentBatch::output |> List.rev 161 | | theList when counter = size -> doBatch ((currentBatch |> List.rev) ::output) [] 0 theList 162 | | head::tail -> doBatch output (head::currentBatch) (counter + 1) tail 163 | doBatch [] [] 0 (source |> Seq.toList) 164 | 165 | let private splitIntoBatches createTableOp entities = 166 | match entities with 167 | | entities when Seq.isEmpty entities -> Seq.empty 168 | | entities -> 169 | let batchSize = entities |> Seq.head |> BatchCalculator.getBatchSize 170 | entities 171 | |> Seq.groupBy(fun (entity:DynamicTableEntity) -> entity.PartitionKey) 172 | |> Seq.collect(fun (partitionKey, entities) -> 173 | entities 174 | |> batch batchSize 175 | |> Seq.map(fun entityBatch -> 176 | let batchForPartition = TableBatchOperation() 177 | entityBatch |> Seq.iter (createTableOp >> batchForPartition.Add) 178 | partitionKey, entityBatch, batchForPartition)) 179 | 180 | let private processErrorResp entityBatch buildEntityId (ex:StorageException) = 181 | let requestResult = ex.RequestInformation 182 | match requestResult.ExtendedErrorInformation.ErrorMessage.Split('\n').[0].Split(':') with 183 | | [|index; _|] -> 184 | match Int32.TryParse(index) with 185 | | true, index -> 186 | entityBatch 187 | |> Seq.mapi(fun entityIndex entity -> 188 | if entityIndex = index then EntityError(buildEntityId entity, requestResult.HttpStatusCode, requestResult.ExtendedErrorInformation.ErrorCode) 189 | else BatchOperationFailedError(buildEntityId entity)) 190 | | _ -> entityBatch |> Seq.map(fun entity -> BatchError(buildEntityId entity, requestResult.HttpStatusCode, requestResult.ExtendedErrorInformation.ErrorCode)) 191 | | [| _ |] -> entityBatch |> Seq.map(fun entity -> EntityError(buildEntityId entity, requestResult.HttpStatusCode, requestResult.ExtendedErrorInformation.ErrorCode)) 192 | | _ -> entityBatch |> Seq.map(fun entity -> BatchError(buildEntityId entity, requestResult.HttpStatusCode, requestResult.ExtendedErrorInformation.ErrorCode)) 193 | 194 | let internal executeBatchAsynchronously batchOp entityBatch buildEntityId (table:CloudTable) = 195 | batchOp 196 | |> table.ExecuteBatchAsync 197 | |> Async.AwaitTask 198 | |> Async.toAsyncResult 199 | |> Async.map(function 200 | | Ok reponse -> 201 | reponse 202 | |> Seq.zip entityBatch 203 | |> Seq.map(fun (entity, res) -> SuccessfulResponse(buildEntityId entity, res.HttpStatusCode)) 204 | | Error ([ :? StorageException as ex ]) -> processErrorResp entityBatch buildEntityId ex 205 | | Error [] -> failwith "An unknown error occurred." 206 | | Error (topException :: _) -> raise topException) 207 | 208 | let internal executeBatchOperationAsync createTableOp (table:CloudTable) entities = async { 209 | return! 210 | splitIntoBatches createTableOp entities 211 | |> Seq.map(fun (partitionKey, entityBatch, batchOperation) -> async{ 212 | let buildEntityId (entity:DynamicTableEntity) = Partition(entity.PartitionKey), Row(entity.RowKey) 213 | let! responses = executeBatchAsynchronously batchOperation entityBatch buildEntityId table 214 | return (partitionKey, responses |> Seq.toArray) }) 215 | |> Async.Parallel } 216 | 217 | let deleteEntities connection tableName entities = 218 | let table = getTable tableName connection 219 | entities 220 | |> Array.map buildDynamicTableEntity 221 | |> executeBatchOperationAsync TableOperation.Delete table 222 | 223 | let deleteEntitiesAsync connection tableName entities = async { 224 | let table = getTable tableName connection 225 | return! 226 | entities 227 | |> Array.map buildDynamicTableEntity 228 | |> executeBatchOperationAsync TableOperation.Delete table } 229 | 230 | let deleteEntityAsync connection tableName entity = async { 231 | let! resp = deleteEntitiesAsync connection tableName [| entity |] 232 | return resp |> Seq.head |> snd |> Seq.head } 233 | 234 | let insertEntityBatchAsync connection tableName insertMode entities = async { 235 | let table = getTable tableName connection 236 | let insertOp = createInsertOperation insertMode 237 | return! 238 | entities 239 | |> Seq.map buildDynamicTableEntity 240 | |> executeBatchOperationAsync insertOp table } 241 | 242 | let insertEntityAsync connection tableName insertMode entity = async { 243 | let! resp = insertEntityBatchAsync connection tableName insertMode [entity] 244 | return resp |> Seq.head |> snd |> Seq.head } 245 | 246 | let composeAllFilters filters = 247 | match filters with 248 | | [] -> String.Empty 249 | | _ -> 250 | filters 251 | |> List.rev 252 | |> List.reduce(fun acc filter -> TableQuery.CombineFilters(acc, TableOperators.And, filter)) 253 | 254 | let buildFilter(propertyName, comparison, value) = 255 | match box value with 256 | | :? string as value -> TableQuery.GenerateFilterCondition(propertyName, comparison, value) 257 | | :? int as value -> TableQuery.GenerateFilterConditionForInt(propertyName, comparison, value) 258 | | :? int64 as value -> TableQuery.GenerateFilterConditionForLong(propertyName, comparison, value) 259 | | :? float as value -> TableQuery.GenerateFilterConditionForDouble(propertyName, comparison, value) 260 | | :? bool as value -> TableQuery.GenerateFilterConditionForBool(propertyName, comparison, value) 261 | | :? DateTime as value -> TableQuery.GenerateFilterConditionForDate(propertyName, comparison, DateTimeOffset value) 262 | | :? Guid as value -> TableQuery.GenerateFilterConditionForGuid(propertyName, comparison, value) 263 | | _ -> TableQuery.GenerateFilterCondition(propertyName, comparison, value.ToString()) 264 | 265 | let buildGetEntityQry rowKey partitionKey = 266 | let (Row rowKey, Partition partitionKey) = rowKey, partitionKey 267 | [ ("RowKey", rowKey); ("PartitionKey", partitionKey) ] 268 | |> List.map(fun (prop, value) -> buildFilter(prop, QueryComparisons.Equal, value)) 269 | |> composeAllFilters 270 | 271 | let parseGetEntityResults results = 272 | match results with 273 | | [| exactMatch |] -> Some exactMatch 274 | | _ -> None 275 | 276 | let getEntityAsync rowKey partitionKey connection tableName = async { 277 | let! results = 278 | buildGetEntityQry rowKey partitionKey 279 | |> executeQueryAsync connection tableName 0 280 | return results |> parseGetEntityResults } 281 | 282 | let getPartitionRowsAsync (partitionKey:string) connection tableName = async { 283 | return! 284 | buildFilter("PartitionKey", QueryComparisons.Equal, partitionKey) 285 | |> executeQueryAsync connection tableName 0 } --------------------------------------------------------------------------------