├── tools ├── apidoc │ └── .gitignore ├── strongnamekeys │ └── fake │ │ └── windows.snk ├── nupkg │ ├── buildDebugNupkg.cmd │ └── buildNupkg.cmd ├── AssemblyInfo │ └── SharedAssemblyInfo.cs └── scripts │ └── InjectBuildNumber.ps1 ├── .nuget ├── NuGet.exe └── NuGet.Config ├── lib ├── dnxcore50 │ └── XUnitForMsTest.dll ├── TransferEnumerators │ ├── INameResolver.cs │ ├── ListContinuationToken.cs │ ├── FileToAzureBlobNameResolver.cs │ ├── AzureFileToAzureNameResolver.cs │ ├── TransferEnumeratorBase.cs │ ├── AzureBlobToAzureBlobNameResolver.cs │ ├── ErrorEntry.cs │ ├── ITransferEnumerator.cs │ ├── FileToAzureFileNameResolver.cs │ ├── FileEntry.cs │ ├── DirectoryEntry.cs │ ├── AzureBlobEntry.cs │ ├── TransferEntry.cs │ ├── AzureFileEntry.cs │ ├── NameResolverHelper.cs │ ├── AzureFileDirectoryEntry.cs │ ├── FileListContinuationToken.cs │ ├── AzureFileListContinuationToken.cs │ └── TaskQueue.cs ├── AssemblyInfo.cs ├── TransferJobs │ ├── TransferLocationType.cs │ ├── TransferMethod.cs │ ├── TransferJobStatus.cs │ └── TransferLocation.cs ├── JournalItem.cs ├── TransferControllers │ ├── ITransferController.cs │ └── ServiceSideSyncCopyControllers │ │ ├── ServiceSideSyncCopySource │ │ ├── IRangeBasedSourceHandler.cs │ │ ├── ISourceHandler.cs │ │ └── PageBlobSourceHandler.cs │ │ └── ServiceSideSyncCopyDest │ │ ├── BlockBlobDestHandler.cs │ │ ├── AppendBlobDestHandler.cs │ │ ├── PageBlobDestHandler.cs │ │ └── IDestHandler.cs ├── TestHook │ ├── TestHookCallbacks.cs │ └── FaultInjectionPoint.cs ├── GlobalMemoryStatusNativeMethods.cs ├── packages.config ├── TransferStatusHelpers │ ├── TransferData.cs │ ├── TransferDownloadBuffer.cs │ ├── TransferDataState.cs │ ├── ReadDataState.cs │ └── SharedTransferData.cs ├── CopyMethod.cs ├── LongPathFile.cs ├── SingleTransferContext.cs ├── TransferCallbacks.cs ├── DirectoryTransferContext.cs ├── TransferEventArgs.cs ├── TransferOptions │ ├── DirectoryOptions.cs │ └── CopyOptions.cs ├── Interop │ └── Interop.OSX.cs ├── Exceptions │ └── TransferSkippedException.cs └── PreserveSMBPermissions.cs ├── samples ├── PreserveFilePropertiesSamples │ ├── netcore │ │ ├── Config.json │ │ ├── azure.png │ │ └── PreserveFilePropertiesSamples.csproj │ ├── PreserveFilePropertiesSamples │ │ ├── azure.png │ │ ├── packages.config │ │ ├── App.config │ │ ├── Properties │ │ │ └── AssemblyInfo.cs │ │ └── FileOperations.cs │ ├── PreserveFilePropertiesSamples_k.sln │ └── PreserveFilePropertiesSamples.sln ├── DataMovementSamples │ ├── DataMovementSamples │ │ ├── azure.png │ │ ├── App.config │ │ ├── packages.config │ │ └── Properties │ │ │ └── AssemblyInfo.cs │ ├── netcore │ │ └── DataMovementSamples │ │ │ ├── Config.json │ │ │ ├── azure.png │ │ │ ├── DataContractSerializerExtensions.cs │ │ │ └── DataMovementSamples.csproj │ ├── DataMovementSamples.sln │ └── DataMovementSamples_k.sln └── S3ToAzureSample │ ├── S3ToAzureSample │ ├── App.config │ ├── packages.config │ └── Properties │ │ └── AssemblyInfo.cs │ └── S3ToAzureSample.sln ├── netcore ├── DMLibTest │ ├── Constants.cs │ ├── XunitWrapper │ │ ├── Classes.cs │ │ ├── DisableTestParallelization.cs │ │ ├── TestCategoryDiscoverer.cs │ │ └── Attributes.cs │ ├── DataContractSerializerExtensions.cs │ ├── TestStartEndAttribute.cs │ ├── CloudBlobDirectoryExtensions.cs │ ├── CloudAppendBlobExtensions.cs │ ├── TestData.xml │ ├── XunitLogger.cs │ ├── CloudFileDirectoryExtensions.cs │ ├── CloudFileShareExtensions.cs │ ├── CloudPageBlobExtensions.cs │ └── CloudBlockBlobExtensions.cs ├── MsTestLib │ ├── StreamWriterExtensions.cs │ ├── DateTimeExtensions.cs │ ├── Properties │ │ └── AssemblyInfo.cs │ └── MsTestLib.csproj ├── Microsoft.Azure.Storage.DataMovement │ └── DataContractSerializationExtensions.cs ├── Build.cmd └── DMTestLib │ └── DMTestLib.csproj ├── wrap └── XUnitForMsTest │ └── project.json ├── .github └── ISSUE_TEMPLATE.md ├── test ├── DMLibTestCodeGen │ ├── packages.config │ ├── Properties │ │ └── AssemblyInfo.cs │ ├── App.config │ ├── TransferDirectionExtensions.cs │ └── Program.cs ├── DMLibTest │ ├── Framework │ │ ├── IDataInfo.cs │ │ ├── DMLibWrapper.cs │ │ ├── TestResult.cs │ │ ├── FileNativeMethodsExtension.cs │ │ ├── MultiDirectionTestInfo.cs │ │ ├── DMLibInputHelper.cs │ │ ├── AssemblyInitCleanup.cs │ │ ├── SharedAccessPermissions.cs │ │ ├── TestExecutionOptions.cs │ │ ├── DataAdaptor.cs │ │ ├── URIBlobDataAdaptor.cs │ │ └── CloudObjectExtensions.cs │ ├── Properties │ │ └── AssemblyInfo.cs │ ├── packages.config │ ├── app.config │ ├── TestData.xml │ └── Util │ │ └── DMLibTestStream.cs ├── DMTestLib │ ├── MultiDirectionTag.cs │ ├── DMLibCopyMethod.cs │ ├── DMLibTestContext.cs │ ├── ITestDirection.cs │ ├── MultiDirectionTestClassAttribute.cs │ ├── MultiDirectionTestContext.cs │ ├── MultiDirectionTestMethodAttribute.cs │ ├── TestMethodDirection.cs │ ├── Properties │ │ └── AssemblyInfo.cs │ ├── DMLibDataType.cs │ ├── MultiDirectionTestMethod.cs │ ├── MultiDirectionTestMethodSetAttribute.cs │ └── DirectionFilter.cs ├── DMLibTest_NetStandard │ ├── Properties │ │ └── AssemblyInfo.cs │ ├── packages.config │ └── app.config └── MsTestLib │ ├── Exceptions.cs │ ├── MethodConfig.cs │ ├── ILogger.cs │ └── ClassConfig.cs ├── CONTRIBUTING.md ├── LICENSE ├── .gitattributes └── DataMovement_k.sln /tools/apidoc/.gitignore: -------------------------------------------------------------------------------- 1 | Help/ -------------------------------------------------------------------------------- /.nuget/NuGet.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azure-storage-net-data-movement/HEAD/.nuget/NuGet.exe -------------------------------------------------------------------------------- /lib/dnxcore50/XUnitForMsTest.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azure-storage-net-data-movement/HEAD/lib/dnxcore50/XUnitForMsTest.dll -------------------------------------------------------------------------------- /tools/strongnamekeys/fake/windows.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azure-storage-net-data-movement/HEAD/tools/strongnamekeys/fake/windows.snk -------------------------------------------------------------------------------- /samples/PreserveFilePropertiesSamples/netcore/Config.json: -------------------------------------------------------------------------------- 1 | { 2 | "StorageConnectionString": "DefaultEndpointsProtocol=http;AccountName=[AccountName];AccountKey=[AccountKey]" 3 | } -------------------------------------------------------------------------------- /samples/PreserveFilePropertiesSamples/netcore/azure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azure-storage-net-data-movement/HEAD/samples/PreserveFilePropertiesSamples/netcore/azure.png -------------------------------------------------------------------------------- /samples/DataMovementSamples/DataMovementSamples/azure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azure-storage-net-data-movement/HEAD/samples/DataMovementSamples/DataMovementSamples/azure.png -------------------------------------------------------------------------------- /samples/DataMovementSamples/netcore/DataMovementSamples/Config.json: -------------------------------------------------------------------------------- 1 | { 2 | "StorageConnectionString": "DefaultEndpointsProtocol=http;AccountName=[AccountName];AccountKey=[AccountKey]" 3 | } -------------------------------------------------------------------------------- /.nuget/NuGet.Config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /samples/DataMovementSamples/netcore/DataMovementSamples/azure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azure-storage-net-data-movement/HEAD/samples/DataMovementSamples/netcore/DataMovementSamples/azure.png -------------------------------------------------------------------------------- /samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/azure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azure-storage-net-data-movement/HEAD/samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/azure.png -------------------------------------------------------------------------------- /netcore/DMLibTest/Constants.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.Azure.Storage.Blob; 3 | 4 | namespace DMLibTest 5 | { 6 | internal static class Collections 7 | { 8 | public const string Global = "global"; 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /netcore/DMLibTest/XunitWrapper/Classes.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | 6 | namespace Microsoft.VisualStudio.TestTools.UnitTesting 7 | { 8 | public abstract class TestContext 9 | { } 10 | } 11 | -------------------------------------------------------------------------------- /netcore/MsTestLib/StreamWriterExtensions.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | 3 | namespace MS.Test.Common.MsTestLib 4 | { 5 | internal static class StreamWriterExtensions 6 | { 7 | public static void Close(this StreamWriter writer) 8 | { 9 | writer.Dispose(); 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /netcore/MsTestLib/DateTimeExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace MS.Test.Common.MsTestLib 4 | { 5 | public static class DateTimeExtensions 6 | { 7 | public static string ToLongTimeString(this DateTime dateTime) 8 | { 9 | return dateTime.ToString("h:mm:ss tt"); 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /wrap/XUnitForMsTest/project.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.0.0-*", 3 | "frameworks": { 4 | "dnxcore50": { 5 | "bin": { 6 | "assembly": "../../lib/dnxcore50/XUnitForMsTest.dll" 7 | } 8 | }, 9 | "dnx451": { 10 | "bin": { 11 | "assembly": "../../lib/dnx451/XUnitForMsTest.dll" 12 | } 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Which service(blob, file) does this issue concern? 2 | 3 | 4 | ### Which version of the SDK was used? 5 | 6 | 7 | ### On which platform were you using? (.Net Framework version or .Net Core version, and OS version) 8 | 9 | 10 | ### How can the problem be reproduced? It'd be better if the code caused the problem can be shared. 11 | 12 | 13 | ### What problem was encountered? 14 | 15 | 16 | ### Have you found a mitigation/solution? 17 | 18 | -------------------------------------------------------------------------------- /test/DMLibTestCodeGen/packages.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /netcore/DMLibTest/XunitWrapper/DisableTestParallelization.cs: -------------------------------------------------------------------------------- 1 | // By default, xUnit runs tests in parallel. DMLibTest relies on a static class 2 | // for logging (MS.Test.Common.MsTestLib.Test), and the architecture of the test 3 | // code doesn't enable a straightforward way to make to make it thread-safe. 4 | // http://xunit.github.io/docs/running-tests-in-parallel.html 5 | // http://xunit.github.io/docs/capturing-output.html 6 | [assembly: Xunit.CollectionBehavior(DisableTestParallelization = true)] -------------------------------------------------------------------------------- /test/DMLibTest/Framework/IDataInfo.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTest 7 | { 8 | public interface IDataInfo 9 | { 10 | string ToString(); 11 | 12 | IDataInfo Clone(); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /test/DMTestLib/MultiDirectionTag.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTestCodeGen 7 | { 8 | public static class MultiDirectionTag 9 | { 10 | public const string MultiDirection = "multiDirection"; 11 | public const string Async = "async"; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /test/DMTestLib/DMLibCopyMethod.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | using System; 8 | 9 | namespace DMLibTestCodeGen 10 | { 11 | [Flags] 12 | public enum DMLibCopyMethod : int 13 | { 14 | SyncCopy = 0x01, 15 | ServiceSideAsyncCopy = 0x02, 16 | ServiceSideSyncCopy = 0x04, 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /test/DMTestLib/DMLibTestContext.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTestCodeGen 7 | { 8 | public class DMLibTestContext : MultiDirectionTestContext 9 | { 10 | public static DMLibCopyMethod CopyMethod 11 | { 12 | get; 13 | set; 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Collaborate & Contribute 2 | 3 | We gladly accept community contributions. 4 | 5 | - Issues: Please report bugs using the Issues section of GitHub 6 | - Forums: Interact with the development teams on StackOverflow or the Microsoft Azure Forums 7 | - Source Code Contributions: Please follow the [contribution guidelines for Microsoft Azure open source](http://azure.github.io/guidelines.html) that details information on onboarding as a contributor 8 | 9 | For general suggestions about Microsoft Azure please use our [UserVoice forum](http://feedback.azure.com/forums/34192--general-feedback). 10 | -------------------------------------------------------------------------------- /netcore/DMLibTest/DataContractSerializerExtensions.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | using System.Runtime.Serialization; 3 | 4 | namespace DMLibTest 5 | { 6 | internal static class DataContractSerializerExtensions 7 | { 8 | public static void Serialize(this DataContractSerializer serializer, Stream stream, object graph) 9 | { 10 | serializer.WriteObject(stream, graph); 11 | } 12 | 13 | public static object Deserialize(this DataContractSerializer serializer, Stream stream) 14 | { 15 | return serializer.ReadObject(stream); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /lib/TransferEnumerators/INameResolver.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferEnumerators 8 | { 9 | /// 10 | /// INameResolver interface. 11 | /// 12 | internal interface INameResolver 13 | { 14 | string ResolveName(TransferEntry sourceEntry); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /netcore/DMLibTest/TestStartEndAttribute.cs: -------------------------------------------------------------------------------- 1 | using MS.Test.Common.MsTestLib; 2 | using System.Reflection; 3 | using Xunit.Sdk; 4 | 5 | namespace DMLibTest 6 | { 7 | public class TestStartEndAttribute : BeforeAfterTestAttribute 8 | { 9 | public override void Before(MethodInfo methodUnderTest) 10 | { 11 | Test.Start(methodUnderTest.DeclaringType.Name, methodUnderTest.Name); 12 | } 13 | 14 | public override void After(MethodInfo methodUnderTest) 15 | { 16 | Test.End(methodUnderTest.DeclaringType.Name, methodUnderTest.Name); 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /netcore/DMLibTest/XunitWrapper/TestCategoryDiscoverer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using Xunit.Abstractions; 3 | using Xunit.Sdk; 4 | 5 | public class TestCategoryDiscoverer : ITraitDiscoverer 6 | { 7 | public IEnumerable> GetTraits(IAttributeInfo traitAttribute) 8 | { 9 | IEnumerator enumerator = traitAttribute.GetConstructorArguments().GetEnumerator(); 10 | while (enumerator.MoveNext()) 11 | { 12 | yield return new KeyValuePair("Category", enumerator.Current.ToString()); 13 | } 14 | yield break; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /test/DMTestLib/ITestDirection.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTestCodeGen 7 | { 8 | public interface ITestDirection where TDataType : struct 9 | { 10 | TDataType SourceType 11 | { 12 | get; 13 | } 14 | 15 | TDataType DestType 16 | { 17 | get; 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /lib/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | using System.Reflection; 7 | 8 | // General Information about an assembly is controlled through the following 9 | // set of attributes. Change these attribute values to modify the information 10 | // associated with an assembly. 11 | [assembly: AssemblyTitle("Microsoft.Azure.Storage.DataMovement.dll")] 12 | [assembly: AssemblyDescription("")] 13 | -------------------------------------------------------------------------------- /samples/DataMovementSamples/netcore/DataMovementSamples/DataContractSerializerExtensions.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | using System.Runtime.Serialization; 3 | 4 | namespace DataMovementSamples 5 | { 6 | internal static class DataContractSerializerExtensions 7 | { 8 | public static void Serialize(this DataContractSerializer serializer, Stream stream, object graph) 9 | { 10 | serializer.WriteObject(stream, graph); 11 | } 12 | 13 | public static object Deserialize(this DataContractSerializer serializer, Stream stream) 14 | { 15 | return serializer.ReadObject(stream); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /test/DMTestLib/MultiDirectionTestClassAttribute.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTestCodeGen 7 | { 8 | using System; 9 | 10 | [AttributeUsage(AttributeTargets.Class, Inherited = false)] 11 | public class MultiDirectionTestClassAttribute : Attribute 12 | { 13 | public MultiDirectionTestClassAttribute() 14 | { 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /netcore/Microsoft.Azure.Storage.DataMovement/DataContractSerializationExtensions.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | using System.Runtime.Serialization; 3 | 4 | namespace Microsoft.Azure.Storage.DataMovement 5 | { 6 | internal static class DataContractSerializationExtensions 7 | { 8 | public static void Serialize(this DataContractSerializer serializer, Stream stream, object graph) 9 | { 10 | serializer.WriteObject(stream, graph); 11 | } 12 | 13 | public static object Deserialize(this DataContractSerializer serializer, Stream stream) 14 | { 15 | return serializer.ReadObject(stream); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /lib/TransferJobs/TransferLocationType.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement 8 | { 9 | internal enum TransferLocationType 10 | { 11 | FilePath, 12 | Stream, 13 | AzureBlob, 14 | AzureFile, 15 | SourceUri, 16 | LocalDirectory, 17 | AzureBlobDirectory, 18 | AzureFileDirectory, 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /samples/S3ToAzureSample/S3ToAzureSample/App.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /test/DMLibTest/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | using System.Reflection; 7 | using System.Runtime.CompilerServices; 8 | using System.Runtime.InteropServices; 9 | 10 | // General Information about an assembly is controlled through the following 11 | // set of attributes. Change these attribute values to modify the information 12 | // associated with an assembly. 13 | // [assembly: AssemblyTitle("DMLibTest")] 14 | // [assembly: AssemblyDescription("")] 15 | -------------------------------------------------------------------------------- /test/DMTestLib/MultiDirectionTestContext.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTestCodeGen 7 | { 8 | public class MultiDirectionTestContext where TDataType : struct 9 | { 10 | public static TDataType SourceType 11 | { 12 | get; 13 | set; 14 | } 15 | 16 | public static TDataType DestType 17 | { 18 | get; 19 | set; 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /lib/TransferEnumerators/ListContinuationToken.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferEnumerators 8 | { 9 | using System; 10 | using System.Runtime.Serialization; 11 | 12 | /// 13 | /// Base class of list continuation tokens 14 | /// 15 | #if !BINARY_SERIALIZATION 16 | [DataContract] 17 | #endif 18 | internal abstract class ListContinuationToken 19 | { 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /test/DMLibTestCodeGen/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | using System.Reflection; 7 | using System.Runtime.CompilerServices; 8 | using System.Runtime.InteropServices; 9 | 10 | // General Information about an assembly is controlled through the following 11 | // set of attributes. Change these attribute values to modify the information 12 | // associated with an assembly. 13 | // [assembly: AssemblyTitle("DMLibTestCodeGen")] 14 | // [assembly: AssemblyDescription("")] 15 | -------------------------------------------------------------------------------- /test/DMLibTest_NetStandard/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | using System.Reflection; 7 | using System.Runtime.CompilerServices; 8 | using System.Runtime.InteropServices; 9 | 10 | // General Information about an assembly is controlled through the following 11 | // set of attributes. Change these attribute values to modify the information 12 | // associated with an assembly. 13 | // [assembly: AssemblyTitle("DMLibTest")] 14 | // [assembly: AssemblyDescription("")] 15 | -------------------------------------------------------------------------------- /lib/JournalItem.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement 8 | { 9 | 10 | #if !BINARY_SERIALIZATION 11 | using System.Runtime.Serialization; 12 | [DataContract] 13 | #endif // BINARY_SERIALIZATION 14 | internal abstract class JournalItem 15 | { 16 | public StreamJournal Journal 17 | { 18 | get; 19 | set; 20 | } 21 | 22 | public long StreamJournalOffset { get; set; } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /samples/DataMovementSamples/DataMovementSamples/App.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /lib/TransferControllers/ITransferController.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace Microsoft.Azure.Storage.DataMovement.TransferControllers 7 | { 8 | using System; 9 | using System.Threading.Tasks; 10 | 11 | internal interface ITransferController 12 | { 13 | bool HasWork 14 | { 15 | get; 16 | } 17 | 18 | bool IsFinished 19 | { 20 | get; 21 | } 22 | 23 | Task DoWorkAsync(); 24 | 25 | void CancelWork(); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/packages.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /lib/TransferEnumerators/FileToAzureBlobNameResolver.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferEnumerators 8 | { 9 | /// 10 | /// Name resolver class for translating Windows file names to Azure blob/file names. 11 | /// 12 | internal class FileToAzureBlobNameResolver : INameResolver 13 | { 14 | public string ResolveName(TransferEntry sourceEntry) 15 | { 16 | return sourceEntry.RelativePath.Replace('\\', '/'); 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /lib/TestHook/TestHookCallbacks.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | 8 | namespace Microsoft.Azure.Storage.DataMovement 9 | { 10 | #if DEBUG 11 | public static class TestHookCallbacks 12 | { 13 | public static Action SetFileAttributesCallback; 14 | public static Func GetFileAttributesCallback; 15 | 16 | public static Action SetFilePermissionsCallback; 17 | public static Func GetFilePermissionsCallback; 18 | 19 | public static bool UnderTesting 20 | { 21 | get; 22 | set; 23 | } 24 | } 25 | #endif 26 | } 27 | -------------------------------------------------------------------------------- /samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/App.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopySource/IRangeBasedSourceHandler.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferControllers.ServiceSideSyncCopySource 8 | { 9 | using System.Collections.Generic; 10 | using System.Threading; 11 | using System.Threading.Tasks; 12 | 13 | interface IRangeBasedSourceHandler : ISourceHandler 14 | { 15 | Task> GetCopyRangesAsync(long startOffset, long length, CancellationToken cancellationToken); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /lib/TransferEnumerators/AzureFileToAzureNameResolver.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferEnumerators 8 | { 9 | /// 10 | /// Name resolver for translating Azure file names to Azure blob or Azure file names. 11 | /// 12 | class AzureFileToAzureNameResolver : INameResolver 13 | { 14 | public string ResolveName(TransferEntry sourceEntry) 15 | { 16 | return sourceEntry.RelativePath.TrimEnd(new char[] {' '}).Replace('\\', '/'); 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /test/DMLibTestCodeGen/App.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /lib/GlobalMemoryStatusNativeMethods.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace Microsoft.Azure.Storage.DataMovement 7 | { 8 | using Microsoft.Azure.Storage.DataMovement.Interop; 9 | 10 | internal class GlobalMemoryStatusNativeMethods 11 | { 12 | public GlobalMemoryStatusNativeMethods() 13 | { 14 | this.AvailablePhysicalMemory = CrossPlatformHelpers.GetAvailableMemory(); 15 | } 16 | 17 | public ulong AvailablePhysicalMemory 18 | { 19 | get; 20 | private set; 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /test/DMLibTest/Framework/DMLibWrapper.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTest 7 | { 8 | using System.Threading.Tasks; 9 | using Microsoft.Azure.Storage.DataMovement; 10 | using MS.Test.Common.MsTestLib; 11 | 12 | public abstract class DMLibWrapper 13 | { 14 | public Task DoTransfer(TransferItem item) 15 | { 16 | Test.Info("Do transfer: {0}", item.ToString()); 17 | return this.DoTransferImp(item); 18 | } 19 | 20 | protected abstract Task DoTransferImp(TransferItem item); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /netcore/DMLibTest/CloudBlobDirectoryExtensions.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Azure.Storage; 2 | using Microsoft.Azure.Storage.Blob; 3 | using System.Collections.Generic; 4 | 5 | namespace DMLibTest 6 | { 7 | public static class CloudBlobDirectoryExtensions 8 | { 9 | public static IEnumerable ListBlobs(this CloudBlobDirectory dir, bool useFlatBlobListing = false, BlobListingDetails blobListingDetails = BlobListingDetails.None, BlobRequestOptions options = null, OperationContext operationContext = null) 10 | { 11 | // this is no longer a lazy method: the maximum (5000) results will be returned at once 12 | return dir.ListBlobsSegmentedAsync(useFlatBlobListing, blobListingDetails, maxResults: null, currentToken: null, options: options, operationContext: operationContext).GetAwaiter().GetResult().Results; 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /lib/TransferEnumerators/TransferEnumeratorBase.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferEnumerators 8 | { 9 | /// 10 | /// Base class of dmlib internal transfer enumerators. It contains the common properties of transfer enumerators. 11 | /// 12 | internal abstract class TransferEnumeratorBase 13 | { 14 | public string SearchPattern 15 | { 16 | get; 17 | set; 18 | } 19 | 20 | public bool Recursive 21 | { 22 | get; 23 | set; 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /test/MsTestLib/Exceptions.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | using System; 7 | using System.Collections.Generic; 8 | using System.Linq; 9 | using System.Text; 10 | 11 | namespace MS.Test.Common.MsTestLib 12 | { 13 | public class TestPauseException : Exception 14 | { 15 | public TestPauseException() 16 | { 17 | } 18 | 19 | public TestPauseException(string message) 20 | : base(message) 21 | { 22 | } 23 | 24 | public TestPauseException(string message, Exception innerException) 25 | : base(message, innerException) 26 | { 27 | } 28 | 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /test/DMTestLib/MultiDirectionTestMethodAttribute.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTestCodeGen 7 | { 8 | using System; 9 | using System.Collections.Generic; 10 | 11 | public abstract class MultiDirectionTestMethodAttribute : Attribute 12 | { 13 | protected string[] Tags 14 | { 15 | get; 16 | private set; 17 | } 18 | 19 | protected MultiDirectionTestMethodAttribute(string[] tags = null) 20 | { 21 | this.Tags = tags ?? new string[0]; 22 | } 23 | 24 | internal abstract IEnumerable ExtractDirections(); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /test/DMLibTest/packages.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /test/DMLibTest_NetStandard/packages.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /netcore/Build.cmd: -------------------------------------------------------------------------------- 1 | 2 | set BuildMode=Debug 3 | if not {%1} == {} ( 4 | set BuildMode=%1 5 | ) 6 | 7 | set OriginalPath=%cd% 8 | set TestPath=%~dp0..\test 9 | "%ProgramFiles(x86)%\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\msbuild" %TestPath%\DMLibTest\DMLibTest.csproj /t:Rebuild /p:Configuration=%BuildMode% >NUL 10 | "%ProgramFiles(x86)%\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\msbuild" %TestPath%\DMLibTestCodeGen\DMLibTestCodeGen.csproj /t:Rebuild /p:Configuration=%BuildMode% >NUL 11 | %TestPath%\DMLibTestCodeGen\bin\Debug\DMLibTestCodeGen.exe %TestPath%\DMLibTest\bin\Debug\DMLibTest.dll %TestPath%\DMLibTest\Generated DNetCore 12 | 13 | dotnet restore -s https://www.nuget.org/api/v2/ %~dp0\DMLibTest 14 | cd %~dp0\Microsoft.Azure.Storage.DataMovement 15 | dotnet build -c %BuildMode% 16 | cd %~dp0\MsTestLib 17 | dotnet build -c %BuildMode% 18 | cd %~dp0\DMTestLib 19 | dotnet build -c %BuildMode% 20 | cd %~dp0\DMLibTest 21 | dotnet build -c %BuildMode% 22 | dotnet publish -c %BuildMode% 23 | cd %OriginalPath% 24 | 25 | -------------------------------------------------------------------------------- /lib/packages.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /lib/TransferEnumerators/AzureBlobToAzureBlobNameResolver.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferEnumerators 8 | { 9 | using System.Diagnostics; 10 | 11 | /// 12 | /// Name resolver class for translating Azure blob names to Azure blob names. 13 | /// 14 | internal class AzureBlobToAzureBlobNameResolver : INameResolver 15 | { 16 | public string ResolveName(TransferEntry sourceEntry) 17 | { 18 | AzureBlobEntry blobEntry = sourceEntry as AzureBlobEntry; 19 | Debug.Assert(blobEntry != null, "blobEntry"); 20 | 21 | return Utils.AppendSnapShotTimeToFileName(sourceEntry.RelativePath, blobEntry.Blob.SnapshotTime); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /test/DMLibTest/app.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopyDest/BlockBlobDestHandler.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | 8 | namespace Microsoft.Azure.Storage.DataMovement.TransferControllers.ServiceSideSyncCopyDest 9 | { 10 | using System; 11 | using System.Threading; 12 | using System.Threading.Tasks; 13 | internal class BlockBlobDestHandler : BlobDestHandler 14 | { 15 | public BlockBlobDestHandler(AzureBlobLocation destLocation, TransferJob transferJob) 16 | : base(destLocation, transferJob) 17 | { 18 | } 19 | 20 | protected override Task CreateDestinationAsync(long length, AccessCondition accessCondition, CancellationToken cancellationToken) 21 | { 22 | throw new NotImplementedException(); 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /netcore/MsTestLib/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("MSTestLib")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("MSTestLib")] 13 | [assembly: AssemblyCopyright("Copyright © 2015")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("af47b5ff-b528-4855-aa9f-d3b31169d6a5")] 24 | -------------------------------------------------------------------------------- /samples/S3ToAzureSample/S3ToAzureSample.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 2013 4 | VisualStudioVersion = 12.0.21005.1 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "S3ToAzureSample", "S3ToAzureSample\S3ToAzureSample.csproj", "{AE167D71-1A21-4561-8361-5C8384BC82E4}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Release|Any CPU = Release|Any CPU 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {AE167D71-1A21-4561-8361-5C8384BC82E4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 15 | {AE167D71-1A21-4561-8361-5C8384BC82E4}.Debug|Any CPU.Build.0 = Debug|Any CPU 16 | {AE167D71-1A21-4561-8361-5C8384BC82E4}.Release|Any CPU.ActiveCfg = Release|Any CPU 17 | {AE167D71-1A21-4561-8361-5C8384BC82E4}.Release|Any CPU.Build.0 = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(SolutionProperties) = preSolution 20 | HideSolutionNode = FALSE 21 | EndGlobalSection 22 | EndGlobal 23 | -------------------------------------------------------------------------------- /samples/DataMovementSamples/DataMovementSamples.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 2013 4 | VisualStudioVersion = 12.0.21005.1 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataMovementSamples", "DataMovementSamples\DataMovementSamples.csproj", "{6004824E-4A84-463E-9094-451B253470CC}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Release|Any CPU = Release|Any CPU 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {6004824E-4A84-463E-9094-451B253470CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 15 | {6004824E-4A84-463E-9094-451B253470CC}.Debug|Any CPU.Build.0 = Debug|Any CPU 16 | {6004824E-4A84-463E-9094-451B253470CC}.Release|Any CPU.ActiveCfg = Release|Any CPU 17 | {6004824E-4A84-463E-9094-451B253470CC}.Release|Any CPU.Build.0 = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(SolutionProperties) = preSolution 20 | HideSolutionNode = FALSE 21 | EndGlobalSection 22 | EndGlobal 23 | -------------------------------------------------------------------------------- /samples/DataMovementSamples/DataMovementSamples/packages.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /samples/DataMovementSamples/DataMovementSamples_k.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.26403.7 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DataMovementSamples", "netcore\DataMovementSamples\DataMovementSamples.csproj", "{0A2BDE2B-3FCC-41D6-9172-417367429188}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Release|Any CPU = Release|Any CPU 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {0A2BDE2B-3FCC-41D6-9172-417367429188}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 15 | {0A2BDE2B-3FCC-41D6-9172-417367429188}.Debug|Any CPU.Build.0 = Debug|Any CPU 16 | {0A2BDE2B-3FCC-41D6-9172-417367429188}.Release|Any CPU.ActiveCfg = Release|Any CPU 17 | {0A2BDE2B-3FCC-41D6-9172-417367429188}.Release|Any CPU.Build.0 = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(SolutionProperties) = preSolution 20 | HideSolutionNode = FALSE 21 | EndGlobalSection 22 | EndGlobal 23 | -------------------------------------------------------------------------------- /test/DMLibTest_NetStandard/app.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Microsoft Corporation 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /tools/nupkg/buildDebugNupkg.cmd: -------------------------------------------------------------------------------- 1 | pushd %~dp0 2 | rmdir /s /q .\workspace 3 | mkdir .\workspace 4 | copy .\Microsoft.Azure.Storage.DataMovement.nuspec .\workspace 5 | mkdir .\workspace\lib\net452 6 | mkdir .\workspace\lib\netstandard2.0 7 | pushd ..\.. 8 | del /q /f *.nupkg 9 | copy .\lib\bin\Debug\Microsoft.Azure.Storage.DataMovement.dll .\tools\nupkg\workspace\lib\net452 10 | copy .\lib\bin\Debug\Microsoft.Azure.Storage.DataMovement.pdb .\tools\nupkg\workspace\lib\net452 11 | copy .\lib\bin\Debug\Microsoft.Azure.Storage.DataMovement.XML .\tools\nupkg\workspace\lib\net452 12 | copy .\netcore\Microsoft.Azure.Storage.DataMovement\bin\Debug\netstandard2.0\Microsoft.Azure.Storage.DataMovement.dll .\tools\nupkg\workspace\lib\netstandard2.0 13 | copy .\netcore\Microsoft.Azure.Storage.DataMovement\bin\Debug\netstandard2.0\Microsoft.Azure.Storage.DataMovement.pdb .\tools\nupkg\workspace\lib\netstandard2.0 14 | copy .\netcore\Microsoft.Azure.Storage.DataMovement\bin\Debug\netstandard2.0\Microsoft.Azure.Storage.DataMovement.xml .\tools\nupkg\workspace\lib\netstandard2.0 15 | .\.nuget\nuget.exe pack .\tools\nupkg\workspace\Microsoft.Azure.Storage.DataMovement.nuspec 16 | popd 17 | rmdir /s /q .\workspace 18 | popd 19 | -------------------------------------------------------------------------------- /tools/nupkg/buildNupkg.cmd: -------------------------------------------------------------------------------- 1 | pushd %~dp0 2 | rmdir /s /q .\workspace 3 | mkdir .\workspace 4 | copy .\Microsoft.Azure.Storage.DataMovement.nuspec .\workspace 5 | mkdir .\workspace\lib\net452 6 | mkdir .\workspace\lib\netstandard2.0 7 | pushd ..\.. 8 | del /q /f *.nupkg 9 | copy .\lib\bin\Release\Microsoft.Azure.Storage.DataMovement.dll .\tools\nupkg\workspace\lib\net452 10 | copy .\lib\bin\Release\Microsoft.Azure.Storage.DataMovement.pdb .\tools\nupkg\workspace\lib\net452 11 | copy .\lib\bin\Release\Microsoft.Azure.Storage.DataMovement.XML .\tools\nupkg\workspace\lib\net452 12 | copy .\netcore\Microsoft.Azure.Storage.DataMovement\bin\Release\netstandard2.0\Microsoft.Azure.Storage.DataMovement.dll .\tools\nupkg\workspace\lib\netstandard2.0 13 | copy .\netcore\Microsoft.Azure.Storage.DataMovement\bin\Release\netstandard2.0\Microsoft.Azure.Storage.DataMovement.pdb .\tools\nupkg\workspace\lib\netstandard2.0 14 | copy .\netcore\Microsoft.Azure.Storage.DataMovement\bin\Release\netstandard2.0\Microsoft.Azure.Storage.DataMovement.xml .\tools\nupkg\workspace\lib\netstandard2.0 15 | .\.nuget\nuget.exe pack .\tools\nupkg\workspace\Microsoft.Azure.Storage.DataMovement.nuspec 16 | popd 17 | rmdir /s /q .\workspace 18 | popd 19 | -------------------------------------------------------------------------------- /lib/TransferEnumerators/ErrorEntry.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferEnumerators 8 | { 9 | /// 10 | /// Class inherit from TransferEntry to indicate transfer enumeration failures. 11 | /// 12 | internal class ErrorEntry : TransferEntry 13 | { 14 | /// 15 | /// Exception received during transfer enumeration. 16 | /// 17 | public readonly TransferException Exception; 18 | 19 | /// 20 | /// Initializes a new instance of the class. 21 | /// 22 | /// Exception to store. 23 | public ErrorEntry(TransferException ex) 24 | : base(null, null) 25 | { 26 | this.Exception = ex; 27 | } 28 | 29 | public override bool IsDirectory { get => false; } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /test/MsTestLib/MethodConfig.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | using System; 7 | using System.Collections.Generic; 8 | using System.Linq; 9 | using System.Text; 10 | 11 | namespace MS.Test.Common.MsTestLib 12 | { 13 | 14 | public class MethodConfig 15 | { 16 | public MethodConfig() 17 | { 18 | methodParams = new Dictionary(); 19 | } 20 | 21 | private Dictionary methodParams; 22 | 23 | public Dictionary MethodParams 24 | { 25 | get { return methodParams; } 26 | set { methodParams = value; } 27 | } 28 | 29 | public string this[string key] 30 | { 31 | get 32 | { 33 | return methodParams[key]; 34 | } 35 | 36 | set 37 | { 38 | methodParams[key] = value; 39 | } 40 | } 41 | } 42 | 43 | 44 | } 45 | -------------------------------------------------------------------------------- /lib/TransferJobs/TransferMethod.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement 8 | { 9 | internal enum TransferMethod 10 | { 11 | /// 12 | /// To read data from source to memory and then write the data in memory to destination. 13 | /// 14 | SyncCopy, 15 | 16 | /// 17 | /// To send a start copy request to azure storage to let it do the copying, 18 | /// and monitor the copying progress until the copy finished. 19 | /// 20 | ServiceSideAsyncCopy, 21 | 22 | /// 23 | /// To copy content of each chunk with with "Put Block From URL", "Append Block From URL" or "Put Page From URL". 24 | /// 25 | ServiceSideSyncCopy, 26 | 27 | /// 28 | /// Creates dummy objects only, no data transfer will happen. 29 | /// 30 | DummyCopy, 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /samples/S3ToAzureSample/S3ToAzureSample/packages.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples_k.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 16 4 | VisualStudioVersion = 16.0.29613.14 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PreserveFilePropertiesSamples", "netcore\PreserveFilePropertiesSamples.csproj", "{2FCFA585-A875-47B5-840D-DE8C8A42F3C8}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Release|Any CPU = Release|Any CPU 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {2FCFA585-A875-47B5-840D-DE8C8A42F3C8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 15 | {2FCFA585-A875-47B5-840D-DE8C8A42F3C8}.Debug|Any CPU.Build.0 = Debug|Any CPU 16 | {2FCFA585-A875-47B5-840D-DE8C8A42F3C8}.Release|Any CPU.ActiveCfg = Release|Any CPU 17 | {2FCFA585-A875-47B5-840D-DE8C8A42F3C8}.Release|Any CPU.Build.0 = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(SolutionProperties) = preSolution 20 | HideSolutionNode = FALSE 21 | EndGlobalSection 22 | GlobalSection(ExtensibilityGlobals) = postSolution 23 | SolutionGuid = {8E2AF0E8-AEC7-4F8E-8811-714510119944} 24 | EndGlobalSection 25 | EndGlobal 26 | -------------------------------------------------------------------------------- /test/MsTestLib/ILogger.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | using System; 7 | using System.Collections.Generic; 8 | using System.Linq; 9 | using System.Text; 10 | 11 | namespace MS.Test.Common.MsTestLib 12 | { 13 | public interface ILogger 14 | { 15 | void WriteError( 16 | string msg, 17 | params object[] objToLog); 18 | 19 | void WriteWarning( 20 | string msg, 21 | params object[] objToLog); 22 | 23 | void WriteInfo( 24 | string msg, 25 | params object[] objToLog); 26 | 27 | void WriteVerbose( 28 | string msg, 29 | params object[] objToLog); 30 | 31 | void StartTest( 32 | string testId); 33 | 34 | void EndTest( 35 | string testId, 36 | TestResult result); 37 | 38 | object GetLogger(); 39 | 40 | void Close(); 41 | 42 | } 43 | 44 | public enum TestResult 45 | { 46 | PASS, 47 | FAIL, 48 | SKIP 49 | } 50 | 51 | 52 | } 53 | -------------------------------------------------------------------------------- /tools/AssemblyInfo/SharedAssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | // 6 | // Assembly global configuration. 7 | // 8 | //------------------------------------------------------------------------------ 9 | 10 | using System; 11 | using System.Reflection; 12 | using System.Resources; 13 | using System.Runtime.InteropServices; 14 | 15 | [assembly: AssemblyVersion("2.0.5.0")] 16 | [assembly: AssemblyFileVersion("2.0.5.0")] 17 | 18 | [assembly: AssemblyCompany("Microsoft")] 19 | [assembly: AssemblyProduct("Microsoft Azure Storage")] 20 | [assembly: AssemblyCopyright("Copyright © 2024 Microsoft Corp.")] 21 | [assembly: AssemblyTrademark("Microsoft ® is a registered trademark of Microsoft Corporation.")] 22 | [assembly: AssemblyCulture("")] 23 | 24 | // Setting ComVisible to false makes the types in this assembly not visible 25 | // to COM components. If you need to access a type in this assembly from 26 | // COM, set the ComVisible attribute to true on that type. 27 | [assembly: ComVisible(false)] 28 | 29 | [assembly: NeutralResourcesLanguageAttribute("en-US")] 30 | 31 | [assembly: CLSCompliant(false)] 32 | -------------------------------------------------------------------------------- /samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 16 4 | VisualStudioVersion = 16.0.29613.14 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PreserveFilePropertiesSamples", "PreserveFilePropertiesSamples\PreserveFilePropertiesSamples.csproj", "{BEE26B27-C556-4811-B445-979C5395AA74}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Release|Any CPU = Release|Any CPU 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {BEE26B27-C556-4811-B445-979C5395AA74}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 15 | {BEE26B27-C556-4811-B445-979C5395AA74}.Debug|Any CPU.Build.0 = Debug|Any CPU 16 | {BEE26B27-C556-4811-B445-979C5395AA74}.Release|Any CPU.ActiveCfg = Release|Any CPU 17 | {BEE26B27-C556-4811-B445-979C5395AA74}.Release|Any CPU.Build.0 = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(SolutionProperties) = preSolution 20 | HideSolutionNode = FALSE 21 | EndGlobalSection 22 | GlobalSection(ExtensibilityGlobals) = postSolution 23 | SolutionGuid = {EA7938F8-3BA4-45E2-8B44-42938FB55E22} 24 | EndGlobalSection 25 | EndGlobal 26 | -------------------------------------------------------------------------------- /test/DMLibTest/Framework/TestResult.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTest 7 | { 8 | using System; 9 | using System.Collections; 10 | using System.Collections.Generic; 11 | 12 | public class TestResult where TDataInfo : IDataInfo 13 | { 14 | private List exceptions; 15 | 16 | public TestResult() 17 | { 18 | this.exceptions = new List(); 19 | this.DataInfo = default(TDataInfo); 20 | } 21 | 22 | public TDataInfo DataInfo 23 | { 24 | set; 25 | get; 26 | } 27 | 28 | public List TransferItems 29 | { 30 | set; 31 | get; 32 | } 33 | 34 | public List Exceptions 35 | { 36 | get 37 | { 38 | return this.exceptions; 39 | } 40 | } 41 | 42 | public void AddException(Exception e) 43 | { 44 | this.exceptions.Add(e); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /tools/scripts/InjectBuildNumber.ps1: -------------------------------------------------------------------------------- 1 | Function UpdateVersionInFile 2 | { 3 | Param ([string]$path, [string]$prefix, [string]$suffix, [int]$verNum) 4 | 5 | if ($env:BUILD_NUMBER) 6 | { 7 | 8 | $lines = Get-Content $path -Encoding UTF8 9 | 10 | $new_lines = $lines | %{ 11 | if ($_.StartsWith($prefix)) 12 | { 13 | $num = $_.Substring($prefix.Length, $_.Length - $prefix.Length - $suffix.Length) 14 | $num_p = $num.Split('.') 15 | $new_num = [System.String]::Join('.', $num_p[0 .. ($verNum-2)] + $env:BUILD_NUMBER) 16 | return $prefix + $new_num + $suffix 17 | } 18 | else 19 | { 20 | return $_ 21 | } 22 | } 23 | 24 | Set-Content -Path $path -Value $new_lines -Encoding UTF8 25 | } 26 | } 27 | 28 | UpdateVersionInFile ((Split-Path -Parent $PSCommandPath) + '\..\nupkg\Microsoft.Azure.Storage.DataMovement.nuspec') ' ' '' 4 29 | 30 | UpdateVersionInFile ((Split-Path -Parent $PSCommandPath) + '\..\AssemblyInfo\SharedAssemblyInfo.cs') '[assembly: AssemblyFileVersion("' '")]' 4 31 | 32 | UpdateVersionInFile ((Split-Path -Parent $PSCommandPath) + '\..\..\netcore\Microsoft.Azure.Storage.DataMovement\Microsoft.Azure.Storage.DataMovement.csproj') ' ' '' 4 -------------------------------------------------------------------------------- /netcore/DMLibTest/CloudAppendBlobExtensions.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Azure.Storage; 2 | using Microsoft.Azure.Storage.Blob; 3 | using System.IO; 4 | 5 | namespace DMLibTest 6 | { 7 | public static class CloudAppendBlobExtensions 8 | { 9 | public static void UploadFromByteArray(this CloudAppendBlob cloudBlob, byte[] buffer, int index, int count, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null) 10 | { 11 | cloudBlob.UploadFromByteArrayAsync(buffer, index, count, accessCondition, options, operationContext).GetAwaiter().GetResult(); 12 | } 13 | 14 | public static void UploadFromFile(this CloudAppendBlob cloudBlob, string path, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null) 15 | { 16 | cloudBlob.UploadFromFileAsync(path, accessCondition, options, operationContext).GetAwaiter().GetResult(); 17 | } 18 | 19 | public static void UploadFromStream(this CloudAppendBlob cloudBlob, Stream source, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null) 20 | { 21 | cloudBlob.UploadFromStreamAsync(source, accessCondition, options, operationContext).GetAwaiter().GetResult(); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /test/DMLibTest/Framework/FileNativeMethodsExtension.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.Interop 8 | { 9 | using System; 10 | using System.Collections.Generic; 11 | using System.IO; 12 | using System.Linq; 13 | using System.Runtime.InteropServices; 14 | using System.Security; 15 | using System.Text; 16 | using System.Threading; 17 | using System.Threading.Tasks; 18 | using Microsoft.Win32.SafeHandles; 19 | 20 | internal static partial class NativeMethods 21 | { 22 | [DllImport("kernel32.dll", EntryPoint = "RemoveDirectoryW", SetLastError = true, CharSet = CharSet.Unicode)] 23 | [return: MarshalAs(UnmanagedType.Bool)] 24 | public static extern bool RemoveDirectoryW([MarshalAs(UnmanagedType.LPWStr)]string lpPathName); 25 | 26 | [DllImport("kernel32.dll", EntryPoint = "DeleteFileW", SetLastError = true, CharSet = CharSet.Unicode)] 27 | [return: MarshalAs(UnmanagedType.Bool)] 28 | public static extern bool DeleteFileW([MarshalAs(UnmanagedType.LPWStr)]string lpFileName); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /lib/TransferEnumerators/ITransferEnumerator.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferEnumerators 8 | { 9 | using System.Collections.Generic; 10 | using System.Threading; 11 | 12 | /// 13 | /// Transfer enumerator interface. 14 | /// 15 | internal interface ITransferEnumerator 16 | { 17 | /// 18 | /// Gets or sets the file enumerate continulation token. 19 | /// 20 | ListContinuationToken EnumerateContinuationToken 21 | { 22 | get; 23 | set; 24 | } 25 | 26 | /// 27 | /// Enumerates the files in the transfer location referenced by this object. 28 | /// 29 | /// CancellationToken to notify the method cancellation. 30 | /// Enumerable list of TransferEntry objects found in the storage location referenced by this object. 31 | IEnumerable EnumerateLocation(CancellationToken cancellationToken); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /test/DMTestLib/TestMethodDirection.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTestCodeGen 7 | { 8 | using System.Collections.Generic; 9 | 10 | public abstract class TestMethodDirection 11 | { 12 | public List Tags 13 | { 14 | get; 15 | private set; 16 | } 17 | 18 | public TestMethodDirection(List tags) 19 | { 20 | if (null != tags) 21 | { 22 | this.Tags = new List(tags); 23 | } 24 | else 25 | { 26 | this.Tags = new List(); 27 | } 28 | } 29 | 30 | public abstract string GetTestMethodNameSuffix(); 31 | 32 | protected abstract IEnumerable GetExtraTags(); 33 | 34 | public IEnumerable GetTags() 35 | { 36 | foreach(var tag in Tags) 37 | { 38 | yield return tag; 39 | } 40 | 41 | foreach (var extraTag in this.GetExtraTags()) 42 | { 43 | yield return extraTag; 44 | } 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /lib/TransferStatusHelpers/TransferData.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement 8 | { 9 | using System.IO; 10 | 11 | internal class TransferData : TransferDataState 12 | { 13 | private MemoryManager memoryManager; 14 | 15 | public TransferData(MemoryManager memoryManager) 16 | { 17 | this.memoryManager = memoryManager; 18 | } 19 | 20 | public Stream Stream 21 | { 22 | get; 23 | set; 24 | } 25 | 26 | protected override void Dispose(bool disposing) 27 | { 28 | if (disposing) 29 | { 30 | if (null != this.Stream) 31 | { 32 | #if DOTNET5_4 33 | this.Stream.Dispose(); 34 | #else 35 | this.Stream.Close(); 36 | #endif 37 | this.Stream = null; 38 | } 39 | 40 | if (null != this.MemoryBuffer) 41 | { 42 | this.memoryManager.ReleaseBuffers(this.MemoryBuffer); 43 | this.MemoryBuffer = null; 44 | } 45 | } 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /netcore/DMLibTest/XunitWrapper/Attributes.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Reflection; 3 | using Xunit; 4 | using Xunit.Sdk; 5 | 6 | namespace Microsoft.VisualStudio.TestTools.UnitTesting 7 | { 8 | [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)] 9 | [TraitDiscoverer("TestCategoryDiscoverer", "MsTestWrapper")] 10 | public class TestCategoryAttribute : Attribute, ITraitAttribute 11 | { 12 | public TestCategoryAttribute(string category) 13 | { } 14 | } 15 | 16 | [AttributeUsage(AttributeTargets.Method)] 17 | public sealed class TestMethodAttribute : FactAttribute 18 | { } 19 | 20 | // Initialization and cleanup attributes do nothing because xUnit 21 | // handles these tasks through constructors and IDisposable: 22 | // http://xunit.github.io/docs/shared-context.html 23 | [AttributeUsage(AttributeTargets.Method)] 24 | public sealed class ClassCleanupAttribute : Attribute 25 | { } 26 | 27 | [AttributeUsage(AttributeTargets.Method)] 28 | public sealed class ClassInitializeAttribute : Attribute 29 | { } 30 | 31 | [AttributeUsage(AttributeTargets.Class)] 32 | public sealed class TestClassAttribute : Attribute 33 | { } 34 | 35 | [AttributeUsage(AttributeTargets.Method)] 36 | public sealed class TestCleanupAttribute : Attribute 37 | { } 38 | 39 | [AttributeUsage(AttributeTargets.Method)] 40 | public sealed class TestInitializeAttribute : Attribute 41 | { } 42 | } 43 | -------------------------------------------------------------------------------- /lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopyDest/AppendBlobDestHandler.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferControllers.ServiceSideSyncCopyDest 8 | { 9 | using System.Threading; 10 | using System.Threading.Tasks; 11 | using Microsoft.Azure.Storage.Blob; 12 | 13 | internal class AppendBlobDestHandler : BlobDestHandler 14 | { 15 | private CloudAppendBlob destAppendBlob; 16 | 17 | public AppendBlobDestHandler(AzureBlobLocation destLocation, TransferJob transferJob) 18 | : base(destLocation, transferJob) 19 | { 20 | this.destAppendBlob = destLocation.Blob as CloudAppendBlob; 21 | } 22 | 23 | protected override Task CreateDestinationAsync(long totalLength, AccessCondition accessCondition, CancellationToken cancellationToken) 24 | { 25 | return this.destAppendBlob.CreateOrReplaceAsync( 26 | accessCondition, 27 | Utils.GenerateBlobRequestOptions(this.DestLocation.BlobRequestOptions), 28 | Utils.GenerateOperationContext(this.TransferContext), 29 | cancellationToken); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopyDest/PageBlobDestHandler.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | 8 | namespace Microsoft.Azure.Storage.DataMovement.TransferControllers.ServiceSideSyncCopyDest 9 | { 10 | using System.Threading; 11 | using System.Threading.Tasks; 12 | using Microsoft.Azure.Storage.Blob; 13 | 14 | internal class PageBlobDestHandler : BlobDestHandler 15 | { 16 | private CloudPageBlob destPageBlob; 17 | 18 | public PageBlobDestHandler(AzureBlobLocation destLocation, TransferJob transferJob) 19 | : base(destLocation, transferJob) 20 | { 21 | this.destPageBlob = destLocation.Blob as CloudPageBlob; 22 | } 23 | 24 | protected override Task CreateDestinationAsync(long totalLength, AccessCondition accessCondition, CancellationToken cancellationToken) 25 | { 26 | return this.destPageBlob.CreateAsync( 27 | totalLength, 28 | accessCondition, 29 | Utils.GenerateBlobRequestOptions(this.DestLocation.BlobRequestOptions), 30 | Utils.GenerateOperationContext(this.TransferContext), 31 | cancellationToken); 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /test/DMLibTest/Framework/MultiDirectionTestInfo.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTest 7 | { 8 | using System.Collections.Generic; 9 | using MS.Test.Common.MsTestLib; 10 | 11 | public class MultiDirectionTestInfo 12 | { 13 | public static List GeneratedSourceDataInfos = new List(); 14 | public static List GeneratedDestDataInfos = new List(); 15 | 16 | public static void Cleanup() 17 | { 18 | MultiDirectionTestInfo.GeneratedSourceDataInfos.Clear(); 19 | MultiDirectionTestInfo.GeneratedDestDataInfos.Clear(); 20 | } 21 | 22 | public static void Print() 23 | { 24 | Test.Info("-----Source Data-----"); 25 | foreach (var sourceDataInfo in MultiDirectionTestInfo.GeneratedSourceDataInfos) 26 | { 27 | MultiDirectionTestHelper.PrintTransferDataInfo(sourceDataInfo); 28 | } 29 | 30 | Test.Info("-----Dest Data-----"); 31 | foreach (var destDataInfo in MultiDirectionTestInfo.GeneratedDestDataInfos) 32 | { 33 | MultiDirectionTestHelper.PrintTransferDataInfo(destDataInfo); 34 | } 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /test/DMTestLib/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("DMTestLib")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("DMTestLib")] 13 | [assembly: AssemblyCopyright("Copyright © 2015")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("f716d97f-1135-4ada-94f4-0ce946b2d4a8")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopyDest/IDestHandler.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferControllers.ServiceSideSyncCopyDest 8 | { 9 | using System; 10 | using System.Threading; 11 | using System.Threading.Tasks; 12 | 13 | /// 14 | /// Service side sync copy can copy between any two ends of append blob, block blob, page blob, and cloud file, 15 | /// which is a matrix with 16 directions. 16 | /// All copying shares similar logic with minor differences. 17 | /// Use SourceHandler and DestHandler to deal with minor differences, and let the main controller deal with the whole copying logic. 18 | /// 19 | internal interface IDestHandler 20 | { 21 | Uri Uri { get; } 22 | 23 | Task CheckAndCreateDestinationAsync( 24 | bool isForceOverwrite, 25 | long totalLength, 26 | Func checkOverwrite, 27 | CancellationToken cancellationToken); 28 | 29 | Task CommitAsync( 30 | bool gotDestAttributes, 31 | Attributes sourceAttributes, 32 | Func setCustomAttributes, 33 | CancellationToken cancellationToken); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /lib/CopyMethod.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | 8 | namespace Microsoft.Azure.Storage.DataMovement 9 | { 10 | /// 11 | /// Enum to indicate how the copying operation is handled in DataMovement Library. 12 | /// 13 | public enum CopyMethod 14 | { 15 | /// 16 | /// To download data from source to memory, and upload the data from memory to destination. 17 | /// 18 | SyncCopy, 19 | 20 | /// 21 | /// To send a start copy request to azure storage to let it do the copying, 22 | /// and monitor the copying progress until the copy completed. 23 | /// 24 | ServiceSideAsyncCopy, 25 | 26 | /// 27 | /// To copy content of each chunk with with Put Block From URL, Append Block From URL or Put Page From URL. 28 | /// See https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-from-url for Put Block From URL, 29 | /// https://docs.microsoft.com/en-us/rest/api/storageservices/append-block-from-url for Append Block From URL, 30 | /// https://docs.microsoft.com/en-us/rest/api/storageservices/put-page-from-url for Put Page From URL for details. 31 | /// 32 | ServiceSideSyncCopy 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /test/MsTestLib/ClassConfig.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | using System; 7 | using System.Collections.Generic; 8 | using System.Linq; 9 | using System.Text; 10 | 11 | namespace MS.Test.Common.MsTestLib 12 | { 13 | 14 | public class ClassConfig 15 | { 16 | public ClassConfig() 17 | { 18 | classParams = new Dictionary(); 19 | classMethods = new Dictionary(); 20 | } 21 | 22 | private Dictionary classParams; 23 | 24 | public Dictionary ClassParams 25 | { 26 | get { return classParams; } 27 | set { classParams = value; } 28 | } 29 | 30 | private Dictionary classMethods; 31 | 32 | public MethodConfig this[string methodName] 33 | { 34 | get 35 | { 36 | if (classMethods.ContainsKey(methodName)) 37 | { 38 | return classMethods[methodName]; 39 | } 40 | else 41 | { 42 | return null; 43 | } 44 | } 45 | 46 | set 47 | { 48 | classMethods[methodName] = value; 49 | } 50 | 51 | } 52 | 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /netcore/DMLibTest/TestData.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | true 4 | false 5 | DMLibTest.log 6 | true 7 | true 8 | true 9 | true 10 | DevFabric 11 | DefaultEndpointsProtocol=https;AccountName=testaccount1;AccountKey=FjUfNl1KiJttbXlsdkMzBTC7WagvrRM9/g6UPBuy0ypCpAbYTL6/KA+dI/7gyoWvLFYmah3IviUP1jykOHHOlA==;BlobEndpoint=http://127.0.0.1:10000/testaccount1;QueueEndpoint=http://127.0.0.1:10001/testaccount1;TableEndpoint=http://127.0.0.1:10002/testaccount1;FileEndpoint=http://127.0.0.1:10004/testaccount1 12 | DefaultEndpointsProtocol=https;AccountName=dmtestaccount1;AccountKey=FjUfNl1KiJttbXlsdkMzBTC7WagvrRM9/g6UPBuy0ypCpAbYTL6/KA+dI/7gyoWvLFYmah3IviUP1jykOHHOlA==;BlobEndpoint=http://127.0.0.1:10000/dmtestaccount1;QueueEndpoint=http://127.0.0.1:10001/dmtestaccount1;TableEndpoint=http://127.0.0.1:10002/dmtestaccount1;FileEndpoint=http://127.0.0.1:10004/dmtestaccount1 13 | testfile 14 | 1024 15 | testfolder 16 | \\127.0.0.1\AzCopy 17 | journal 18 | 200 19 | 20 20 | 3 21 | 3 22 | 23 | -------------------------------------------------------------------------------- /test/DMLibTest/TestData.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | true 4 | false 5 | DMLibTest.log 6 | true 7 | true 8 | true 9 | true 10 | DevFabric 11 | DefaultEndpointsProtocol=https;AccountName=testaccount1;AccountKey=FjUfNl1KiJttbXlsdkMzBTC7WagvrRM9/g6UPBuy0ypCpAbYTL6/KA+dI/7gyoWvLFYmah3IviUP1jykOHHOlA==;BlobEndpoint=http://127.0.0.1:10000/testaccount1;QueueEndpoint=http://127.0.0.1:10001/testaccount1;TableEndpoint=http://127.0.0.1:10002/testaccount1;FileEndpoint=http://127.0.0.1:10004/testaccount1 12 | DefaultEndpointsProtocol=https;AccountName=dmtestaccount1;AccountKey=FjUfNl1KiJttbXlsdkMzBTC7WagvrRM9/g6UPBuy0ypCpAbYTL6/KA+dI/7gyoWvLFYmah3IviUP1jykOHHOlA==;BlobEndpoint=http://127.0.0.1:10000/dmtestaccount1;QueueEndpoint=http://127.0.0.1:10001/dmtestaccount1;TableEndpoint=http://127.0.0.1:10002/dmtestaccount1;FileEndpoint=http://127.0.0.1:10004/dmtestaccount1 13 | testfile 14 | 1024 15 | testfolder 16 | \\127.0.0.1\AzCopy 17 | journal 18 | 200 19 | 20 20 | 3 21 | 3 22 | 23 | -------------------------------------------------------------------------------- /lib/TransferEnumerators/FileToAzureFileNameResolver.cs: -------------------------------------------------------------------------------- 1 | //----------------------------------------------------------------------------- 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //----------------------------------------------------------------------------- 6 | namespace Microsoft.Azure.Storage.DataMovement.TransferEnumerators 7 | { 8 | using System.Globalization; 9 | using Microsoft.Azure.Storage.DataMovement.Interop; 10 | 11 | internal class FileToAzureFileNameResolver : INameResolver 12 | { 13 | private static char[] invalidPathChars = new char[] { '"', '\\', ':', '|', '<', '>', '*', '?' }; 14 | 15 | protected static string EscapeInvalidCharacters(string fileName) 16 | { 17 | // Replace invalid characters with %HH, with HH being the hexadecimal 18 | // representation of the invalid character. 19 | foreach (char c in invalidPathChars) 20 | { 21 | fileName = fileName.Replace(c.ToString(), string.Format(CultureInfo.InvariantCulture, "%{0:X2}", (int)c)); 22 | } 23 | 24 | return fileName; 25 | } 26 | 27 | public string ResolveName(TransferEntry sourceEntry) 28 | { 29 | if (CrossPlatformHelpers.IsWindows) 30 | { 31 | return sourceEntry.RelativePath.Replace('\\', '/'); 32 | } 33 | else 34 | { 35 | return EscapeInvalidCharacters(sourceEntry.RelativePath); 36 | } 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("PreserveFilePropertiesSamples1")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("PreserveFilePropertiesSamples1")] 13 | [assembly: AssemblyCopyright("Copyright © 2020")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("bee26b27-c556-4811-b445-979c5395aa74")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /lib/TransferEnumerators/FileEntry.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferEnumerators 8 | { 9 | using System; 10 | 11 | /// 12 | /// FileEntry class to represent a single transfer entry on file system. 13 | /// 14 | internal class FileEntry : TransferEntry 15 | { 16 | /// 17 | /// Initializes a new instance of the class. 18 | /// 19 | /// Relative path of the file indicated by this file entry. 20 | /// Full path of the file indicated by this file entry. 21 | /// Continuation token when listing to this entry. 22 | public FileEntry(string relativePath, string fullPath, FileListContinuationToken continuationToken) 23 | : base(relativePath, continuationToken) 24 | { 25 | this.FullPath = fullPath; 26 | } 27 | 28 | /// 29 | /// Gets the full path of the file. 30 | /// 31 | public string FullPath 32 | { 33 | get; 34 | private set; 35 | } 36 | 37 | public override bool IsDirectory { get => false; } 38 | 39 | public override string ToString() 40 | { 41 | return FullPath; 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /test/DMTestLib/DMLibDataType.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTestCodeGen 7 | { 8 | using System; 9 | using System.Collections.Generic; 10 | 11 | [Flags] 12 | public enum DMLibDataType : int 13 | { 14 | Unspecified = 0x0, 15 | Stream = 0x01, 16 | URI = 0x02, 17 | Local = 0x04, 18 | CloudFile = 0x08, 19 | BlockBlob = 0x10, 20 | PageBlob = 0x20, 21 | AppendBlob = 0x40, 22 | 23 | CloudBlob = PageBlob | BlockBlob | AppendBlob, 24 | Cloud = CloudBlob | CloudFile, 25 | All = Local | Cloud, 26 | } 27 | 28 | internal static class DMLibDataTypeExtentions 29 | { 30 | public static IEnumerable Extract(this DMLibDataType type) 31 | { 32 | DMLibDataType[] dataTypesToExtract = 33 | { 34 | DMLibDataType.Stream, 35 | DMLibDataType.URI, 36 | DMLibDataType.Local, 37 | DMLibDataType.CloudFile, 38 | DMLibDataType.BlockBlob, 39 | DMLibDataType.PageBlob, 40 | DMLibDataType.AppendBlob, 41 | }; 42 | 43 | foreach (var dataType in dataTypesToExtract) 44 | { 45 | if (type.HasFlag(dataType)) 46 | { 47 | yield return dataType; 48 | } 49 | } 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /lib/TransferEnumerators/DirectoryEntry.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferEnumerators 8 | { 9 | using System; 10 | 11 | /// 12 | /// FileEntry class to represent a single transfer entry on file system. 13 | /// 14 | internal class DirectoryEntry : TransferEntry 15 | { 16 | /// 17 | /// Initializes a new instance of the class. 18 | /// 19 | /// Relative path of the file indicated by this file entry. 20 | /// Full path of the file indicated by this file entry. 21 | /// Continuation token when listing to this entry. 22 | public DirectoryEntry(string relativePath, string fullPath, FileListContinuationToken continuationToken) 23 | : base(relativePath, continuationToken) 24 | { 25 | this.FullPath = fullPath; 26 | } 27 | 28 | /// 29 | /// Gets the full path of the file. 30 | /// 31 | public string FullPath 32 | { 33 | get; 34 | private set; 35 | } 36 | 37 | public override bool IsDirectory { get => true; } 38 | 39 | public override string ToString() 40 | { 41 | return FullPath; 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /lib/LongPathFile.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement 8 | { 9 | using System; 10 | using System.IO; 11 | using Microsoft.Azure.Storage.DataMovement.Interop; 12 | 13 | internal static partial class LongPathFile 14 | { 15 | public static bool Exists(string path) 16 | { 17 | #if DOTNET5_4 18 | return File.Exists(path); 19 | #else 20 | try 21 | { 22 | if (String.IsNullOrEmpty(path)) 23 | { 24 | return false; 25 | } 26 | bool success = NativeMethods.PathFileExistsW(path); 27 | if (!success) 28 | { 29 | NativeMethods.ThrowExceptionForLastWin32ErrorIfExists(new int[] { 0, NativeMethods.ERROR_DIRECTORY_NOT_FOUND, NativeMethods.ERROR_FILE_NOT_FOUND }); 30 | } 31 | var fileAttributes = GetAttributes(path); 32 | return success && (FileAttributes.Directory != (fileAttributes & FileAttributes.Directory)); 33 | } 34 | catch (ArgumentException) { } 35 | catch (NotSupportedException) { } // Security can throw this on ":" 36 | catch (System.Security.SecurityException) { } 37 | catch (IOException) { } 38 | catch (UnauthorizedAccessException) { } 39 | 40 | return false; 41 | #endif 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /lib/TransferEnumerators/AzureBlobEntry.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferEnumerators 8 | { 9 | using Microsoft.Azure.Storage.Blob; 10 | 11 | /// 12 | /// AzureBlobEntry class to represent a single transfer entry on Azure blob service. 13 | /// 14 | internal class AzureBlobEntry : TransferEntry 15 | { 16 | /// 17 | /// Initializes a new instance of the class. 18 | /// 19 | /// Relative path of the blob indicated by this blob entry. 20 | /// Corresponding CloudBlob. 21 | /// Continuation token when listing to this entry. 22 | public AzureBlobEntry(string relativePath, CloudBlob cloudBlob, AzureBlobListContinuationToken continuationToken) 23 | : base(relativePath, continuationToken) 24 | { 25 | this.Blob = cloudBlob; 26 | } 27 | 28 | /// 29 | /// Gets the reference to the blob. 30 | /// 31 | public CloudBlob Blob 32 | { 33 | get; 34 | private set; 35 | } 36 | 37 | public override string ToString() 38 | { 39 | return Blob.ConvertToString(); 40 | } 41 | 42 | public override bool IsDirectory { get => false; } 43 | } 44 | } -------------------------------------------------------------------------------- /test/DMLibTest/Framework/DMLibInputHelper.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTest 7 | { 8 | using System; 9 | using System.Threading; 10 | using System.Threading.Tasks; 11 | using Microsoft.Azure.Storage.DataMovement; 12 | using MS.Test.Common.MsTestLib; 13 | 14 | public static class DMLibInputHelper 15 | { 16 | public static ShouldOverwriteCallbackAsync GetDefaultOverwiteCallbackY() 17 | { 18 | return async (source, dest) => 19 | { 20 | return await Task.Run(() => 21 | { 22 | Thread.Sleep((new Random()).Next(100, 500)); 23 | Test.Info("Overwrite true: {0} -> {1}", DMLibTestHelper.TransferInstanceToString(source), DMLibTestHelper.TransferInstanceToString(dest)); 24 | return true; 25 | }); 26 | }; 27 | } 28 | 29 | public static ShouldOverwriteCallbackAsync GetDefaultOverwiteCallbackN() 30 | { 31 | return async (source, dest) => 32 | { 33 | return await Task.Run(() => 34 | { 35 | Thread.Sleep((new Random()).Next(100, 500)); 36 | Test.Info("Overwrite false: {0} -> {1}", DMLibTestHelper.TransferInstanceToString(source), DMLibTestHelper.TransferInstanceToString(dest)); 37 | return false; 38 | }); 39 | }; 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /lib/TransferEnumerators/TransferEntry.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferEnumerators 8 | { 9 | using System; 10 | 11 | /// 12 | /// Base class of transfer entries. 13 | /// 14 | abstract class TransferEntry 15 | { 16 | /// 17 | /// Initializes a new instance of the class. 18 | /// 19 | /// Relative path of the file indicated by this entry. 20 | /// Continuation token when listing to this entry. 21 | public TransferEntry(string relativePath, ListContinuationToken continuationToken) 22 | { 23 | this.RelativePath = relativePath; 24 | this.ContinuationToken = continuationToken; 25 | } 26 | 27 | /// 28 | /// Gets the relative path of the file indicated by this transfer entry. 29 | /// 30 | public string RelativePath 31 | { 32 | get; 33 | private set; 34 | } 35 | 36 | /// 37 | /// Continuation token when list to this file entry. 38 | /// 39 | public ListContinuationToken ContinuationToken 40 | { 41 | get; 42 | private set; 43 | } 44 | 45 | abstract public bool IsDirectory 46 | { 47 | get; 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /test/DMLibTest/Framework/AssemblyInitCleanup.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTest 7 | { 8 | using System; 9 | using Microsoft.VisualStudio.TestTools.UnitTesting; 10 | using MS.Test.Common.MsTestLib; 11 | 12 | #if DNXCORE50 13 | public class AssemblyInitCleanup : IDisposable 14 | { 15 | public AssemblyInitCleanup() 16 | { 17 | Test.Init(null); // use default config file name (TestData.xml, expected next to DMLibTest.xproj) 18 | Test.AssertFail = new AssertFailDelegate(Assert.Fail); 19 | } 20 | 21 | public void Dispose() 22 | { 23 | Test.Close(); 24 | } 25 | } 26 | 27 | [Xunit.CollectionDefinition(Collections.Global)] 28 | public class AssemblyCollection : Xunit.ICollectionFixture 29 | { } 30 | #else 31 | [TestClass] 32 | public class AssemblyInitCleanup 33 | { 34 | [AssemblyInitialize] 35 | public static void TestInit(TestContext testContext) 36 | { 37 | // init loggers and load test config data 38 | String config = testContext.Properties["config"] as string; 39 | Test.Init(config); 40 | // set the assertfail delegate to report failure in VS 41 | Test.AssertFail = new AssertFailDelegate(Assert.Fail); 42 | } 43 | 44 | [AssemblyCleanup] 45 | public static void TestCleanup() 46 | { 47 | Test.Close(); 48 | } 49 | } 50 | #endif 51 | } 52 | -------------------------------------------------------------------------------- /lib/TransferJobs/TransferJobStatus.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace Microsoft.Azure.Storage.DataMovement 7 | { 8 | /// 9 | /// Status for TransferEntry. 10 | /// NotStarted -> Skipped 11 | /// -> Transfer -> [Monitor ->] Finished. 12 | /// Failed. 13 | /// 14 | internal enum TransferJobStatus 15 | { 16 | /// 17 | /// Transfer is not started. 18 | /// 19 | NotStarted, 20 | 21 | /// 22 | /// The transfer should not be done because ShouldTransferCallbackAsync returned false on the transfer job. 23 | /// This is only used when the transfer instance is created from a directory transfer. 24 | /// 25 | SkippedDueToShouldNotTransfer, 26 | 27 | /// 28 | /// Transfer is skipped because ShouldOverwriteCallbackAsync returned false on the transfer job. 29 | /// 30 | Skipped, 31 | 32 | /// 33 | /// Transfer file. 34 | /// 35 | Transfer, 36 | 37 | /// 38 | /// Monitor transfer process. 39 | /// 40 | Monitor, 41 | 42 | /// 43 | /// Transfer is finished successfully. 44 | /// 45 | Finished, 46 | 47 | /// 48 | /// Transfer is failed. 49 | /// 50 | Failed, 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /samples/PreserveFilePropertiesSamples/netcore/PreserveFilePropertiesSamples.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp3.1 6 | 7 | 8 | 9 | TRACE;DOTNET5_4 10 | 11 | 12 | 13 | TRACE;DOTNET5_4 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | PreserveNewest 33 | 34 | 35 | PreserveNewest 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopySource/ISourceHandler.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferControllers.ServiceSideSyncCopySource 8 | { 9 | using System; 10 | using System.IO; 11 | using System.Threading; 12 | using System.Threading.Tasks; 13 | 14 | /// 15 | /// Service side sync copy can copy between any two ends of append blob, block blob, page blob, and cloud file, 16 | /// which is a matrix with 16 directions. 17 | /// All copying shares similar logic with minor differences. 18 | /// Use ISourceHandler and IDestHandler to deal with minor differences, and let main controller deal with the whole copying logic. 19 | /// 20 | internal interface ISourceHandler 21 | { 22 | Task FetchAttributesAsync(CancellationToken cancellationToken); 23 | 24 | Task DownloadRangeToStreamAsync(Stream stream, 25 | long startOffset, 26 | long length, 27 | AccessCondition accessCondition, 28 | bool useTransactionalMD5, 29 | OperationContext operationContext, 30 | CancellationToken cancellationToken); 31 | 32 | Uri GetCopySourceUri(); 33 | 34 | Uri Uri { get; } 35 | 36 | Attributes SourceAttributes { get; } 37 | long TotalLength { get; } 38 | 39 | string ETag { get; } 40 | 41 | AccessCondition AccessCondition { get; } 42 | 43 | bool NeedToCheckAccessCondition { get; } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /lib/TransferEnumerators/AzureFileEntry.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferEnumerators 8 | { 9 | using System; 10 | using Microsoft.Azure.Storage.DataMovement; 11 | using Microsoft.Azure.Storage.File; 12 | 13 | /// 14 | /// AzureFileEntry class to represent a single transfer entry on Azure file service. 15 | /// 16 | internal class AzureFileEntry : TransferEntry 17 | { 18 | /// 19 | /// Initializes a new instance of the class. 20 | /// 21 | /// Relative path of the file indicated by this file entry. 22 | /// Corresponding CloudFile. 23 | /// Continuation token when listing to this entry. 24 | public AzureFileEntry(string relativePath, CloudFile cloudFile, AzureFileListContinuationToken continuationToken) 25 | : base(relativePath, continuationToken) 26 | { 27 | this.File = cloudFile; 28 | } 29 | 30 | /// 31 | /// Gets the reference to the cloud file. 32 | /// 33 | public CloudFile File 34 | { 35 | get; 36 | private set; 37 | } 38 | 39 | public override bool IsDirectory { get => false; } 40 | 41 | public override string ToString() 42 | { 43 | return File.ConvertToString(); 44 | } 45 | } 46 | } -------------------------------------------------------------------------------- /lib/SingleTransferContext.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement 8 | { 9 | using System.IO; 10 | 11 | /// 12 | /// Represents the context for a single transfer, and provides additional runtime information about its execution. 13 | /// 14 | public class SingleTransferContext : TransferContext 15 | { 16 | /// 17 | /// Initializes a new instance of the class. 18 | /// 19 | public SingleTransferContext() 20 | : base() 21 | { 22 | } 23 | 24 | /// 25 | /// Initializes a new instance of the class. 26 | /// 27 | /// An object representing the last checkpoint from which the transfer continues on. 28 | public SingleTransferContext(TransferCheckpoint checkpoint) 29 | :base(checkpoint) 30 | { 31 | } 32 | 33 | /// 34 | /// Initializes a new instance of the class. 35 | /// 36 | /// The stream into which the transfer journal info will be written into. 37 | /// It can resume the previours paused transfer from its journal stream. 38 | public SingleTransferContext(Stream journalStream) 39 | :base(journalStream) 40 | { 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /samples/S3ToAzureSample/S3ToAzureSample/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | using System.Reflection; 7 | using System.Runtime.CompilerServices; 8 | using System.Runtime.InteropServices; 9 | 10 | // General Information about an assembly is controlled through the following 11 | // set of attributes. Change these attribute values to modify the information 12 | // associated with an assembly. 13 | [assembly: AssemblyTitle("S3ToAzureSample")] 14 | [assembly: AssemblyDescription("")] 15 | [assembly: AssemblyConfiguration("")] 16 | [assembly: AssemblyCompany("")] 17 | [assembly: AssemblyProduct("S3ToAzureSample")] 18 | [assembly: AssemblyCopyright("Copyright © 2015")] 19 | [assembly: AssemblyTrademark("")] 20 | [assembly: AssemblyCulture("")] 21 | 22 | // Setting ComVisible to false makes the types in this assembly not visible 23 | // to COM components. If you need to access a type in this assembly from 24 | // COM, set the ComVisible attribute to true on that type. 25 | [assembly: ComVisible(false)] 26 | 27 | // The following GUID is for the ID of the typelib if this project is exposed to COM 28 | [assembly: Guid("95debd45-95e0-4560-9287-547550248dc9")] 29 | 30 | // Version information for an assembly consists of the following four values: 31 | // 32 | // Major Version 33 | // Minor Version 34 | // Build Number 35 | // Revision 36 | // 37 | // You can specify all the values or you can default the Build and Revision Numbers 38 | // by using the '*' as shown below: 39 | // [assembly: AssemblyVersion("1.0.*")] 40 | [assembly: AssemblyVersion("1.0.0.0")] 41 | [assembly: AssemblyFileVersion("1.0.0.0")] 42 | -------------------------------------------------------------------------------- /lib/TransferCallbacks.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace Microsoft.Azure.Storage.DataMovement 7 | { 8 | using System; 9 | using System.Threading.Tasks; 10 | 11 | /// 12 | /// Callback invoked to tell whether to overwrite an existing destination. 13 | /// 14 | /// Instance of source used to overwrite the destination. 15 | /// Instance of destination to be overwritten. 16 | /// True if the file should be overwritten; otherwise false. 17 | public delegate Task ShouldOverwriteCallbackAsync( 18 | object source, 19 | object destination); 20 | 21 | /// 22 | /// Callback invoked to tell whether a transfer should be done. 23 | /// 24 | /// Instance of the transfer source. 25 | /// Instance of the transfer destination. 26 | /// True if the transfer should be done; otherwise false. 27 | public delegate Task ShouldTransferCallbackAsync( 28 | object source, 29 | object destination); 30 | 31 | /// 32 | /// Callback invoked to set destination's attributes in memory. 33 | /// The attributes set in this callback will be sent to azure storage service. 34 | /// 35 | /// Source instance in the transfer. 36 | /// Instance of destination to be overwritten. 37 | public delegate Task SetAttributesCallbackAsync(object source, object destination); 38 | } 39 | -------------------------------------------------------------------------------- /samples/DataMovementSamples/DataMovementSamples/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | using System.Reflection; 8 | using System.Runtime.CompilerServices; 9 | using System.Runtime.InteropServices; 10 | 11 | // General Information about an assembly is controlled through the following 12 | // set of attributes. Change these attribute values to modify the information 13 | // associated with an assembly. 14 | [assembly: AssemblyTitle("DataMovementSamples")] 15 | [assembly: AssemblyDescription("")] 16 | [assembly: AssemblyConfiguration("")] 17 | [assembly: AssemblyCompany("")] 18 | [assembly: AssemblyProduct("DataMovementSamples")] 19 | [assembly: AssemblyCopyright("Copyright © 2015")] 20 | [assembly: AssemblyTrademark("")] 21 | [assembly: AssemblyCulture("")] 22 | 23 | // Setting ComVisible to false makes the types in this assembly not visible 24 | // to COM components. If you need to access a type in this assembly from 25 | // COM, set the ComVisible attribute to true on that type. 26 | [assembly: ComVisible(false)] 27 | 28 | // The following GUID is for the ID of the typelib if this project is exposed to COM 29 | [assembly: Guid("d4c65175-d41e-4b9e-a898-ff45d07ee90c")] 30 | 31 | // Version information for an assembly consists of the following four values: 32 | // 33 | // Major Version 34 | // Minor Version 35 | // Build Number 36 | // Revision 37 | // 38 | // You can specify all the values or you can default the Build and Revision Numbers 39 | // by using the '*' as shown below: 40 | // [assembly: AssemblyVersion("1.0.*")] 41 | [assembly: AssemblyVersion("1.0.0.0")] 42 | [assembly: AssemblyFileVersion("1.0.0.0")] 43 | -------------------------------------------------------------------------------- /lib/TransferEnumerators/NameResolverHelper.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferEnumerators 8 | { 9 | using System; 10 | using System.Globalization; 11 | using System.IO; 12 | 13 | /// 14 | /// Helper class for resolving file names 15 | /// 16 | internal static class NameResolverHelper 17 | { 18 | /// 19 | /// Append snapshot time to a file name. 20 | /// 21 | /// Original file name. 22 | /// Snapshot time to append. 23 | /// A file name with appended snapshot time. 24 | public static string AppendSnapShotTimeToFileName(string fileName, DateTimeOffset? snapshotTime) 25 | { 26 | string resultName = fileName; 27 | 28 | if (snapshotTime.HasValue) 29 | { 30 | string pathAndFileNameNoExt = Path.ChangeExtension(fileName, null); 31 | string extension = Path.GetExtension(fileName); 32 | string timeStamp = string.Format( 33 | CultureInfo.InvariantCulture, 34 | "{0:yyyy-MM-dd HHmmss fff}", 35 | snapshotTime.Value); 36 | 37 | resultName = string.Format( 38 | CultureInfo.InvariantCulture, 39 | "{0} ({1}){2}", 40 | pathAndFileNameNoExt, 41 | timeStamp, 42 | extension); 43 | } 44 | 45 | return resultName; 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /netcore/DMLibTest/XunitLogger.cs: -------------------------------------------------------------------------------- 1 | using MS.Test.Common.MsTestLib; 2 | using System; 3 | using System.Text; 4 | using Xunit.Abstractions; 5 | 6 | namespace DMLibTest 7 | { 8 | public class XunitLogger : ILogger 9 | { 10 | private readonly ITestOutputHelper output; 11 | 12 | public XunitLogger(ITestOutputHelper outputHelper) 13 | { 14 | output = outputHelper; 15 | } 16 | 17 | public void Close() 18 | { } 19 | 20 | public void EndTest(string testId, TestResult result) 21 | { 22 | output.WriteLine($"[END] Test: {testId} RESULT: {result.ToString()}"); 23 | } 24 | 25 | public object GetLogger() 26 | { 27 | return this; 28 | } 29 | 30 | public void StartTest(string testId) 31 | { 32 | output.WriteLine($"[START] Test: {testId}"); 33 | } 34 | 35 | public void WriteError(string msg, params object[] objToLog) 36 | { 37 | WriteMessage("ERROR", msg, objToLog); 38 | } 39 | 40 | public void WriteInfo(string msg, params object[] objToLog) 41 | { 42 | WriteMessage("INFO", msg, objToLog); 43 | } 44 | 45 | public void WriteVerbose(string msg, params object[] objToLog) 46 | { 47 | WriteMessage("VERB", msg, objToLog); 48 | } 49 | 50 | public void WriteWarning(string msg, params object[] objToLog) 51 | { 52 | WriteMessage("WARN", msg, objToLog); 53 | } 54 | 55 | private void WriteMessage(string prefix, string msg, params object[] objToLog) 56 | { 57 | DateTime dt = DateTime.Now; 58 | StringBuilder sBuilder = new StringBuilder($"[{prefix}][{dt.ToString()}.{dt.Millisecond}]"); 59 | sBuilder.Append(MessageBuilder.FormatString(msg, objToLog)); 60 | output.WriteLine(sBuilder.ToString()); 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /lib/TransferEnumerators/AzureFileDirectoryEntry.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferEnumerators 8 | { 9 | using System; 10 | using Microsoft.Azure.Storage.DataMovement; 11 | using Microsoft.Azure.Storage.File; 12 | 13 | /// 14 | /// AzureFileEntry class to represent an azure file directory entry on Azure file service. 15 | /// 16 | internal class AzureFileDirectoryEntry : TransferEntry 17 | { 18 | /// 19 | /// Initializes a new instance of the class. 20 | /// 21 | /// Relative path of the directory indicated by this directory entry. 22 | /// Corresponding CloudFileDirectory instance. 23 | /// Continuation token when listing to this entry. 24 | public AzureFileDirectoryEntry(string relativePath, CloudFileDirectory cloudFileDirectory, AzureFileListContinuationToken continuationToken) 25 | : base(relativePath, continuationToken) 26 | { 27 | this.FileDirectory = cloudFileDirectory; 28 | } 29 | 30 | /// 31 | /// Gets the reference to the cloud file directory. 32 | /// 33 | public CloudFileDirectory FileDirectory 34 | { 35 | get; 36 | private set; 37 | } 38 | 39 | public override bool IsDirectory { get => true; } 40 | 41 | public override string ToString() 42 | { 43 | return FileDirectory.ConvertToString(); 44 | } 45 | } 46 | } -------------------------------------------------------------------------------- /test/DMLibTest/Framework/SharedAccessPermissions.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTest 7 | { 8 | using System; 9 | using Microsoft.Azure.Storage.Blob; 10 | using Microsoft.Azure.Storage.File; 11 | 12 | [Flags] 13 | public enum SharedAccessPermissions 14 | { 15 | None = 0, 16 | Read = 1, 17 | Write = 2, 18 | Delete = 4, 19 | List = 8, 20 | Query = 16, 21 | Add = 32, 22 | Update = 64, 23 | } 24 | 25 | public static class SharedAccessPermissionsExtensions 26 | { 27 | public const SharedAccessPermissions LeastPermissionDest = SharedAccessPermissions.Write | SharedAccessPermissions.Read; 28 | public const SharedAccessPermissions LeastPermissionSource = SharedAccessPermissions.List | SharedAccessPermissions.Read; 29 | public const SharedAccessPermissions LeastPermissionSourceList = SharedAccessPermissions.List; 30 | 31 | public static SharedAccessBlobPermissions ToBlobPermissions(this SharedAccessPermissions sap) 32 | { 33 | return (SharedAccessBlobPermissions)Enum.Parse(typeof(SharedAccessBlobPermissions), sap.ToString()); 34 | } 35 | 36 | public static SharedAccessFilePermissions ToFilePermissions(this SharedAccessPermissions sap) 37 | { 38 | return (SharedAccessFilePermissions)Enum.Parse(typeof(SharedAccessFilePermissions), sap.ToString()); 39 | } 40 | 41 | public static SharedAccessPermissions ToCommonPermissions(this Enum specificPermissions) 42 | { 43 | return (SharedAccessPermissions)Enum.Parse(typeof(SharedAccessPermissions), specificPermissions.ToString()); 44 | } 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /samples/DataMovementSamples/netcore/DataMovementSamples/DataMovementSamples.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.0 5 | $(DefineConstants);DOTNET5_4 6 | portable 7 | DataMovementSamples 8 | Exe 9 | DataMovementSamples 10 | $(AssetTargetFallback);dnxcore50;portable-net45+win8 11 | false 12 | false 13 | false 14 | false 15 | false 16 | false 17 | false 18 | false 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | PreserveNewest 35 | 36 | 37 | PreserveNewest 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /test/DMLibTestCodeGen/TransferDirectionExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.CodeDom; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | 8 | namespace DMLibTestCodeGen 9 | { 10 | static class TransferDirectionExtensions 11 | { 12 | public static IEnumerable EnumerateUpdateContextStatements(DMLibTransferDirection transferDirection) 13 | { 14 | CodeFieldReferenceExpression sourceType = new CodeFieldReferenceExpression( 15 | new CodeTypeReferenceExpression(typeof(DMLibDataType)), 16 | transferDirection.SourceType.ToString()); 17 | CodeFieldReferenceExpression destType = new CodeFieldReferenceExpression( 18 | new CodeTypeReferenceExpression(typeof(DMLibDataType)), 19 | transferDirection.DestType.ToString()); 20 | CodeFieldReferenceExpression copyMethod = new CodeFieldReferenceExpression( 21 | new CodeTypeReferenceExpression(typeof(DMLibCopyMethod)), 22 | transferDirection.CopyMethod.ToString()); 23 | 24 | CodePropertyReferenceExpression sourceTypeProperty = new CodePropertyReferenceExpression( 25 | new CodeTypeReferenceExpression(typeof(DMLibTestContext)), 26 | "SourceType"); 27 | 28 | CodePropertyReferenceExpression destTypeProperty = new CodePropertyReferenceExpression( 29 | new CodeTypeReferenceExpression(typeof(DMLibTestContext)), 30 | "DestType"); 31 | 32 | CodePropertyReferenceExpression copyMethodProperty = new CodePropertyReferenceExpression( 33 | new CodeTypeReferenceExpression(typeof(DMLibTestContext)), 34 | "CopyMethod"); 35 | 36 | yield return new CodeAssignStatement(sourceTypeProperty, sourceType); 37 | 38 | yield return new CodeAssignStatement(destTypeProperty, destType); 39 | 40 | yield return new CodeAssignStatement(copyMethodProperty, copyMethod); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /samples/PreserveFilePropertiesSamples/PreserveFilePropertiesSamples/FileOperations.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace PreserveFilePropertiesSamples 8 | { 9 | using System; 10 | using System.IO; 11 | 12 | static class FileOperations 13 | { 14 | public static void GetFileProperties( 15 | string path, 16 | out DateTimeOffset? creationTime, 17 | out DateTimeOffset? lastWriteTime, 18 | out FileAttributes? fileAttributes, 19 | bool isDirectory = false) 20 | { 21 | fileAttributes = File.GetAttributes(path); 22 | 23 | if (isDirectory) 24 | { 25 | creationTime = Directory.GetCreationTimeUtc(path); 26 | lastWriteTime = Directory.GetLastWriteTimeUtc(path); 27 | } 28 | else 29 | { 30 | creationTime = File.GetCreationTimeUtc(path); 31 | lastWriteTime = File.GetLastWriteTimeUtc(path); 32 | } 33 | } 34 | 35 | public static void SetFileProperties(string path, 36 | DateTimeOffset creationTime, 37 | DateTimeOffset lastWriteTime, 38 | FileAttributes fileAttributes, 39 | bool isDirectory = false) 40 | { 41 | File.SetAttributes(path, fileAttributes); 42 | 43 | if (isDirectory) 44 | { 45 | Directory.SetCreationTimeUtc(path, creationTime.UtcDateTime); 46 | Directory.SetLastWriteTimeUtc(path, lastWriteTime.UtcDateTime); 47 | } 48 | else 49 | { 50 | File.SetCreationTimeUtc(path, creationTime.UtcDateTime); 51 | File.SetLastWriteTimeUtc(path, lastWriteTime.UtcDateTime); 52 | } 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /test/DMTestLib/MultiDirectionTestMethod.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTestCodeGen 7 | { 8 | using System; 9 | using System.Collections.Generic; 10 | using System.Linq; 11 | using System.Reflection; 12 | 13 | public class MultiDirectionTestMethod 14 | { 15 | private HashSet transferDirections; 16 | 17 | public MethodInfo MethodInfoObj 18 | { 19 | get; 20 | private set; 21 | } 22 | 23 | public MultiDirectionTestMethod(MethodInfo methodInfo) 24 | { 25 | this.MethodInfoObj = methodInfo; 26 | transferDirections = new HashSet(); 27 | 28 | foreach (Attribute attribute in methodInfo.GetCustomAttributes(true)) 29 | { 30 | MultiDirectionTestMethodAttribute multiDirectionAttr = attribute as MultiDirectionTestMethodAttribute; 31 | if (null != multiDirectionAttr) 32 | { 33 | this.ParseMultiDirectionAttribute(multiDirectionAttr); 34 | } 35 | } 36 | } 37 | 38 | public IEnumerable GetTransferDirections() 39 | { 40 | return this.transferDirections; 41 | } 42 | 43 | private void ParseMultiDirectionAttribute(MultiDirectionTestMethodAttribute multiDirectionAttr) 44 | { 45 | foreach (var direction in multiDirectionAttr.ExtractDirections()) 46 | { 47 | if (this.transferDirections.Contains(direction) && direction.Tags.Any()) 48 | { 49 | this.transferDirections.Remove(direction); 50 | } 51 | 52 | this.transferDirections.Add(direction); 53 | } 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /test/DMLibTestCodeGen/Program.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTestCodeGen 7 | { 8 | using System; 9 | using System.Reflection; 10 | using System.Threading; 11 | 12 | public enum FrameworkType 13 | { 14 | DNet, 15 | DNetCore 16 | }; 17 | 18 | public class Program 19 | { 20 | public static FrameworkType FrameWorkType = FrameworkType.DNet; 21 | 22 | public static void Main(string[] args) 23 | { 24 | if (args == null || (args.Length != 2 && args.Length != 3)) 25 | { 26 | PrintHelp(); 27 | return; 28 | } 29 | 30 | string dllName = args[0]; 31 | string sourceFolder = args[1]; 32 | 33 | if (args.Length == 3) 34 | { 35 | FrameWorkType = (FrameworkType)Enum.Parse(typeof(FrameworkType), args[2]); 36 | } 37 | 38 | GenerateCode(dllName, sourceFolder); 39 | } 40 | 41 | private static void GenerateCode(string dllName, string outputFolder) 42 | { 43 | SourceCodeGenerator codeGen = new SourceCodeGenerator(outputFolder); 44 | 45 | Assembly assembly = Assembly.LoadFrom(dllName); 46 | 47 | foreach (Type type in assembly.GetTypes()) 48 | { 49 | if (null != type.GetCustomAttribute(typeof(MultiDirectionTestClassAttribute))) 50 | { 51 | MultiDirectionTestClass testClass = new MultiDirectionTestClass(type); 52 | codeGen.GenerateSourceCode(testClass); 53 | } 54 | } 55 | } 56 | 57 | private static void PrintHelp() 58 | { 59 | Console.WriteLine("Usage: DMLibTestCodeGen.exe [InputDll] [OutputSourceFolder] [FramworkType]"); 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /lib/TestHook/FaultInjectionPoint.cs: -------------------------------------------------------------------------------- 1 | 2 | //------------------------------------------------------------------------------ 3 | // 4 | // Copyright (c) Microsoft Corporation. All rights reserved. 5 | // 6 | // 7 | // Fault injection point definition 8 | // 9 | //------------------------------------------------------------------------------ 10 | 11 | namespace Microsoft.Azure.Storage.DataMovement 12 | { 13 | using System; 14 | 15 | #if DEBUG 16 | /// 17 | /// Static fault injection point for testing. Each fault injection point 18 | /// consists of a name and optionally a value. 19 | /// To enable a fault injection point, add a process-wide environment 20 | /// variable, with variable value = #faultValue. 21 | /// 22 | internal class FaultInjectionPoint 23 | { 24 | public static string FIP_ThrowExceptionAfterEnumerated = "ThrowExceptionAfterEnumerated"; 25 | public static string FIP_ThrowExceptionOnDirectory = "ThrowExceptionOnDirectory"; 26 | 27 | /// 28 | /// Initializes a new instance of the class 29 | /// 30 | /// fault injection point name 31 | public FaultInjectionPoint(string name) 32 | { 33 | this.Name = name; 34 | } 35 | 36 | /// 37 | /// Gets name of the fault injection point 38 | /// 39 | public string Name 40 | { 41 | get; 42 | private set; 43 | } 44 | 45 | /// 46 | /// Gets value of the fault injection point 47 | /// 48 | /// value of the fault injection point 49 | /// true the fault injection point is enable, false otherwise 50 | public bool TryGetValue(out string value) 51 | { 52 | value = Environment.GetEnvironmentVariable(this.Name); 53 | return (null != value); 54 | } 55 | } 56 | #endif 57 | } 58 | -------------------------------------------------------------------------------- /lib/DirectoryTransferContext.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement 8 | { 9 | using System.IO; 10 | 11 | /// 12 | /// Represents the context for a directory transfer, and provides additional runtime information about its execution. 13 | /// 14 | public class DirectoryTransferContext : TransferContext 15 | { 16 | /// 17 | /// Initializes a new instance of the class. 18 | /// 19 | public DirectoryTransferContext() 20 | : base() 21 | { 22 | } 23 | 24 | /// 25 | /// Initializes a new instance of the class. 26 | /// 27 | /// An object representing the last checkpoint from which the transfer continues on. 28 | public DirectoryTransferContext(TransferCheckpoint checkpoint) 29 | :base(checkpoint) 30 | { 31 | } 32 | 33 | /// 34 | /// Initializes a new instance of the class. 35 | /// 36 | /// The stream into which the transfer journal info will be written into. 37 | /// It can resume the previous paused transfer from its journal stream. 38 | public DirectoryTransferContext(Stream journalStream) 39 | :base(journalStream) 40 | { 41 | } 42 | 43 | /// 44 | /// Gets or sets the callback invoked to tell whether a transfer should be done. 45 | /// 46 | public ShouldTransferCallbackAsync ShouldTransferCallbackAsync 47 | { 48 | get; 49 | set; 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /lib/TransferControllers/ServiceSideSyncCopyControllers/ServiceSideSyncCopySource/PageBlobSourceHandler.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | 8 | namespace Microsoft.Azure.Storage.DataMovement.TransferControllers.ServiceSideSyncCopySource 9 | { 10 | using System.Collections.Generic; 11 | using System.Threading; 12 | using System.Threading.Tasks; 13 | using Microsoft.Azure.Storage.Blob; 14 | 15 | class PageBlobSourceHandler : BlobSourceHandler, IRangeBasedSourceHandler 16 | { 17 | CloudPageBlob pageBlob; 18 | 19 | public PageBlobSourceHandler(AzureBlobLocation sourceBlobLocation, TransferJob transferJob) 20 | : base(sourceBlobLocation, transferJob) 21 | { 22 | this.pageBlob = sourceBlobLocation.Blob as CloudPageBlob; 23 | } 24 | 25 | public async Task> GetCopyRangesAsync(long startOffset, long length, CancellationToken cancellationToken) 26 | { 27 | var pageRanges = await this.pageBlob.GetPageRangesAsync( 28 | startOffset, 29 | length, 30 | Utils.GenerateConditionWithCustomerCondition(this.SourceLocation.AccessCondition, this.SourceLocation.CheckedAccessCondition), 31 | Utils.GenerateBlobRequestOptions(this.SourceLocation.BlobRequestOptions), 32 | Utils.GenerateOperationContext(this.TransferContext), 33 | cancellationToken); 34 | 35 | List ranges = new List(); 36 | 37 | foreach (var pageRange in pageRanges) 38 | { 39 | ranges.Add(new Utils.Range() 40 | { 41 | StartOffset = pageRange.StartOffset, 42 | EndOffset = pageRange.EndOffset, 43 | HasData = true 44 | }); 45 | } 46 | 47 | return ranges; 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /lib/TransferStatusHelpers/TransferDownloadBuffer.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement 8 | { 9 | using System.Threading; 10 | 11 | class TransferDownloadBuffer 12 | { 13 | private int finishedLength = 0; 14 | 15 | private int processed = 0; 16 | 17 | public TransferDownloadBuffer(long startOffset, int expectedLength, byte[][] buffer) 18 | { 19 | this.Length = expectedLength; 20 | this.StartOffset = startOffset; 21 | this.MemoryBuffer = buffer; 22 | } 23 | 24 | public int Length 25 | { 26 | get; 27 | private set; 28 | } 29 | 30 | public long StartOffset 31 | { 32 | get; 33 | private set; 34 | } 35 | 36 | public byte[][] MemoryBuffer 37 | { 38 | get; 39 | private set; 40 | } 41 | 42 | public bool Finished 43 | { 44 | get 45 | { 46 | return this.finishedLength == this.Length; 47 | } 48 | } 49 | 50 | /// 51 | /// Mark this buffer as processed. The return value indicates whether the buffer 52 | /// is marked as processed by invocation of this method. This method returns true 53 | /// exactly once. The caller is supposed to invoke this method before processing 54 | /// the buffer and proceed only if this method returns true. 55 | /// 56 | /// Whether this instance is marked as processed by invocation of this method. 57 | public bool MarkAsProcessed() 58 | { 59 | return 0 == Interlocked.CompareExchange(ref this.processed, 1, 0); 60 | } 61 | 62 | public void ReadFinish(int length) 63 | { 64 | Interlocked.Add(ref this.finishedLength, length); 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /test/DMTestLib/MultiDirectionTestMethodSetAttribute.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTestCodeGen 7 | { 8 | using System; 9 | using System.Collections.Generic; 10 | using System.Reflection; 11 | 12 | public abstract class MultiDirectionTestMethodSetAttribute : MultiDirectionTestMethodAttribute 13 | { 14 | private List testMethodAttributes = new List(); 15 | 16 | private List directionFilters = new List(); 17 | 18 | protected void AddTestMethodAttribute(MultiDirectionTestMethodAttribute testMethodAttribute) 19 | { 20 | if (testMethodAttribute == null) 21 | { 22 | throw new ArgumentNullException("testMethodAttribute"); 23 | } 24 | 25 | this.testMethodAttributes.Add(testMethodAttribute); 26 | } 27 | 28 | internal void AddDirectionFilter(DirectionFilter directionFilter) 29 | { 30 | this.directionFilters.Add(directionFilter); 31 | } 32 | 33 | internal override IEnumerable ExtractDirections() 34 | { 35 | foreach(var attribute in this.testMethodAttributes) 36 | { 37 | foreach(var direction in attribute.ExtractDirections()) 38 | { 39 | if (this.Filter(direction)) 40 | { 41 | yield return direction; 42 | } 43 | } 44 | } 45 | } 46 | 47 | private bool Filter(TestMethodDirection direction) 48 | { 49 | foreach(var directionFilter in this.directionFilters) 50 | { 51 | if (!directionFilter.Filter(direction)) 52 | { 53 | return false; 54 | } 55 | } 56 | 57 | return true; 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /lib/TransferEventArgs.cs: -------------------------------------------------------------------------------- 1 | //----------------------------------------------------------------------------- 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //----------------------------------------------------------------------------- 6 | namespace Microsoft.Azure.Storage.DataMovement 7 | { 8 | using System; 9 | 10 | /// 11 | /// Transfer event args. 12 | /// 13 | public sealed class TransferEventArgs : EventArgs 14 | { 15 | /// 16 | /// Initializes a new instance of the class. 17 | /// 18 | /// Instance representation of transfer source location. 19 | /// Instance representation of transfer destination location. 20 | public TransferEventArgs(object source, object destination) 21 | { 22 | this.Source = source; 23 | this.Destination = destination; 24 | } 25 | 26 | /// 27 | /// Gets the instance representation of transfer source location. 28 | /// 29 | public object Source 30 | { 31 | get; 32 | private set; 33 | } 34 | 35 | /// 36 | /// Gets the instance representation of transfer destination location. 37 | /// 38 | public object Destination 39 | { 40 | get; 41 | private set; 42 | } 43 | 44 | /// 45 | /// Gets transfer start time. 46 | /// 47 | public DateTime StartTime 48 | { 49 | get; 50 | internal set; 51 | } 52 | 53 | /// 54 | /// Gets transfer end time. 55 | /// 56 | public DateTime EndTime 57 | { 58 | get; 59 | internal set; 60 | } 61 | 62 | /// 63 | /// Gets the exception if the transfer is failed, or null if the transfer is success. 64 | /// 65 | public Exception Exception 66 | { 67 | get; 68 | internal set; 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /lib/TransferStatusHelpers/TransferDataState.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace Microsoft.Azure.Storage.DataMovement 7 | { 8 | using System; 9 | 10 | /// 11 | /// Calculate and show transfer speed. 12 | /// 13 | internal abstract class TransferDataState : IDisposable 14 | { 15 | /// 16 | /// Gets or sets a handle to the memory buffer to ensure the 17 | /// memory buffer remains in memory during the entire operation. 18 | /// 19 | public byte[][] MemoryBuffer 20 | { 21 | get; 22 | set; 23 | } 24 | 25 | /// 26 | /// Gets or sets the starting offset of this part of data. 27 | /// 28 | public long StartOffset 29 | { 30 | get; 31 | set; 32 | } 33 | 34 | /// 35 | /// Gets or sets the length of this part of data. 36 | /// 37 | public int Length 38 | { 39 | get; 40 | set; 41 | } 42 | 43 | /// 44 | /// Gets or sets how many bytes have been read. 45 | /// 46 | public int BytesRead 47 | { 48 | get; 49 | set; 50 | } 51 | 52 | /// 53 | /// Public dispose method to release all resources owned. 54 | /// 55 | public void Dispose() 56 | { 57 | this.Dispose(true); 58 | GC.SuppressFinalize(this); 59 | } 60 | 61 | /// 62 | /// Private dispose method to release managed/unmanaged objects. 63 | /// If disposing = true clean up managed resources as well as unmanaged resources. 64 | /// If disposing = false only clean up unmanaged resources. 65 | /// 66 | /// Indicates whether or not to dispose managed resources. 67 | protected abstract void Dispose(bool disposing); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /lib/TransferEnumerators/FileListContinuationToken.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferEnumerators 8 | { 9 | using System; 10 | using System.Runtime.Serialization; 11 | 12 | #if BINARY_SERIALIZATION 13 | [Serializable] 14 | #else 15 | [DataContract] 16 | #endif // BINARY_SERIALIZATION 17 | sealed class FileListContinuationToken : ListContinuationToken 18 | #if BINARY_SERIALIZATION 19 | , ISerializable 20 | #endif // BINARY_SERIALIZATION 21 | { 22 | private const string FilePathName = "FilePath"; 23 | 24 | public FileListContinuationToken(string filePath) 25 | { 26 | this.FilePath = filePath; 27 | } 28 | 29 | #if BINARY_SERIALIZATION 30 | private FileListContinuationToken(SerializationInfo info, StreamingContext context) 31 | { 32 | if (info == null) 33 | { 34 | throw new System.ArgumentNullException("info"); 35 | } 36 | 37 | this.FilePath = info.GetString(FilePathName); 38 | } 39 | #endif // BINARY_SERIALIZATION 40 | 41 | /// 42 | /// Gets relative path of the last listed file. 43 | /// 44 | #if !BINARY_SERIALIZATION 45 | [DataMember] 46 | #endif 47 | public string FilePath 48 | { 49 | get; 50 | private set; 51 | } 52 | 53 | #if BINARY_SERIALIZATION 54 | /// 55 | /// Serializes the object. 56 | /// 57 | /// Serialization info object. 58 | /// Streaming context. 59 | public void GetObjectData(SerializationInfo info, StreamingContext context) 60 | { 61 | if (info == null) 62 | { 63 | throw new System.ArgumentNullException("info"); 64 | } 65 | 66 | info.AddValue(FilePathName, this.FilePath, typeof(string)); 67 | } 68 | #endif // BINARY_SERIALIZATION 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /lib/TransferOptions/DirectoryOptions.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace Microsoft.Azure.Storage.DataMovement 7 | { 8 | /// 9 | /// Represents a set of options that may be specified for directory transfer operation 10 | /// 11 | public class DirectoryOptions 12 | { 13 | /// 14 | /// Gets or sets a string that will be used to match against the names of files. 15 | /// 16 | /// 17 | /// Behavior of SearchPattern match varies for different source directory types and setting of Recursive: 18 | /// When source is local directory path, SearchPattern is matched against source file name as standard wildcards. If 19 | /// recuresive is set to false, only files directly under the source directory will be matched. Otherwise, all files in the 20 | /// sub-directory will be matched as well. 21 | /// 22 | /// When source is Azure blob directory, if recuresive is set to true, SearchPattern is matched against source blob as name prefix. 23 | /// Otherwise, only Azure blob with the exact name specified by SearchPattern will be matched. 24 | /// 25 | /// When source is Azure file directory, if recursive is set to true, SearchPattern is not supported. Otherwise, only Azure file 26 | /// with the exact name specified by SearchPattern will be matched. 27 | /// 28 | /// If SearchPattern is not specified, "*.*" will be used for local directory source while empty string for Azure blob/file 29 | /// directory. So please either specify the Search Pattern or set Recursive to true when source is Azure blob/file directory, 30 | /// otherwise, no blob/file will be matched. 31 | /// 32 | public string SearchPattern { get; set; } 33 | 34 | /// 35 | /// Gets or sets a boolean that indicates whether to include subdirectories when doing a directory transfer operation. 36 | /// 37 | public bool Recursive { get; set; } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /lib/TransferStatusHelpers/ReadDataState.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace Microsoft.Azure.Storage.DataMovement 7 | { 8 | using System.IO; 9 | 10 | /// 11 | /// Keep the state of reading a single block from the input stream. 12 | /// 13 | internal class ReadDataState : TransferDataState 14 | { 15 | /// 16 | /// Gets or sets the memory stream used to encapsulate the memory 17 | /// buffer for passing the methods such as PutBlock, WritePages, 18 | /// DownloadToStream and DownloadRangeToStream, as these methods 19 | /// requires a stream and doesn't allow for a byte array as input. 20 | /// 21 | public Stream MemoryStream 22 | { 23 | get; 24 | set; 25 | } 26 | 27 | /// 28 | /// Gets or sets the memory manager that controls global memory 29 | /// allocation. 30 | /// 31 | public MemoryManager MemoryManager 32 | { 33 | get; 34 | set; 35 | } 36 | 37 | /// 38 | /// Private dispose method to release managed/unmanaged objects. 39 | /// If disposing = true clean up managed resources as well as unmanaged resources. 40 | /// If disposing = false only clean up unmanaged resources. 41 | /// 42 | /// Indicates whether or not to dispose managed resources. 43 | protected override void Dispose(bool disposing) 44 | { 45 | if (disposing) 46 | { 47 | if (null != this.MemoryStream) 48 | { 49 | this.MemoryStream.Dispose(); 50 | this.MemoryStream = null; 51 | } 52 | 53 | if (null != this.MemoryBuffer) 54 | { 55 | this.MemoryManager.ReleaseBuffers(this.MemoryBuffer); 56 | this.MemoryManager = null; 57 | } 58 | } 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /lib/TransferEnumerators/AzureFileListContinuationToken.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferEnumerators 8 | { 9 | using System; 10 | using System.Runtime.Serialization; 11 | 12 | #if BINARY_SERIALIZATION 13 | [Serializable] 14 | #else 15 | [DataContract] 16 | #endif // BINARY_SERIALIZATION 17 | sealed class AzureFileListContinuationToken : ListContinuationToken 18 | #if BINARY_SERIALIZATION 19 | , ISerializable 20 | #endif // BINARY_SERIALIZATION 21 | { 22 | private const string FilePathName = "FilePath"; 23 | 24 | public AzureFileListContinuationToken(string filePath) 25 | { 26 | this.FilePath = filePath; 27 | } 28 | 29 | #if BINARY_SERIALIZATION 30 | private AzureFileListContinuationToken(SerializationInfo info, StreamingContext context) 31 | { 32 | if (info == null) 33 | { 34 | throw new System.ArgumentNullException("info"); 35 | } 36 | 37 | this.FilePath = info.GetString(FilePathName); 38 | } 39 | #endif // BINARY_SERIALIZATION 40 | 41 | /// 42 | /// Gets relative path of the last listed file. 43 | /// 44 | #if !BINARY_SERIALIZATION 45 | [DataMember] 46 | #endif 47 | public string FilePath 48 | { 49 | get; 50 | private set; 51 | } 52 | 53 | #if BINARY_SERIALIZATION 54 | /// 55 | /// Serializes the object. 56 | /// 57 | /// Serialization info object. 58 | /// Streaming context. 59 | public void GetObjectData(SerializationInfo info, StreamingContext context) 60 | { 61 | if (info == null) 62 | { 63 | throw new System.ArgumentNullException("info"); 64 | } 65 | 66 | info.AddValue(FilePathName, this.FilePath, typeof(string)); 67 | } 68 | #endif // BINARY_SERIALIZATION 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /test/DMLibTest/Framework/TestExecutionOptions.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTest 7 | { 8 | using Microsoft.Azure.Storage.Auth; 9 | using System; 10 | 11 | public class TestExecutionOptions where TDataInfo : IDataInfo 12 | { 13 | public const int DefaultTimeoutInMs = 20 * 60 * 1000; // 20 min 14 | 15 | public TestExecutionOptions() 16 | { 17 | this.TimeoutInMs = DefaultTimeoutInMs; 18 | this.DestTransferDataInfo = default(TDataInfo); 19 | this.DisableSourceGenerator = false; 20 | this.DisableDestinationFetch = false; 21 | this.DisableSourceCleaner = false; 22 | this.LimitSpeed = false; 23 | } 24 | 25 | public int TimeoutInMs 26 | { 27 | get; 28 | set; 29 | } 30 | 31 | public TDataInfo DestTransferDataInfo 32 | { 33 | get; 34 | set; 35 | } 36 | 37 | public bool IsDirectoryTransfer 38 | { 39 | get; 40 | set; 41 | } 42 | 43 | public bool DisableSourceGenerator 44 | { 45 | get; 46 | set; 47 | } 48 | 49 | public bool DisableSourceCleaner 50 | { 51 | get; 52 | set; 53 | } 54 | 55 | public bool DisableDestinationFetch 56 | { 57 | get; 58 | set; 59 | } 60 | 61 | public bool LimitSpeed 62 | { 63 | get; 64 | set; 65 | } 66 | 67 | public int? BlockSize { get; set; } 68 | 69 | public StorageCredentials SourceCredentials 70 | { 71 | get; 72 | set; 73 | } 74 | 75 | public StorageCredentials DestCredentials 76 | { 77 | get; 78 | set; 79 | } 80 | 81 | 82 | public Action TransferItemModifier; 83 | 84 | public Action AfterDataPrepared; 85 | 86 | public Action AfterAllItemAdded; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /netcore/DMLibTest/CloudFileDirectoryExtensions.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Azure.Storage; 2 | using Microsoft.Azure.Storage.File; 3 | using System.Collections.Generic; 4 | using System; 5 | using System.Net; 6 | 7 | namespace DMLibTest 8 | { 9 | public static class CloudFileDirectoryExtensions 10 | { 11 | public static bool CreateIfNotExists(this CloudFileDirectory dir, FileRequestOptions requestOptions = null, OperationContext operationContext = null) 12 | { 13 | try 14 | { 15 | return dir.CreateIfNotExistsAsync(requestOptions, operationContext).GetAwaiter().GetResult(); 16 | } 17 | catch (StorageException se) 18 | { 19 | // Creation against root directory throws 405 exception, 20 | // here swallow the error. 21 | if (null != se 22 | && null != se.RequestInformation 23 | && se.RequestInformation.HttpStatusCode == (int)HttpStatusCode.MethodNotAllowed) 24 | { 25 | return true; 26 | } 27 | else 28 | { 29 | throw; 30 | } 31 | } 32 | } 33 | 34 | public static void Delete(this CloudFileDirectory dir, AccessCondition accessCondition = null, FileRequestOptions options = null, OperationContext operationContext = null) 35 | { 36 | dir.DeleteAsync(accessCondition, options, operationContext).GetAwaiter().GetResult(); 37 | } 38 | 39 | public static bool Exists(this CloudFileDirectory dir, FileRequestOptions requestOptions = null, OperationContext operationContext = null) 40 | { 41 | return dir.ExistsAsync(requestOptions, operationContext).GetAwaiter().GetResult(); 42 | } 43 | 44 | public static IEnumerable ListFilesAndDirectories(this CloudFileDirectory dir, FileRequestOptions options = null, OperationContext operationContext = null) 45 | { 46 | FileContinuationToken continuationToken = new FileContinuationToken(); 47 | 48 | // this is no longer a lazy method: ListFilesAndDirectoriesSegmentedAsync will return the maximum results (5000) at once 49 | return dir.ListFilesAndDirectoriesSegmentedAsync(null, continuationToken, options, operationContext).GetAwaiter().GetResult().Results; 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /netcore/DMLibTest/CloudFileShareExtensions.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Azure.Storage; 2 | using Microsoft.Azure.Storage.File; 3 | 4 | namespace DMLibTest 5 | { 6 | public static class CloudFileShareExtensions 7 | { 8 | public static void Create(this CloudFileShare share, FileRequestOptions requestOptions = null, OperationContext operationContext = null) 9 | { 10 | share.CreateAsync(requestOptions, operationContext).GetAwaiter().GetResult(); 11 | } 12 | 13 | public static bool CreateIfNotExists(this CloudFileShare share, FileRequestOptions requestOptions = null, OperationContext operationContext = null) 14 | { 15 | return share.CreateIfNotExistsAsync(requestOptions, operationContext).GetAwaiter().GetResult(); 16 | } 17 | 18 | public static void Delete(this CloudFileShare share, AccessCondition accessCondition = null, FileRequestOptions options = null, OperationContext operationContext = null) 19 | { 20 | share.DeleteAsync(accessCondition, options, operationContext).GetAwaiter().GetResult(); 21 | } 22 | 23 | public static bool DeleteIfExists(this CloudFileShare share, AccessCondition accessCondition = null, FileRequestOptions options = null, OperationContext operationContext = null) 24 | { 25 | return share.DeleteIfExistsAsync(accessCondition, options, operationContext).GetAwaiter().GetResult(); 26 | } 27 | 28 | public static bool Exists(this CloudFileShare share, FileRequestOptions requestOptions = null, OperationContext operationContext = null) 29 | { 30 | return share.ExistsAsync(requestOptions, operationContext).GetAwaiter().GetResult(); 31 | } 32 | 33 | public static FileSharePermissions GetPermissions(this CloudFileShare share, AccessCondition accessCondition = null, FileRequestOptions options = null, OperationContext operationContext = null) 34 | { 35 | return share.GetPermissionsAsync(accessCondition, options, operationContext).GetAwaiter().GetResult(); 36 | } 37 | 38 | public static void SetPermissions(this CloudFileShare share, FileSharePermissions permissions, AccessCondition accessCondition = null, FileRequestOptions options = null, OperationContext operationContext = null) 39 | { 40 | share.SetPermissionsAsync(permissions, accessCondition, options, operationContext).GetAwaiter().GetResult(); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /netcore/DMTestLib/DMTestLib.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | DMTestLib Class Library 5 | emmazhu 6 | netcoreapp2.0 7 | DMTestLib 8 | DMTestLib 9 | $(AssetTargetFallback);dnxcore50;portable-net45+win8 10 | 2.0 11 | $(DefineConstants);DOTNET5_4 12 | true 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /test/DMLibTest/Framework/DataAdaptor.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTest 7 | { 8 | using Microsoft.Azure.Storage.Auth; 9 | 10 | public abstract class DataAdaptor where TDataInfo : IDataInfo 11 | { 12 | public abstract string StorageKey 13 | { 14 | get; 15 | } 16 | 17 | public SourceOrDest SourceOrDest 18 | { 19 | get; 20 | protected set; 21 | } 22 | 23 | public abstract string GetAddress(params string[] list); 24 | 25 | public abstract string GetSecondaryAddress(params string[] list); 26 | 27 | public abstract object GetTransferObject(string rootPath, FileNode fileNode, StorageCredentials credentials = null); 28 | 29 | public abstract object GetTransferObject(string rootPath, DirNode dirNode, StorageCredentials credentials = null); 30 | 31 | public abstract void CreateIfNotExists(); 32 | 33 | public abstract bool Exists(); 34 | 35 | public abstract void WaitForGEO(); 36 | 37 | public abstract void ValidateMD5ByDownloading(object file); 38 | 39 | public void GenerateData(TDataInfo dataInfo) 40 | { 41 | this.GenerateDataImp(dataInfo); 42 | 43 | if (SourceOrDest.Source == this.SourceOrDest) 44 | { 45 | MultiDirectionTestInfo.GeneratedSourceDataInfos.Add(dataInfo == null ? dataInfo : dataInfo.Clone()); 46 | } 47 | else 48 | { 49 | MultiDirectionTestInfo.GeneratedDestDataInfos.Add(dataInfo == null ? dataInfo : dataInfo.Clone()); 50 | } 51 | } 52 | 53 | public abstract TDataInfo GetTransferDataInfo(string rootDir); 54 | 55 | public abstract void Cleanup(); 56 | 57 | public abstract void DeleteLocation(); 58 | 59 | public abstract string GenerateSAS(SharedAccessPermissions sap, int validatePeriod, string policySignedIdentifier = null); 60 | 61 | public abstract void RevokeSAS(); 62 | 63 | public abstract void MakePublic(); 64 | 65 | public abstract void Reset(); 66 | 67 | protected abstract void GenerateDataImp(TDataInfo dataInfo); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /lib/TransferStatusHelpers/SharedTransferData.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace Microsoft.Azure.Storage.DataMovement 7 | { 8 | using System; 9 | using System.Collections.Concurrent; 10 | 11 | internal class SharedTransferData 12 | { 13 | private long totalLength; 14 | 15 | /// 16 | /// Gets or sets length of source. 17 | /// 18 | public long TotalLength 19 | { 20 | get { return this.totalLength; } 21 | set 22 | { 23 | this.totalLength = value; 24 | 25 | var handler = OnTotalLengthChanged; 26 | handler?.Invoke(this, null); 27 | } 28 | } 29 | 30 | /// 31 | /// Update totallength but don't update BlockSize accordingly. 32 | /// 33 | /// 34 | public void UpdateTotalLength(long value) 35 | { 36 | this.totalLength = value; 37 | } 38 | 39 | public int BlockSize { get; set; } 40 | 41 | public int MemoryChunksRequiredEachTime { get; set; } 42 | 43 | /// 44 | /// Gets or sets the job instance representing the transfer. 45 | /// 46 | public TransferJob TransferJob { get; set; } 47 | 48 | /// 49 | /// Gets or sets list of available transfer data from source. 50 | /// 51 | public ConcurrentDictionary AvailableData { get; set; } 52 | 53 | /// 54 | /// Gets or sets a value indicating whether should disable validation of content md5. 55 | /// The reader should get this value from source's RequestOptions, 56 | /// the writer should do or not do validation on content md5 according to this value. 57 | /// 58 | public bool DisableContentMD5Validation { get; set; } 59 | 60 | /// 61 | /// Gets or sets attribute for blob/azure file. 62 | /// 63 | public Attributes Attributes { get; set; } 64 | 65 | public event EventHandler OnTotalLengthChanged; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /netcore/MsTestLib/MsTestLib.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | MsTestLib Class Library 5 | 0.0.0 6 | emmazhu 7 | netcoreapp2.0 8 | MsTestLib 9 | MsTestLib 10 | $(AssetTargetFallback);dnxcore50;portable-net45+win8 11 | 2.0 12 | false 13 | false 14 | false 15 | false 16 | false 17 | false 18 | $(DefineConstants);DOTNET5_4 19 | true 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /netcore/DMLibTest/CloudPageBlobExtensions.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Azure.Storage; 2 | using Microsoft.Azure.Storage.Blob; 3 | using System.Collections.Generic; 4 | using System.IO; 5 | 6 | namespace DMLibTest 7 | { 8 | public static class CloudPageBlobExtensions 9 | { 10 | public static void Create(this CloudPageBlob blob, long size, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null) 11 | { 12 | blob.CreateAsync(size, accessCondition, options, operationContext).GetAwaiter().GetResult(); 13 | } 14 | 15 | public static IEnumerable GetPageRanges(this CloudPageBlob blob, long? offset = default(long?), long? length = default(long?), AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null) 16 | { 17 | return blob.GetPageRangesAsync(offset, length, accessCondition, options, operationContext).GetAwaiter().GetResult(); 18 | } 19 | 20 | public static void UploadFromByteArray(this CloudPageBlob blob, byte[] buffer, int index, int count, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null) 21 | { 22 | blob.UploadFromByteArrayAsync(buffer, index, count, accessCondition, options, operationContext).GetAwaiter().GetResult(); 23 | } 24 | 25 | public static void UploadFromFile(this CloudPageBlob blob, string path, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null) 26 | { 27 | blob.UploadFromFileAsync(path, accessCondition, options, operationContext).GetAwaiter().GetResult(); 28 | } 29 | 30 | public static void UploadFromStream(this CloudPageBlob blob, Stream source, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null) 31 | { 32 | blob.UploadFromStreamAsync(source, accessCondition, options, operationContext).GetAwaiter().GetResult(); 33 | } 34 | 35 | public static void WritePages(this CloudPageBlob blob, Stream pageData, long startOffset, string contentMD5 = null, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null) 36 | { 37 | blob.WritePagesAsync(pageData, startOffset, contentMD5).GetAwaiter().GetResult(); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /test/DMLibTest/Framework/URIBlobDataAdaptor.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTest 7 | { 8 | using DMLibTestCodeGen; 9 | using Microsoft.Azure.Storage.Auth; 10 | using System; 11 | using BlobTypeConst = DMLibTest.BlobType; 12 | internal class URIBlobDataAdaptor : CloudBlobDataAdaptor 13 | { 14 | public URIBlobDataAdaptor(TestAccount testAccount, string containerName) 15 | : base (testAccount, containerName, BlobTypeConst.Block, SourceOrDest.Source) 16 | { 17 | base.MakePublic(); 18 | } 19 | 20 | public override void Reset() 21 | { 22 | // Do nothing, keep the container public 23 | } 24 | 25 | public override object GetTransferObject(string rootPath, FileNode fileNode, StorageCredentials credentials = null) 26 | { 27 | Uri result= base.GetCloudBlobReference(rootPath, fileNode).SnapshotQualifiedUri; 28 | 29 | if (credentials != null) 30 | { 31 | result = credentials.TransformUri(result); 32 | } 33 | 34 | return result; 35 | } 36 | 37 | public override void ValidateMD5ByDownloading(object file) 38 | { 39 | throw new NotSupportedException(); 40 | } 41 | 42 | public override object GetTransferObject(string rootPath, DirNode dirNode, StorageCredentials credentials = null) 43 | { 44 | throw new InvalidOperationException("Can't get directory transfer object in URI data adaptor."); 45 | } 46 | 47 | protected override string BlobType 48 | { 49 | get 50 | { 51 | DMLibDataType destDataType = DMLibTestContext.DestType; 52 | if (destDataType == DMLibDataType.PageBlob) 53 | { 54 | return BlobTypeConst.Page; 55 | } 56 | else if (destDataType == DMLibDataType.AppendBlob) 57 | { 58 | return BlobTypeConst.Append; 59 | } 60 | else 61 | { 62 | return BlobTypeConst.Block; 63 | } 64 | } 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /lib/Interop/Interop.OSX.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.Interop 8 | { 9 | using System; 10 | using System.Runtime.InteropServices; 11 | 12 | internal static partial class NativeMethods 13 | { 14 | const int HOST_VM_INFO = 2; 15 | 16 | #region P/Invokes 17 | [DllImport("libc", CharSet = CharSet.Ansi, BestFitMapping = false, ThrowOnUnmappableChar = true)] 18 | internal static extern int sysctlbyname([MarshalAs(UnmanagedType.LPTStr)]string name, ref int oldp, ref int oldlenp, IntPtr newp, int newlen); 19 | 20 | [DllImport("libc")] 21 | private static extern IntPtr mach_host_self(); 22 | 23 | [DllImport("libc")] 24 | private static extern IntPtr host_statistics(IntPtr host, int hostFlavor, ref vm_statistics vmStat, ref int count); 25 | #endregion // P/Invokes 26 | 27 | #region Helper methods 28 | internal static vm_statistics GetOSXHostStatistics() 29 | { 30 | var mach_host = mach_host_self(); 31 | #if GENERIC_MARSHAL_SIZEOF 32 | var statisticsInfoCount = Marshal.SizeOf() / Marshal.SizeOf(); 33 | #else // GENERIC_MARSHAL_SIZEOF 34 | var statisticsInfoCount = Marshal.SizeOf(typeof(vm_statistics)) / Marshal.SizeOf(typeof(int)); 35 | #endif // GENERIC_MARSHAL_SIZEOF 36 | vm_statistics vmStats = new vm_statistics(); 37 | host_statistics(mach_host, HOST_VM_INFO, ref vmStats, ref statisticsInfoCount); 38 | return vmStats; 39 | } 40 | #endregion // Helper methods 41 | 42 | #region Helper structs 43 | internal struct vm_statistics 44 | { 45 | public int free_count; 46 | public int active_count; 47 | public int inactive_count; 48 | public int wire_count; 49 | public int zero_fill_count; 50 | public int reactivations; 51 | public int pageins; 52 | public int pageouts; 53 | public int faults; 54 | public int cow_faults; 55 | public int lookups; 56 | public int hits; 57 | } 58 | #endregion // Helper structs 59 | } 60 | } -------------------------------------------------------------------------------- /lib/Exceptions/TransferSkippedException.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement 8 | { 9 | using System; 10 | using System.Runtime.Serialization; 11 | 12 | /// 13 | /// Exceptions thrown when transfer skips. 14 | /// 15 | #if BINARY_SERIALIZATION 16 | [Serializable] 17 | #else 18 | [DataContract] 19 | #endif // BINARY_SERIALIZATION 20 | public class TransferSkippedException : TransferException 21 | { 22 | /// 23 | /// Initializes a new instance of the class. 24 | /// 25 | public TransferSkippedException() 26 | : base(TransferErrorCode.NotOverwriteExistingDestination) 27 | { 28 | } 29 | 30 | /// 31 | /// Initializes a new instance of the class. 32 | /// 33 | /// The message that describes the error. 34 | public TransferSkippedException(string errorMessage) 35 | : base(TransferErrorCode.NotOverwriteExistingDestination, errorMessage) 36 | { 37 | } 38 | 39 | /// 40 | /// Initializes a new instance of the class. 41 | /// 42 | /// Exception message. 43 | /// Inner exception. 44 | public TransferSkippedException(string errorMessage, Exception innerException) 45 | : base(TransferErrorCode.NotOverwriteExistingDestination, errorMessage, innerException) 46 | { 47 | } 48 | 49 | #if BINARY_SERIALIZATION 50 | /// 51 | /// Initializes a new instance of the class. 52 | /// 53 | /// Serialization information. 54 | /// Streaming context. 55 | protected TransferSkippedException(SerializationInfo info, StreamingContext context) 56 | : base(info, context) 57 | { 58 | } 59 | #endif // BINARY_SERIALIZATION 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /netcore/DMLibTest/CloudBlockBlobExtensions.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Azure.Storage; 2 | using Microsoft.Azure.Storage.Blob; 3 | using System.Collections.Generic; 4 | using System.IO; 5 | 6 | namespace DMLibTest 7 | { 8 | public static class CloudBlockBlobExtensions 9 | { 10 | public static IEnumerable DownloadBlockList(this CloudBlockBlob blob, BlockListingFilter blockListingFilter = BlockListingFilter.Committed, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null) 11 | { 12 | return blob.DownloadBlockListAsync(blockListingFilter, accessCondition, options, operationContext).GetAwaiter().GetResult(); 13 | } 14 | 15 | public static void PutBlock(this CloudBlockBlob blob, string blockId, Stream blockData, string contentMD5, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null) 16 | { 17 | blob.PutBlockAsync(blockId, blockData, contentMD5, accessCondition, options, operationContext).GetAwaiter().GetResult(); 18 | } 19 | 20 | public static void PutBlockList(this CloudBlockBlob blob, IEnumerable blockList, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null) 21 | { 22 | blob.PutBlockListAsync(blockList, accessCondition, options, operationContext).GetAwaiter().GetResult(); 23 | } 24 | 25 | public static void UploadFromByteArray(this CloudBlockBlob blob, byte[] buffer, int index, int count, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null) 26 | { 27 | blob.UploadFromByteArrayAsync(buffer, index, count, accessCondition, options, operationContext).GetAwaiter().GetResult(); 28 | } 29 | 30 | public static void UploadFromFile(this CloudBlockBlob blob, string path, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null) 31 | { 32 | blob.UploadFromFileAsync(path, accessCondition, options, operationContext).GetAwaiter().GetResult(); 33 | } 34 | 35 | public static void UploadFromStream(this CloudBlockBlob blob, Stream source, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null) 36 | { 37 | blob.UploadFromStreamAsync(source, accessCondition, options, operationContext).GetAwaiter().GetResult(); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /lib/PreserveSMBPermissions.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement 8 | { 9 | using System; 10 | 11 | /// 12 | /// Enum to indicate what permission would be perserved DataMovement Library. 13 | /// 14 | [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "SMB")] 15 | [Flags] 16 | public enum PreserveSMBPermissions : int 17 | { 18 | /// 19 | /// Indicate to not preserve any permission 20 | /// 21 | None = 0, 22 | 23 | /// 24 | /// To preserve Owner permission. 25 | /// In some cases, it requires to enable SeRestorePrivilege to set owner info to local file. 26 | /// See https://docs.microsoft.com/en-us/windows/win32/api/aclapi/nf-aclapi-setnamedsecurityinfow for details. 27 | /// To set owner permission to local file during downloading, the process needs to run with an account who has been assigned 28 | /// the privilege, for example run the process with administrator account. 29 | /// 30 | Owner = 0x00000001, 31 | 32 | /// 33 | /// To preserve Group permission. 34 | /// 35 | Group = 0x00000002, 36 | 37 | /// 38 | /// To preserve DACL permission. 39 | /// 40 | [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "DACL")] 41 | DACL = 0x00000004, 42 | 43 | /// 44 | /// To preserve SACL permission. 45 | /// It requires to enable SeSecurityPrivilege to get or set SACL from or to local file. 46 | /// See https://docs.microsoft.com/en-us/windows/win32/api/aclapi/nf-aclapi-setnamedsecurityinfow for details. 47 | /// To get or set owner permission from or to local file, the process needs to run with an account who has been assigned 48 | /// the privilege, for example run the process with administrator account. 49 | /// 50 | [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "SACL")] 51 | SACL = 0x00000008 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /lib/TransferJobs/TransferLocation.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement 8 | { 9 | using System.Runtime.Serialization; 10 | 11 | #if !BINARY_SERIALIZATION 12 | [DataContract] 13 | #endif 14 | internal abstract class TransferLocation 15 | { 16 | /// 17 | /// Gets transfer location type. 18 | /// 19 | public abstract TransferLocationType Type 20 | { 21 | get; 22 | } 23 | 24 | /// 25 | /// Gets source/destination instance in transfer. 26 | /// 27 | public abstract object Instance 28 | { 29 | get; 30 | } 31 | 32 | /// 33 | /// Gets or sets a value indicating whether detailed information(ETag, LMT, Length, Properties and metadata) 34 | /// of the instance contained in the current transfer location has been fetched. 35 | /// 36 | public bool? IsInstanceInfoFetched 37 | { 38 | get; 39 | set; 40 | } 41 | 42 | /// 43 | /// Validates the transfer location. 44 | /// 45 | public abstract void Validate(); 46 | 47 | // Summary: 48 | // Determines whether the specified transfer location is equal to the current transfer location. 49 | // 50 | // Parameters: 51 | // obj: 52 | // The transfer location to compare with the current transfer location. 53 | // 54 | // Returns: 55 | // true if the specified transfer location is equal to the current transfer location; otherwise, false. 56 | public override bool Equals(object obj) 57 | { 58 | TransferLocation location = obj as TransferLocation; 59 | if (location == null || this.Type != location.Type) 60 | return false; 61 | 62 | return this.ToString() == location.ToString(); 63 | } 64 | 65 | // 66 | // Summary: 67 | // Returns the hash code for the transfer location. 68 | // 69 | // Returns: 70 | // A 32-bit signed integer hash code. 71 | public override int GetHashCode() 72 | { 73 | return this.ToString().GetHashCode(); 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /lib/TransferOptions/CopyOptions.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement 8 | { 9 | using Microsoft.Azure.Storage.File; 10 | /// 11 | /// Represents a set of options that may be specified for copy operation 12 | /// 13 | public sealed class CopyOptions 14 | { 15 | /// 16 | /// Gets or sets a flag that indicates whether to preserve SMB attributes during copying. 17 | /// If set to true, destination Azure File's attributes will be set as source local file's attributes. 18 | /// SMB attributes includes last write time, creation time and . 19 | /// This flag only takes effect when copying from Azure File Service to Azure File Service. 20 | /// 21 | [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "SMB")] 22 | public bool PreserveSMBAttributes { get; set; } 23 | 24 | /// 25 | /// Gets or sets a flag that indicates whether to preserve SMB permissions during copying. 26 | /// This flag only takes effect when copying from Azure File Service to Azure File Service. 27 | /// 28 | [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "SMB")] 29 | public bool PreserveSMBPermissions { get; set; } 30 | 31 | /// 32 | /// Gets or sets an object that represents the access conditions for the source object. If null, no condition is used. 33 | /// 34 | public AccessCondition SourceAccessCondition { get; set; } 35 | 36 | /// 37 | /// Gets or sets an object that represents the access conditions for the destination object. If null, no condition is used. 38 | /// 39 | public AccessCondition DestinationAccessCondition { get; set; } 40 | 41 | /// 42 | /// Gets or sets a value which specifies the name of the encryption scope to use to encrypt the data provided in the request. 43 | /// This value only takes effect when destination is Azure Blob Service. 44 | /// 45 | public string EncryptionScope { get; set; } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /test/DMTestLib/DirectionFilter.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTestCodeGen 7 | { 8 | using System; 9 | using System.Collections.Generic; 10 | using System.Reflection; 11 | 12 | internal abstract class DirectionFilter 13 | { 14 | private IDictionary> valueGenerators = new Dictionary>(); 15 | 16 | protected void SetProperties(string queryString) 17 | { 18 | if (string.IsNullOrEmpty(queryString)) 19 | { 20 | return; 21 | } 22 | 23 | this.AddValueGenerators(); 24 | 25 | string[] keyValuePairs = queryString.Split(new char[] { ',' }, StringSplitOptions.None); 26 | 27 | foreach (var keyValuePair in keyValuePairs) 28 | { 29 | string key; 30 | string value; 31 | if (this.TryParseKeyValuePair(keyValuePair, out key, out value)) 32 | { 33 | var valueGen = this.valueGenerators[key]; 34 | object valueObject = valueGen(value); 35 | 36 | PropertyInfo prop = this.GetType().GetProperty(key); 37 | prop.SetValue(this, valueObject); 38 | } 39 | else 40 | { 41 | throw new ArgumentException(string.Format("Invalid queryString: {0}", queryString), "queryString"); 42 | } 43 | } 44 | } 45 | 46 | private bool TryParseKeyValuePair(string keyValuePair, out string key, out string value) 47 | { 48 | string[] keyValueArray = keyValuePair.Split(new char[] { '=' }, StringSplitOptions.None); 49 | if (keyValueArray.Length != 2) 50 | { 51 | key = null; 52 | value = null; 53 | return false; 54 | } 55 | 56 | key = keyValueArray[0].Trim(); 57 | value = keyValueArray[1].Trim(); 58 | return true; 59 | } 60 | 61 | protected virtual void AddValueGenerators() 62 | { 63 | } 64 | 65 | protected void AddValueGenerator(string propertyName, Func valueGenerator) 66 | { 67 | this.valueGenerators.Add(propertyName, valueGenerator); 68 | } 69 | 70 | public abstract bool Filter(TestMethodDirection direction); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /test/DMLibTest/Util/DMLibTestStream.cs: -------------------------------------------------------------------------------- 1 | 2 | 3 | namespace DMLibTest.Util 4 | { 5 | using System; 6 | using System.IO; 7 | 8 | class DMLibTestStream : Stream 9 | { 10 | FileStream internalStream = null; 11 | bool seekable; 12 | bool fixedSize; 13 | 14 | public DMLibTestStream(FileStream stream, bool seekable = true, bool fixedSize = true) 15 | { 16 | this.internalStream = stream; 17 | this.seekable = seekable; 18 | this.fixedSize = fixedSize; 19 | } 20 | 21 | public override bool CanRead => true; 22 | public override bool CanWrite => true; 23 | public override bool CanSeek => this.seekable; 24 | public override long Position 25 | { 26 | get 27 | { 28 | if (this.seekable) 29 | { 30 | return this.internalStream.Position; 31 | } 32 | 33 | throw new NotSupportedException(); 34 | } 35 | 36 | set 37 | { 38 | if (this.seekable) 39 | { 40 | this.internalStream.Position = value; 41 | } 42 | 43 | throw new NotSupportedException(); 44 | } 45 | } 46 | 47 | public override long Length 48 | { 49 | get 50 | { 51 | if (fixedSize) 52 | { 53 | return this.internalStream.Length; 54 | } 55 | 56 | throw new NotSupportedException(); 57 | } 58 | } 59 | 60 | public override void Flush() 61 | { 62 | this.internalStream.Flush(); 63 | } 64 | 65 | public override long Seek(long offset, SeekOrigin origin) 66 | { 67 | if (this.seekable) 68 | { 69 | return this.internalStream.Seek(offset, origin); 70 | } 71 | 72 | throw new NotSupportedException(); 73 | } 74 | 75 | public override void SetLength(long value) 76 | { 77 | throw new NotImplementedException(); 78 | } 79 | 80 | public override int Read(byte[] buffer, int offset, int count) 81 | { 82 | return this.internalStream.Read(buffer, offset, count); 83 | } 84 | 85 | public override void Write(byte[] buffer, int offset, int count) 86 | { 87 | this.internalStream.Write(buffer, offset, count); 88 | } 89 | 90 | public override void Close() 91 | { 92 | base.Close(); 93 | this.internalStream.Close(); 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /test/DMLibTest/Framework/CloudObjectExtensions.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | namespace DMLibTest 7 | { 8 | using System; 9 | using Microsoft.Azure.Storage.Blob; 10 | using Microsoft.Azure.Storage.File; 11 | using Microsoft.Azure.Storage.RetryPolicies; 12 | 13 | internal static class CloudObjectExtensions 14 | { 15 | public static string GetShortName(this CloudBlob cloudBlob) 16 | { 17 | CloudBlobDirectory parentDir = cloudBlob.Parent; 18 | 19 | if (null == parentDir) 20 | { 21 | // Root directory 22 | return cloudBlob.Name; 23 | } 24 | 25 | return GetShortNameFromUri(cloudBlob.Uri.ToString(), parentDir.Uri.ToString()); 26 | } 27 | 28 | public static string GetShortName(this CloudBlobDirectory cloudBlobDirectory) 29 | { 30 | CloudBlobDirectory parentDir = cloudBlobDirectory.Parent; 31 | 32 | if (null == parentDir) 33 | { 34 | // Root directory 35 | return String.Empty; 36 | } 37 | 38 | return GetShortNameFromUri(cloudBlobDirectory.Uri.ToString(), parentDir.Uri.ToString()); 39 | } 40 | 41 | private static string GetShortNameFromUri(string uri, string parentUri) 42 | { 43 | string delimiter = "/"; 44 | 45 | if (!parentUri.EndsWith(delimiter, StringComparison.Ordinal)) 46 | { 47 | parentUri += delimiter; 48 | } 49 | 50 | string shortName = uri.Substring(parentUri.Length); 51 | 52 | if (shortName.EndsWith(delimiter, StringComparison.Ordinal)) 53 | { 54 | shortName = shortName.Substring(0, shortName.Length - delimiter.Length); 55 | } 56 | 57 | return Uri.UnescapeDataString(shortName); 58 | } 59 | } 60 | 61 | internal static class HelperConst 62 | { 63 | public static BlobRequestOptions DefaultBlobOptions = new BlobRequestOptions 64 | { 65 | RetryPolicy = new LinearRetry(TimeSpan.FromSeconds(90), 3), 66 | MaximumExecutionTime = TimeSpan.FromMinutes(15) 67 | }; 68 | 69 | public static FileRequestOptions DefaultFileOptions = new FileRequestOptions 70 | { 71 | RetryPolicy = new LinearRetry(TimeSpan.FromSeconds(90), 3), 72 | MaximumExecutionTime = TimeSpan.FromMinutes(15) 73 | }; 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /DataMovement_k.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.26228.10 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DMLibTest", "netcore\DMLibTest\DMLibTest.csproj", "{1FBF2858-65D7-418B-92C4-75BAF207158C}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DMTestLib", "netcore\DMTestLib\DMTestLib.csproj", "{AA7B6E4B-E574-4005-89CC-97A84B41E63A}" 9 | EndProject 10 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MsTestLib", "netcore\MsTestLib\MsTestLib.csproj", "{0F62F89A-6DE6-4623-821B-DD6D2CAFCF99}" 11 | EndProject 12 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Storage.DataMovement", "netcore\Microsoft.Azure.Storage.DataMovement\Microsoft.Azure.Storage.DataMovement.csproj", "{9CFC8F0E-85B4-43FB-AAB8-EECF9EE0F289}" 13 | EndProject 14 | Global 15 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 16 | Debug|Any CPU = Debug|Any CPU 17 | Release|Any CPU = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 20 | {1FBF2858-65D7-418B-92C4-75BAF207158C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 21 | {1FBF2858-65D7-418B-92C4-75BAF207158C}.Debug|Any CPU.Build.0 = Debug|Any CPU 22 | {1FBF2858-65D7-418B-92C4-75BAF207158C}.Release|Any CPU.ActiveCfg = Release|Any CPU 23 | {1FBF2858-65D7-418B-92C4-75BAF207158C}.Release|Any CPU.Build.0 = Release|Any CPU 24 | {AA7B6E4B-E574-4005-89CC-97A84B41E63A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 25 | {AA7B6E4B-E574-4005-89CC-97A84B41E63A}.Debug|Any CPU.Build.0 = Debug|Any CPU 26 | {AA7B6E4B-E574-4005-89CC-97A84B41E63A}.Release|Any CPU.ActiveCfg = Release|Any CPU 27 | {AA7B6E4B-E574-4005-89CC-97A84B41E63A}.Release|Any CPU.Build.0 = Release|Any CPU 28 | {0F62F89A-6DE6-4623-821B-DD6D2CAFCF99}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 29 | {0F62F89A-6DE6-4623-821B-DD6D2CAFCF99}.Debug|Any CPU.Build.0 = Debug|Any CPU 30 | {0F62F89A-6DE6-4623-821B-DD6D2CAFCF99}.Release|Any CPU.ActiveCfg = Release|Any CPU 31 | {0F62F89A-6DE6-4623-821B-DD6D2CAFCF99}.Release|Any CPU.Build.0 = Release|Any CPU 32 | {9CFC8F0E-85B4-43FB-AAB8-EECF9EE0F289}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 33 | {9CFC8F0E-85B4-43FB-AAB8-EECF9EE0F289}.Debug|Any CPU.Build.0 = Debug|Any CPU 34 | {9CFC8F0E-85B4-43FB-AAB8-EECF9EE0F289}.Release|Any CPU.ActiveCfg = Release|Any CPU 35 | {9CFC8F0E-85B4-43FB-AAB8-EECF9EE0F289}.Release|Any CPU.Build.0 = Release|Any CPU 36 | EndGlobalSection 37 | GlobalSection(SolutionProperties) = preSolution 38 | HideSolutionNode = FALSE 39 | EndGlobalSection 40 | GlobalSection(ExtensibilityGlobals) = postSolution 41 | SolutionGuid = {98FDD843-F0CD-482D-9E73-75844F47C472} 42 | EndGlobalSection 43 | EndGlobal 44 | -------------------------------------------------------------------------------- /lib/TransferEnumerators/TaskQueue.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // Copyright (c) Microsoft Corporation 4 | // 5 | //------------------------------------------------------------------------------ 6 | 7 | namespace Microsoft.Azure.Storage.DataMovement.TransferEnumerators 8 | { 9 | using System; 10 | using System.Collections.Concurrent; 11 | using System.Threading; 12 | using System.Threading.Tasks; 13 | 14 | /// 15 | /// Task scheduler that schedule the tasks concorrently and FIFO return the result. 16 | /// 17 | /// Return type of the scheduled tasks. 18 | internal class TaskQueue : IDisposable 19 | { 20 | private BlockingCollection> queue; 21 | 22 | public TaskQueue(int capacity) 23 | { 24 | if (capacity < 0) 25 | { 26 | throw new ArgumentException("Capacity must be a positive integer.", "capacity"); 27 | } 28 | 29 | queue = new BlockingCollection>(capacity); 30 | } 31 | 32 | /// 33 | /// Enqueue a task into the tasks queue. This method blocks if the queue reaches its capacity. 34 | /// 35 | /// Task to enqueue. 36 | public void EnqueueJob(Func> func) 37 | { 38 | queue.Add(Task.Run(func)); 39 | } 40 | 41 | /// 42 | /// Dequeue the result returned by the enqueued task. This method blocks if the queue is empty. 43 | /// 44 | /// Result from enqueued task. 45 | public T DequeueResult() 46 | { 47 | return queue.Take().Result; 48 | } 49 | 50 | /// 51 | /// Set this queue to CompleteAdding. Call to DequeueResult throws InvalidOperationException afterwards. 52 | /// 53 | public void CompleteAdding() 54 | { 55 | this.queue.CompleteAdding(); 56 | } 57 | 58 | #region IDisposable Support 59 | private bool disposedValue = false; // To detect redundant calls 60 | 61 | protected virtual void Dispose(bool disposing) 62 | { 63 | if (!disposedValue) 64 | { 65 | if (disposing) 66 | { 67 | this.queue.Dispose(); 68 | this.queue = null; 69 | } 70 | 71 | disposedValue = true; 72 | } 73 | } 74 | 75 | public void Dispose() 76 | { 77 | Dispose(true); 78 | } 79 | #endregion 80 | } 81 | } 82 | --------------------------------------------------------------------------------