├── azbfs ├── version.go ├── zz_generated_version.go ├── zt_hiddenfuncs_test.go ├── path_extensions.go ├── zc_mmf_unix.go ├── zc_filesystemRequestOptions.go ├── zc_fileRequestOptions.go ├── datalake_storage_transforms.go ├── zc_directoryRequestOptions.go ├── zc_policy_unique_request_id.go ├── README.md ├── zz_generated_client.go ├── zc_mmf_windows.go ├── zc_pipeline.go ├── zc_policy_telemetry.go ├── zc_credential_anonymous.go ├── zc_uuid.go ├── zc_util_validate.go ├── zz_generated_responder_policy.go ├── zt_url_filesystem_test.go ├── zc_storage_error.go └── url_service.go ├── testSuite-exec ├── readme-command-prompt.png ├── testSuite ├── main.go ├── cmd │ ├── platdiff_unix.go │ ├── platdiff_windows.go │ ├── mmap_unix.go │ ├── info.go │ ├── mmap_windows.go │ ├── common.go │ └── root.go └── scripts │ ├── test_clfsload.py │ ├── test_azcopy_operations.py │ └── test_blobfs_upload_SAS.py ├── cmd ├── zc_traverser_local_other.go ├── zc_pipeline_init.go ├── env.go ├── jobs.go ├── zt_sync_cmd_test.go ├── zt_remove_copy_test.go ├── load.go ├── zc_attr_filter_notwin.go ├── zc_traverser_local_test.go ├── zt_block_size_to_bytes_test.go ├── logout.go ├── zt_scenario_helpers_for_windows_test.go ├── zc_traverser_local_windows.go ├── zt_parseSize_test.go ├── copyEnumeratorHelper_test.go ├── jobsList_test.go ├── zt_user_input_test.go ├── loadCLFS_test.go ├── doc.go ├── bytesizetostring_test.go ├── zt_pathUtils_test.go ├── removeProcessor.go ├── loginStatus.go ├── versionChecker_test.go └── pause.go ├── common ├── version.go ├── credCacheGnomeKeyringShim_linux.go ├── osOpen_fallback.go ├── folderCreationTracker_interface.go ├── nocopy.go ├── CountPerSecond.go ├── LongPathHandler_test.go ├── access.go ├── parallel │ └── zt_FileSystemCrawlerTest_windowsInit_test.go ├── gcpModels.go ├── credCacheModel.go ├── osOpen_windows.go ├── credCacheGnomeKeyring_linux.go ├── nullHasher.go ├── uuid_test.go ├── gcpURLParts_test.go ├── prologueState.go ├── azError.go ├── uuid.go ├── emptyChunkReader.go ├── zt_multiSliceBytePooler_test.go ├── writeThoughFile_linux.go ├── writeThroughFile_darwin.go ├── zt_exclusiveStringMap_test.go └── s3Models.go ├── website └── .editorconfig ├── tool_clean.sh ├── CODE_OF_CONDUCT.md ├── .github └── ISSUE_TEMPLATE.md ├── ste ├── joblog_lcm_wrapper.go ├── concurrency_test.go ├── ErrorExt.go ├── putListNeed.go ├── pacer-nullAutoPacer.go ├── sender_pageBlobFromURL_test.go ├── sourceInfoProvider-Benchmark.go ├── remoteObjectExists.go ├── zt_ste_misc_windows_test.go ├── sender_blockBlob_test.go ├── mgr-JobPartMgr_test.go ├── sourceInfoProvider-Local.go ├── pacedReadSeeker.go └── downloader.go ├── e2etest ├── zt_copy_file_smb_test.go ├── zt_blob_versioning_test.go ├── zt_logging_test.go ├── zt_preserve_names_test.go ├── zt_managed_disks_test.go ├── zt_overwrite_test.go ├── zt_resume_windows_test.go ├── zt_enumeration_other_test.go ├── azcopyDebugHelper.go ├── scenario_os_helpers.go ├── zt_naming_file_and_folder_test.go └── zt_error_handling_test.go ├── LICENSE ├── sddl ├── sidTranslation_other.go └── sidTranslation_windows.go ├── tool_test_perf.sh ├── go.mod ├── tool_distributed_mutex.py └── main_windows.go /azbfs/version.go: -------------------------------------------------------------------------------- 1 | package azbfs 2 | 3 | const serviceLibVersion = "0.1" 4 | -------------------------------------------------------------------------------- /testSuite-exec: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nelsonjchen/azure-storage-azcopy/main/testSuite-exec -------------------------------------------------------------------------------- /readme-command-prompt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nelsonjchen/azure-storage-azcopy/main/readme-command-prompt.png -------------------------------------------------------------------------------- /testSuite/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/Azure/azure-storage-azcopy/v10/testSuite/cmd" 5 | ) 6 | 7 | func main() { 8 | cmd.Execute() 9 | } 10 | -------------------------------------------------------------------------------- /cmd/zc_traverser_local_other.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | package cmd 4 | 5 | import ( 6 | "os" 7 | ) 8 | 9 | func WrapFolder(fullpath string, stat os.FileInfo) (os.FileInfo, error) { 10 | return stat, nil 11 | } -------------------------------------------------------------------------------- /common/version.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | const AzcopyVersion = "10.14.0" 4 | const UserAgent = "AzCopy/" + AzcopyVersion 5 | const S3ImportUserAgent = "S3Import " + UserAgent 6 | const GCPImportUserAgent = "GCPImport " + UserAgent 7 | const BenchmarkUserAgent = "Benchmark " + UserAgent 8 | -------------------------------------------------------------------------------- /website/.editorconfig: -------------------------------------------------------------------------------- 1 | # Editor configuration, see https://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | charset = utf-8 6 | indent_style = space 7 | indent_size = 4 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [*.md] 12 | max_line_length = off 13 | trim_trailing_whitespace = false 14 | -------------------------------------------------------------------------------- /tool_clean.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # find and kill the ste 4 | # no longer useful since azcopy doesn't run in the background mode any more 5 | # the [] around 'a' is there so that grep doesn't find itself, it is a known trick 6 | # kill $(ps aux | grep '[a]zcopy' | awk '{print $2}') 7 | 8 | # remove logs and memory mapped plan files 9 | rm -rf ~/.azcopy/* 10 | rm ./azcopy* -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /common/credCacheGnomeKeyringShim_linux.go: -------------------------------------------------------------------------------- 1 | // +build !se_integration 2 | // For public version azcopy, gnome keyring is not necessary, and no need to 3 | // involve additional dependencies to libsecret-1 and glib-2.0 4 | 5 | package common 6 | 7 | import "errors" 8 | 9 | type gnomeKeyring struct{} 10 | 11 | func (p gnomeKeyring) Get(Service string, Account string) (string, error) { 12 | // By design, not useful for non integration scenario. 13 | return "", errors.New("Not implemented") 14 | } 15 | -------------------------------------------------------------------------------- /azbfs/zz_generated_version.go: -------------------------------------------------------------------------------- 1 | package azbfs 2 | 3 | // Code generated by Microsoft (R) AutoRest Code Generator. 4 | // Changes may cause incorrect behavior and will be lost if the code is regenerated. 5 | 6 | // UserAgent returns the UserAgent string to use when sending http.Requests. 7 | func UserAgent() string { 8 | return "Azure-SDK-For-Go/0.0.0 azbfs/2018-11-09" 9 | } 10 | 11 | // Version returns the semantic version (see http://semver.org) of the client. 12 | func Version() string { 13 | return "0.0.0" 14 | } 15 | -------------------------------------------------------------------------------- /azbfs/zt_hiddenfuncs_test.go: -------------------------------------------------------------------------------- 1 | package azbfs 2 | 3 | // This file isn't normally compiled when not testing. 4 | // Therefore, since we're in the azbfs package, we can export a method to help us construct a failing retry reader options. 5 | 6 | func InjectErrorInRetryReaderOptions(err error) RetryReaderOptions { 7 | return RetryReaderOptions{ 8 | MaxRetryRequests: 1, 9 | doInjectError: true, 10 | doInjectErrorRound: 0, 11 | injectedError: err, 12 | NotifyFailedRead: nil, 13 | TreatEarlyCloseAsError: false, 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Which version of the AzCopy was used? 2 | ##### Note: The version is visible when running AzCopy without any argument 3 | 4 | ### Which platform are you using? (ex: Windows, Mac, Linux) 5 | 6 | ### What command did you run? 7 | ##### Note: Please remove the SAS to avoid exposing your credentials. If you cannot remember the exact command, please retrieve it from the beginning of the log file. 8 | 9 | ### What problem was encountered? 10 | 11 | ### How can we reproduce the problem in the simplest way? 12 | 13 | ### Have you found a mitigation/solution? 14 | -------------------------------------------------------------------------------- /testSuite/cmd/platdiff_unix.go: -------------------------------------------------------------------------------- 1 | // +build linux darwin 2 | 3 | package cmd 4 | 5 | import ( 6 | "os" 7 | "path" 8 | ) 9 | 10 | // GetAzCopyAppPath returns the path of Azcopy folder in local appdata. 11 | // Azcopy folder in local appdata contains all the files created by azcopy locally. 12 | func GetAzCopyAppPath() string { 13 | localAppData := os.Getenv("HOME") 14 | azcopyAppDataFolder := path.Join(localAppData, "/.azcopy") 15 | if err := os.Mkdir(azcopyAppDataFolder, os.ModeDir|os.ModePerm); err != nil && !os.IsExist(err) { 16 | return "" 17 | } 18 | return azcopyAppDataFolder 19 | } 20 | -------------------------------------------------------------------------------- /testSuite/cmd/platdiff_windows.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | "path" 6 | 7 | "github.com/Azure/azure-storage-azcopy/v10/common" 8 | ) 9 | 10 | // GetAzCopyAppPath returns the path of Azcopy in local appdata. 11 | func GetAzCopyAppPath() string { 12 | lcm := common.GetLifecycleMgr() 13 | userProfile := lcm.GetEnvironmentVariable(common.EEnvironmentVariable.UserDir()) 14 | azcopyAppDataFolder := path.Join(userProfile, ".azcopy") 15 | if err := os.Mkdir(azcopyAppDataFolder, os.ModeDir); err != nil && !os.IsExist(err) { 16 | return "" 17 | } 18 | return azcopyAppDataFolder 19 | } 20 | -------------------------------------------------------------------------------- /azbfs/path_extensions.go: -------------------------------------------------------------------------------- 1 | package azbfs 2 | 3 | import ( 4 | "encoding/base64" 5 | "time" 6 | ) 7 | 8 | func (p Path) LastModifiedTime() time.Time { 9 | if p.LastModified == nil { 10 | return time.Time{} 11 | } 12 | 13 | t, err := time.Parse(time.RFC1123, *p.LastModified) 14 | if err != nil { 15 | return time.Time{} 16 | } 17 | 18 | return t 19 | } 20 | 21 | func (p Path) ContentMD5() []byte { 22 | if p.ContentMD5Base64 == nil { 23 | return nil 24 | } 25 | 26 | md5, err := base64.StdEncoding.DecodeString(*p.ContentMD5Base64) 27 | if err != nil { 28 | return nil 29 | } 30 | return md5 31 | } 32 | -------------------------------------------------------------------------------- /common/osOpen_fallback.go: -------------------------------------------------------------------------------- 1 | // Be certain to add the build tags below when we use a specialized implementation. 2 | // This file contains forwards to default, fallback implementations of os operations 3 | // +build !windows 4 | 5 | package common 6 | 7 | import ( 8 | "os" 9 | ) 10 | 11 | // NOTE: OSOpenFile not safe to use on directories on Windows. See comment on the Windows version of this routine 12 | func OSOpenFile(name string, flag int, perm os.FileMode) (*os.File, error) { 13 | return os.OpenFile(name, flag, perm) 14 | } 15 | 16 | func OSStat(name string) (os.FileInfo, error) { 17 | return os.Stat(name) 18 | } 19 | -------------------------------------------------------------------------------- /azbfs/zc_mmf_unix.go: -------------------------------------------------------------------------------- 1 | // +build linux darwin 2 | 3 | package azbfs 4 | 5 | import ( 6 | "os" 7 | "syscall" 8 | ) 9 | 10 | type mmf []byte 11 | 12 | func newMMF(file *os.File, writable bool, offset int64, length int) (mmf, error) { 13 | prot, flags := syscall.PROT_READ, syscall.MAP_SHARED // Assume read-only 14 | if writable { 15 | prot, flags = syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED 16 | } 17 | addr, err := syscall.Mmap(int(file.Fd()), offset, length, prot, flags) 18 | return mmf(addr), err 19 | } 20 | 21 | func (m *mmf) unmap() { 22 | err := syscall.Munmap(*m) 23 | *m = nil 24 | if err != nil { 25 | panic(err) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /testSuite/cmd/mmap_unix.go: -------------------------------------------------------------------------------- 1 | // +build linux darwin 2 | 3 | package cmd 4 | 5 | import ( 6 | "os" 7 | "syscall" 8 | ) 9 | 10 | type MMF []byte 11 | 12 | func NewMMF(file *os.File, writable bool, offset int64, length int64) (MMF, error) { 13 | prot, flags := syscall.PROT_READ, syscall.MAP_SHARED // Assume read-only 14 | if writable { 15 | prot, flags = syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED 16 | } 17 | addr, err := syscall.Mmap(int(file.Fd()), offset, int(length), prot, flags) 18 | return MMF(addr), err 19 | } 20 | 21 | func (m *MMF) Unmap() { 22 | err := syscall.Munmap(*m) 23 | *m = nil 24 | if err != nil { 25 | panic(err) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /common/folderCreationTracker_interface.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | // folderCreationTracker is used to ensure than in an overwrite=false situation we 4 | // only set folder properties on folders which were created by the current job. (To be consistent 5 | // with the fact that when overwrite == false, we only set file properties on files created 6 | // by the current job) 7 | type FolderCreationTracker interface { 8 | RecordCreation(folder string) 9 | ShouldSetProperties(folder string, overwrite OverwriteOption, prompter Prompter) bool 10 | StopTracking(folder string) 11 | } 12 | 13 | type Prompter interface { 14 | ShouldOverwrite(objectPath string, objectType EntityType) bool 15 | } 16 | -------------------------------------------------------------------------------- /azbfs/zc_filesystemRequestOptions.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT License. 3 | 4 | package azbfs 5 | 6 | // For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/list. 7 | type ListPathsFilesystemOptions struct { 8 | // Filters results to paths in this directory. 9 | Path *string 10 | // Whether or not to list recursively. 11 | Recursive bool 12 | // Whether or not AAD Object IDs will be converted to user principal name. 13 | UpnReturned *bool 14 | // The maximum number of items to return. 15 | MaxResults *int32 16 | // The continuation token to resume listing. 17 | ContinuationToken *string 18 | } -------------------------------------------------------------------------------- /ste/joblog_lcm_wrapper.go: -------------------------------------------------------------------------------- 1 | package ste 2 | 3 | import ( 4 | "github.com/Azure/azure-pipeline-go/pipeline" 5 | 6 | "github.com/Azure/azure-storage-azcopy/v10/common" 7 | ) 8 | 9 | type jobLogLCMWrapper struct { 10 | jobManager IJobMgr 11 | common.LifecycleMgr 12 | } 13 | 14 | func (j jobLogLCMWrapper) Progress(builder common.OutputBuilder) { 15 | builderWrapper := func(format common.OutputFormat) string { 16 | if format != common.EOutputFormat.Text() { 17 | j.jobManager.Log(pipeline.LogInfo, builder(common.EOutputFormat.Text())) 18 | return builder(format) 19 | } else { 20 | output := builder(common.EOutputFormat.Text()) 21 | j.jobManager.Log(pipeline.LogInfo, output) 22 | return output 23 | } 24 | } 25 | 26 | j.LifecycleMgr.Progress(builderWrapper) 27 | } 28 | -------------------------------------------------------------------------------- /e2etest/zt_copy_file_smb_test.go: -------------------------------------------------------------------------------- 1 | package e2etest 2 | 3 | import ( 4 | "github.com/Azure/azure-storage-azcopy/v10/common" 5 | "testing" 6 | ) 7 | 8 | func TestSMB_FromShareSnapshot(t *testing.T) { 9 | RunScenarios(t, eOperation.Copy(), eTestFromTo.Other(common.EFromTo.FileFile()), eValidate.AutoPlusContent(), params{ 10 | recursive: true, 11 | preserveSMBInfo: true, 12 | preserveSMBPermissions: true, 13 | }, &hooks{ 14 | // create a snapshot for the source share 15 | beforeRunJob: func(h hookHelper) { 16 | h.CreateSourceSnapshot() 17 | }, 18 | }, testFiles{ 19 | defaultSize: "4M", 20 | shouldTransfer: []interface{}{ 21 | folder(""), // root folder 22 | folder("folder1"), 23 | f("folder1/filea"), 24 | }, 25 | }, EAccountType.Standard(), EAccountType.Standard(), "") 26 | } 27 | -------------------------------------------------------------------------------- /common/nocopy.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "errors" 4 | 5 | // The NoCopy struct is used as a field inside another struct that should not be copied by value. 6 | // After embedded this field, the out struct's members can call the Check method which will panic 7 | // if it detects the out struct has been copied by value. 8 | type NoCopy struct { 9 | nocopy *NoCopy 10 | } 11 | 12 | // Check panics if the struct embedded this NoCopy field has been copied by value. 13 | func (nc *NoCopy) Check() { 14 | if nc.nocopy == nc { 15 | return // The reference matches the 1st-time reference 16 | } 17 | if nc.nocopy == nil { // The reference was never set, set it 18 | nc.nocopy = nc 19 | return 20 | } 21 | // The receiver's reference doesn't match the persisted reference; this must be a copy 22 | panic(errors.New("nocopy detected copy by value")) 23 | } 24 | -------------------------------------------------------------------------------- /azbfs/zc_fileRequestOptions.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT License. 3 | 4 | package azbfs 5 | 6 | // For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/create. 7 | type CreateFileOptions struct { 8 | // Custom headers to apply to the file. 9 | Headers BlobFSHTTPHeaders 10 | // User defined properties to be stored with the file. 11 | Metadata map[string]string 12 | } 13 | 14 | // For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/create. 15 | type RenameFileOptions struct { 16 | // The optional destination file system for the file. 17 | DestinationFileSystem *string 18 | // The destination path for the file. 19 | DestinationPath string 20 | // The new SAS for a destination file 21 | DestinationSas *string 22 | } 23 | -------------------------------------------------------------------------------- /azbfs/datalake_storage_transforms.go: -------------------------------------------------------------------------------- 1 | package azbfs 2 | 3 | import ( 4 | "encoding/base64" 5 | "strings" 6 | ) 7 | 8 | // Converts metadata into a string of format "key1=value1, key2=value2" and Base64 encodes the values. 9 | func buildMetadataString(md map[string]string) *string { 10 | if md == nil { 11 | return nil 12 | } 13 | var sb strings.Builder 14 | var i int 15 | for key, value := range md { 16 | sb.WriteString(key) 17 | sb.WriteRune('=') 18 | /* 19 | The service has an internal base64 decode when metadata is copied from ADLS to Storage, so getMetadata 20 | will work as normal. Doing this encoding for the customers preserves the existing behavior of 21 | metadata. 22 | */ 23 | sb.WriteString(base64.StdEncoding.EncodeToString([]byte(value))) 24 | if i != len(md)-1 { 25 | sb.WriteRune(',') 26 | } 27 | i += 1 28 | } 29 | mdStr := sb.String() 30 | return &mdStr 31 | } 32 | -------------------------------------------------------------------------------- /testSuite/cmd/info.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | var infoType string 10 | 11 | var defaultInfoType = "AzCopyAppPath" 12 | 13 | // initializes the info command, its aliases and description. 14 | func init() { 15 | infoCmd := &cobra.Command{ 16 | Use: "info", 17 | Aliases: []string{"info"}, 18 | Short: "info gets AzCopy related info.", 19 | 20 | Args: func(cmd *cobra.Command, args []string) error { 21 | if len(args) > 1 { 22 | return fmt.Errorf("invalid arguments for info command") 23 | } 24 | infoType = args[0] 25 | return nil 26 | }, 27 | Run: func(cmd *cobra.Command, args []string) { 28 | if infoType == defaultInfoType { 29 | fmt.Print(GetAzCopyAppPath()) 30 | } 31 | }, 32 | } 33 | rootCmd.AddCommand(infoCmd) 34 | 35 | infoCmd.PersistentFlags().StringVar(&infoType, "infoType", defaultInfoType, "info to get.") 36 | } 37 | -------------------------------------------------------------------------------- /azbfs/zc_directoryRequestOptions.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT License. 3 | 4 | package azbfs 5 | 6 | // For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/create. 7 | type CreateDirectoryOptions struct { 8 | // Whether or not to recreate the directory if it exists. 9 | RecreateIfExists bool 10 | // User defined properties to be stored with the directory. 11 | Metadata map[string]string 12 | } 13 | 14 | // For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/create. 15 | type RenameDirectoryOptions struct { 16 | // The optional destination file system for the directory. 17 | DestinationFileSystem *string 18 | // The destination path for the directory. 19 | DestinationPath string 20 | // The new SAS for a destination Directory 21 | DestinationSas *string 22 | } 23 | -------------------------------------------------------------------------------- /ste/concurrency_test.go: -------------------------------------------------------------------------------- 1 | package ste 2 | 3 | import ( 4 | chk "gopkg.in/check.v1" 5 | ) 6 | 7 | type mainTestSuite struct{} 8 | 9 | var _ = chk.Suite(&mainTestSuite{}) 10 | 11 | const ( 12 | minConcurrency = 32 13 | maxConcurrency = 300 14 | ) 15 | 16 | func (s *mainTestSuite) TestConcurrencyValue(c *chk.C) { 17 | // weak machines 18 | for i := 1; i < 5; i++ { 19 | min, max := getMainPoolSize(i, false) 20 | c.Assert(min, chk.Equals, minConcurrency) 21 | c.Assert(max.Value, chk.Equals, minConcurrency) 22 | } 23 | 24 | // moderately powerful machines 25 | for i := 5; i < 19; i++ { 26 | min, max := getMainPoolSize(i, false) 27 | c.Assert(min, chk.Equals, 16*i) 28 | c.Assert(max.Value, chk.Equals, 16*i) 29 | } 30 | 31 | // powerful machines 32 | for i := 19; i < 24; i++ { 33 | min, max := getMainPoolSize(i, false) 34 | c.Assert(min, chk.Equals, maxConcurrency) 35 | c.Assert(max.Value, chk.Equals, maxConcurrency) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /azbfs/zc_policy_unique_request_id.go: -------------------------------------------------------------------------------- 1 | package azbfs 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/Azure/azure-pipeline-go/pipeline" 7 | ) 8 | 9 | // NewUniqueRequestIDPolicyFactory creates a UniqueRequestIDPolicyFactory object 10 | // that sets the request's x-ms-client-request-id header if it doesn't already exist. 11 | func NewUniqueRequestIDPolicyFactory() pipeline.Factory { 12 | return pipeline.FactoryFunc(func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc { 13 | // This is Policy's Do method: 14 | return func(ctx context.Context, request pipeline.Request) (pipeline.Response, error) { 15 | id := request.Header.Get(xMsClientRequestID) 16 | if id == "" { // Add a unique request ID if the caller didn't specify one already 17 | request.Header.Set(xMsClientRequestID, newUUID().String()) 18 | } 19 | return next.Do(ctx, request) 20 | } 21 | }) 22 | } 23 | 24 | const xMsClientRequestID = "x-ms-client-request-id" 25 | -------------------------------------------------------------------------------- /azbfs/README.md: -------------------------------------------------------------------------------- 1 | # Azcopy - azbfs 2 | 3 | azbfs is an internal only package for Azcopy that implements REST APIs for HNS enabled accounts. 4 | 5 | # Generation 6 | # Azure Data Lakes Gen 2 for Golang 7 | > see [https://aka.ms/autorest](https://aka.ms/autorest) 8 | 9 | ## [Installation Instructions](https://github.com/Azure/autorest/blob/main/docs/install/readme.md) 10 | 1. Install [Node js](https://github.com/nodesource/distributions/blob/master/README.md#installation-instructions) version 12.18.2 11 | 2. Install autorest using npm 12 | ```bash 13 | sudo apt install npm 14 | npm install -g autorest 15 | # run using command 'autorest' to check if installation worked 16 | autorest --help 17 | ``` 18 | 19 | ## Generation Instructions. 20 | From the root of azure-storage-azcopy, run the following commands 21 | ```bash 22 | cd azbfs 23 | sudo autorest --use=@microsoft.azure/autorest.go@v3.0.63 --input-file=./azure_dfs_swagger_manually_edited.json --go=true --output-folder=./ --namespace=azbfs --go-export-clients=false --file-prefix=zz_generated_ 24 | cd .. 25 | gofmt -w azbfs 26 | ``` -------------------------------------------------------------------------------- /cmd/zc_pipeline_init.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/Azure/azure-pipeline-go/pipeline" 8 | 9 | "github.com/Azure/azure-storage-azcopy/v10/common" 10 | ) 11 | 12 | func InitPipeline(ctx context.Context, location common.Location, credential common.CredentialInfo, logLevel pipeline.LogLevel) (p pipeline.Pipeline, err error) { 13 | switch location { 14 | case common.ELocation.Local(), 15 | common.ELocation.Benchmark(): 16 | // Gracefully return 17 | return nil, nil 18 | case common.ELocation.Blob(): 19 | p, err = createBlobPipeline(ctx, credential, logLevel) 20 | case common.ELocation.File(): 21 | p, err = createFilePipeline(ctx, credential, logLevel) 22 | case common.ELocation.BlobFS(): 23 | p, err = createBlobFSPipeline(ctx, credential, logLevel) 24 | case common.ELocation.S3(): 25 | case common.ELocation.GCP(): 26 | // Gracefully return because pipelines aren't used for S3 or GCP 27 | return nil, nil 28 | default: 29 | err = fmt.Errorf("can't produce new pipeline for location %s", location) 30 | } 31 | 32 | return 33 | } 34 | -------------------------------------------------------------------------------- /azbfs/zz_generated_client.go: -------------------------------------------------------------------------------- 1 | package azbfs 2 | 3 | // Code generated by Microsoft (R) AutoRest Code Generator. 4 | // Changes may cause incorrect behavior and will be lost if the code is regenerated. 5 | 6 | import ( 7 | "github.com/Azure/azure-pipeline-go/pipeline" 8 | "net/url" 9 | ) 10 | 11 | const ( 12 | // ServiceVersion specifies the version of the operations used in this package. 13 | ServiceVersion = "2018-11-09" 14 | ) 15 | 16 | // managementClient is the base client for Azbfs. 17 | type managementClient struct { 18 | url url.URL 19 | p pipeline.Pipeline 20 | } 21 | 22 | // newManagementClient creates an instance of the managementClient client. 23 | func newManagementClient(url url.URL, p pipeline.Pipeline) managementClient { 24 | return managementClient{ 25 | url: url, 26 | p: p, 27 | } 28 | } 29 | 30 | // URL returns a copy of the URL for this client. 31 | func (mc managementClient) URL() url.URL { 32 | return mc.url 33 | } 34 | 35 | // Pipeline returns the pipeline for this client. 36 | func (mc managementClient) Pipeline() pipeline.Pipeline { 37 | return mc.p 38 | } 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright © 2017 Microsoft 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /testSuite/cmd/mmap_windows.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | "reflect" 6 | "syscall" 7 | "unsafe" 8 | ) 9 | 10 | type MMF []byte 11 | 12 | func NewMMF(file *os.File, writable bool, offset int64, length int64) (MMF, error) { 13 | prot, access := uint32(syscall.PAGE_READONLY), uint32(syscall.FILE_MAP_READ) // Assume read-only 14 | if writable { 15 | prot, access = uint32(syscall.PAGE_READWRITE), uint32(syscall.FILE_MAP_WRITE) 16 | } 17 | hMMF, errno := syscall.CreateFileMapping(syscall.Handle(file.Fd()), nil, prot, uint32(int64(length)>>32), uint32(int64(length)&0xffffffff), nil) 18 | if hMMF == 0 { 19 | return nil, os.NewSyscallError("CreateFileMapping", errno) 20 | } 21 | defer syscall.CloseHandle(hMMF) 22 | addr, errno := syscall.MapViewOfFile(hMMF, access, uint32(offset>>32), uint32(offset&0xffffffff), uintptr(length)) 23 | m := MMF{} 24 | h := (*reflect.SliceHeader)(unsafe.Pointer(&m)) 25 | h.Data = addr 26 | h.Len = int(length) 27 | h.Cap = h.Len 28 | return m, nil 29 | } 30 | 31 | func (m *MMF) Unmap() { 32 | addr := uintptr(unsafe.Pointer(&(([]byte)(*m)[0]))) 33 | *m = MMF{} 34 | err := syscall.UnmapViewOfFile(addr) 35 | if err != nil { 36 | panic(err) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /azbfs/zc_mmf_windows.go: -------------------------------------------------------------------------------- 1 | package azbfs 2 | 3 | import ( 4 | "os" 5 | "reflect" 6 | "syscall" 7 | "unsafe" 8 | ) 9 | 10 | type mmf []byte 11 | 12 | func newMMF(file *os.File, writable bool, offset int64, length int) (mmf, error) { 13 | prot, access := uint32(syscall.PAGE_READONLY), uint32(syscall.FILE_MAP_READ) // Assume read-only 14 | if writable { 15 | prot, access = uint32(syscall.PAGE_READWRITE), uint32(syscall.FILE_MAP_WRITE) 16 | } 17 | maxSize := int64(offset + int64(length)) 18 | hMMF, errno := syscall.CreateFileMapping(syscall.Handle(file.Fd()), nil, prot, uint32(maxSize>>32), uint32(maxSize&0xffffffff), nil) 19 | if hMMF == 0 { 20 | return nil, os.NewSyscallError("CreateFileMapping", errno) 21 | } 22 | defer syscall.CloseHandle(hMMF) 23 | addr, errno := syscall.MapViewOfFile(hMMF, access, uint32(offset>>32), uint32(offset&0xffffffff), uintptr(length)) 24 | m := mmf{} 25 | h := (*reflect.SliceHeader)(unsafe.Pointer(&m)) 26 | h.Data = addr 27 | h.Len = length 28 | h.Cap = h.Len 29 | return m, nil 30 | } 31 | 32 | func (m *mmf) unmap() { 33 | addr := uintptr(unsafe.Pointer(&(([]byte)(*m)[0]))) 34 | *m = mmf{} 35 | err := syscall.UnmapViewOfFile(addr) 36 | if err != nil { 37 | panic(err) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /e2etest/zt_blob_versioning_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package e2etest 22 | -------------------------------------------------------------------------------- /common/CountPerSecond.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "sync/atomic" 5 | "time" 6 | ) 7 | 8 | func NewCountPerSecond() CountPerSecond { 9 | cps := countPerSecond{} 10 | cps.Reset() 11 | return &cps 12 | } 13 | 14 | // CountPerSecond ... 15 | type CountPerSecond interface { 16 | // Add atomically adds delta to *addr and returns the new value. 17 | // To subtract a signed positive constant value c, do Add(^uint64(c-1)). 18 | Add(delta uint64) uint64 // Pass 0 to get the current count value 19 | LatestRate() float64 20 | Reset() 21 | } 22 | 23 | type countPerSecond struct { 24 | start int64 // Unix time allowing atomic update: Seconds since 1/1/1970 25 | count uint64 26 | nocopy NoCopy 27 | } 28 | 29 | func (cps *countPerSecond) Add(delta uint64) uint64 { 30 | cps.nocopy.Check() 31 | return atomic.AddUint64(&cps.count, delta) 32 | } 33 | 34 | func (cps *countPerSecond) LatestRate() float64 { 35 | cps.nocopy.Check() 36 | dur := time.Now().Sub(time.Unix(cps.start, 0)) 37 | if dur <= 0 { 38 | dur = 1 39 | } 40 | return float64(atomic.LoadUint64(&cps.count)) / dur.Seconds() 41 | } 42 | 43 | func (cps *countPerSecond) Reset() { 44 | cps.nocopy.Check() 45 | atomic.StoreInt64(&cps.start, time.Now().Unix()) 46 | atomic.StoreUint64(&cps.count, 0) 47 | } 48 | -------------------------------------------------------------------------------- /testSuite/scripts/test_clfsload.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | 4 | import utility as util 5 | 6 | 7 | # Temporary tests to guarantee simple load scenario works 8 | class LoadUserScenario(unittest.TestCase): 9 | 10 | def test_load_entire_directory(self): 11 | dir_name = "dir_load_test" 12 | dir_path = util.create_test_n_files(1024, 10, dir_name) 13 | 14 | # create sub-directory inside directory 15 | sub_dir_name = os.path.join(dir_name, "sub_dir_load_test") 16 | util.create_test_n_files(1024, 10, sub_dir_name) 17 | 18 | # clean out the container 19 | # execute azcopy command 20 | # ignore the error since the container might be already empty 21 | util.Command("rm").add_arguments(util.test_container_url). \ 22 | add_flags("recursive", "true").execute_azcopy_copy_command() 23 | 24 | # invoke the load command 25 | state_path = os.path.join(util.test_directory_path, "clfsload-state") 26 | result = util.Command("load clfs").add_arguments(dir_path).add_arguments(util.test_container_url). \ 27 | add_flags("max-errors", "8").add_flags("state-path", state_path).add_flags("preserve-hardlinks", "true") \ 28 | .add_flags("compression-type", "LZ4").execute_azcopy_copy_command() 29 | self.assertTrue(result) 30 | -------------------------------------------------------------------------------- /common/LongPathHandler_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | chk "gopkg.in/check.v1" 5 | ) 6 | 7 | type pathHandlerSuite struct{} 8 | 9 | var _ = chk.Suite(&pathHandlerSuite{}) 10 | 11 | func (p *pathHandlerSuite) TestShortToLong(c *chk.C) { 12 | if OS_PATH_SEPARATOR == `\` { 13 | c.Assert(ToExtendedPath(`c:`), chk.Equals, `\\?\C:\`) 14 | c.Assert(ToExtendedPath(`c:/`), chk.Equals, `\\?\C:\`) 15 | c.Assert(ToExtendedPath(`c:/myPath`), chk.Equals, `\\?\C:\myPath`) 16 | c.Assert(ToExtendedPath(`C:\myPath`), chk.Equals, `\\?\C:\myPath`) 17 | c.Assert(ToExtendedPath(`\\myHost\myPath`), chk.Equals, `\\?\UNC\myHost\myPath`) 18 | c.Assert(ToExtendedPath(`\\?\C:\myPath`), chk.Equals, `\\?\C:\myPath`) 19 | c.Assert(ToExtendedPath(`\\?\UNC\myHost\myPath`), chk.Equals, `\\?\UNC\myHost\myPath`) 20 | } else { 21 | c.Skip("Test only pertains to Windows.") 22 | } 23 | } 24 | 25 | func (p *pathHandlerSuite) TestLongToShort(c *chk.C) { 26 | if OS_PATH_SEPARATOR == `\` { 27 | c.Assert(ToShortPath(`\\?\C:\myPath`), chk.Equals, `C:\myPath`) 28 | c.Assert(ToShortPath(`\\?\UNC\myHost\myPath`), chk.Equals, `\\myHost\myPath`) 29 | c.Assert(ToShortPath(`\\myHost\myPath`), chk.Equals, `\\myHost\myPath`) 30 | c.Assert(ToShortPath(`C:\myPath`), chk.Equals, `C:\myPath`) 31 | } else { 32 | c.Skip("Test only pertains to Windows.") 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /e2etest/zt_logging_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package e2etest 22 | 23 | // Purpose: Tests specifically about logging? E.g. redaction of SAS tokens? Do we even need logging tests tho? 24 | -------------------------------------------------------------------------------- /common/access.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "context" 5 | "net/url" 6 | 7 | "github.com/Azure/azure-storage-blob-go/azblob" 8 | ) 9 | 10 | func IsSourcePublicBlob(sourceURI string, ctx context.Context) bool { 11 | uri, err := url.Parse(sourceURI) 12 | if err != nil { 13 | // this should never, would never be hit. 14 | // a job plan file couldn't be created by AzCopy with an invalid URI. 15 | panic("Source URI was invalid.") 16 | } 17 | 18 | blobParts := azblob.NewBlobURLParts(*uri) 19 | 20 | // only containers can be public access 21 | if blobParts.ContainerName != "" { 22 | if blobParts.BlobName != "" { 23 | // first test that it's a blob 24 | bURL := azblob.NewBlobURL(*uri, azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{})) 25 | _, err := bURL.GetProperties(ctx, azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) 26 | if err == nil { 27 | return true 28 | } 29 | 30 | // since that failed, maybe it doesn't exist and is public to list? 31 | blobParts.BlobName = "" 32 | } 33 | 34 | cURL := azblob.NewContainerURL(blobParts.URL(), azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{})) 35 | 36 | _, err := cURL.ListBlobsFlatSegment(ctx, azblob.Marker{}, azblob.ListBlobsSegmentOptions{}) 37 | if err == nil { 38 | return true 39 | } 40 | } 41 | 42 | return false 43 | } 44 | -------------------------------------------------------------------------------- /e2etest/zt_preserve_names_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package e2etest 22 | 23 | // Purpose: Tests for checking we correctly escape or transform object names, when necessary. 24 | 25 | // TODO: from S3. To/from Windows/Azure Files. 26 | -------------------------------------------------------------------------------- /e2etest/zt_managed_disks_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package e2etest 22 | 23 | // Purpose: Tests for the special cases that relate to moving managed disks (default local VHD to page blob; special handling for 24 | // md- and md-impex URLs. 25 | -------------------------------------------------------------------------------- /common/parallel/zt_FileSystemCrawlerTest_windowsInit_test.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | // Copyright © 2017 Microsoft 3 | // 4 | // Permission is hereby granted, free of charge, to any person obtaining a copy 5 | // of this software and associated documentation files (the "Software"), to deal 6 | // in the Software without restriction, including without limitation the rights 7 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | // copies of the Software, and to permit persons to whom the Software is 9 | // furnished to do so, subject to the following conditions: 10 | // 11 | // The above copyright notice and this permission notice shall be included in 12 | // all copies or substantial portions of the Software. 13 | // 14 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | // THE SOFTWARE. 21 | 22 | package parallel 23 | 24 | import "golang.org/x/sys/windows" 25 | 26 | func init() { 27 | windowsSystemDirectory, _ = windows.GetSystemDirectory() 28 | } 29 | -------------------------------------------------------------------------------- /ste/ErrorExt.go: -------------------------------------------------------------------------------- 1 | package ste 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/Azure/azure-storage-azcopy/v10/azbfs" 7 | "github.com/Azure/azure-storage-blob-go/azblob" 8 | "github.com/Azure/azure-storage-file-go/azfile" 9 | ) 10 | 11 | type ErrorEx struct { 12 | error 13 | } 14 | 15 | // TODO: consider rolling MSRequestID into this, so that all places that use this can pick up, and log, the request ID too 16 | func (errex ErrorEx) ErrorCodeAndString() (string, int, string) { 17 | switch e := interface{}(errex.error).(type) { 18 | case azblob.StorageError: 19 | return string(e.ServiceCode()), e.Response().StatusCode, e.Response().Status 20 | case azfile.StorageError: 21 | return string(e.ServiceCode()), e.Response().StatusCode, e.Response().Status 22 | case azbfs.StorageError: 23 | return string(e.ServiceCode()), e.Response().StatusCode, e.Response().Status 24 | default: 25 | return "", 0, errex.Error() 26 | } 27 | } 28 | 29 | type hasResponse interface { 30 | Response() *http.Response 31 | } 32 | 33 | // MSRequestID gets the request ID guid associated with the failed request. 34 | // Returns "" if there isn't one (either no request, or there is a request but it doesn't have the header) 35 | func (errex ErrorEx) MSRequestID() string { 36 | if respErr, ok := errex.error.(hasResponse); ok { 37 | r := respErr.Response() 38 | if r != nil { 39 | return r.Header.Get("X-Ms-Request-Id") 40 | } 41 | } 42 | return "" 43 | } 44 | -------------------------------------------------------------------------------- /common/gcpModels.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | gcpUtils "cloud.google.com/go/storage" 5 | "strings" 6 | ) 7 | 8 | type GCPObjectInfoExtension struct { 9 | ObjectInfo gcpUtils.ObjectAttrs 10 | } 11 | 12 | func (gie *GCPObjectInfoExtension) ContentType() string { 13 | return gie.ObjectInfo.ContentType 14 | } 15 | 16 | func (gie *GCPObjectInfoExtension) CacheControl() string { 17 | return gie.ObjectInfo.CacheControl 18 | } 19 | 20 | func (gie *GCPObjectInfoExtension) ContentDisposition() string { 21 | return gie.ObjectInfo.ContentDisposition 22 | } 23 | 24 | func (gie *GCPObjectInfoExtension) ContentEncoding() string { 25 | return gie.ObjectInfo.ContentEncoding 26 | } 27 | 28 | func (gie *GCPObjectInfoExtension) ContentLanguage() string { 29 | return gie.ObjectInfo.ContentLanguage 30 | } 31 | 32 | func (gie *GCPObjectInfoExtension) ContentMD5() []byte { 33 | b := gie.ObjectInfo.MD5 34 | return b 35 | } 36 | 37 | const gcpMetadataPrefix = "x-goog-meta-" 38 | const gcpMetadataPrefixLen = len(gcpMetadataPrefix) 39 | 40 | //NewCommonMetadata returns a map of user-defined key/value pairs 41 | func (gie *GCPObjectInfoExtension) NewCommonMetadata() Metadata { 42 | md := Metadata{} 43 | for k, v := range gie.ObjectInfo.Metadata { 44 | if len(k) > gcpMetadataPrefixLen { 45 | if prefix := k[0:gcpMetadataPrefixLen]; strings.EqualFold(prefix, gcpMetadataPrefix) { 46 | md[k[gcpMetadataPrefixLen:]] = v 47 | } 48 | } 49 | } 50 | return md 51 | } 52 | -------------------------------------------------------------------------------- /sddl/sidTranslation_other.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | // Copyright © Microsoft 4 | // 5 | // Permission is hereby granted, free of charge, to any person obtaining a copy 6 | // of this software and associated documentation files (the "Software"), to deal 7 | // in the Software without restriction, including without limitation the rights 8 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | // copies of the Software, and to permit persons to whom the Software is 10 | // furnished to do so, subject to the following conditions: 11 | // 12 | // The above copyright notice and this permission notice shall be included in 13 | // all copies or substantial portions of the Software. 14 | // 15 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | // THE SOFTWARE. 22 | 23 | package sddl 24 | 25 | import ( 26 | "errors" 27 | ) 28 | 29 | // Note that all usages of TranslateSID gracefully handle the error, rather than throwing the error. 30 | func OSTranslateSID(SID string) (string, error) { 31 | return SID, errors.New("unsupported on this OS") 32 | } 33 | -------------------------------------------------------------------------------- /tool_test_perf.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # this script provides a quick way to validate whether a change causes a performance gain versus regression 4 | 5 | RunCurrent () { 6 | ./azcopy_current cp 'src' 'dst' --recursive --log-level=WARNING --check-length=false 7 | } 8 | 9 | RunNew () { 10 | ./azcopy_new cp 'src' 'dst' --recursive --log-level=WARNING --check-length=false 11 | } 12 | 13 | export AZCOPY_LOG_LOCATION=/datadrive/logs 14 | export AZCOPY_JOB_PLAN_LOCATION=/datadrive/plans 15 | 16 | for i in {1..5} 17 | do 18 | # start with the new version, which might give it a disadvantage 19 | # but since we run the experiment repeately, the results average out eventually 20 | # in addition, it's better to give the disadvantage to the new version so that we can be extra sure that it's better/equal to current 21 | echo Running new version for "$i"th time >> cmd-output.txt 22 | 23 | # keep the result of the run, in case any error occurs 24 | start_time=$(date +%s) 25 | RunNew >> cmd-output.txt 26 | end_time=$(date +%s) 27 | 28 | # insert a record into the result CSV file 29 | # this format is used so that we can import it easily into Excel 30 | echo $i, new, $(expr "$end_time" - "$start_time") >> result.csv 31 | 32 | # run the current version immedietely after 33 | echo Running current version for "$i"th time >> cmd-output.txt 34 | 35 | # do the same for current version 36 | start_time=$(date +%s) 37 | RunCurrent >> cmd-output.txt 38 | end_time=$(date +%s) 39 | echo $i, current, $(expr "$end_time" - "$start_time") >> result.csv 40 | done -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Azure/azure-storage-azcopy/v10 2 | 3 | require ( 4 | cloud.google.com/go/compute v1.5.0 // indirect 5 | cloud.google.com/go/iam v0.2.0 // indirect 6 | cloud.google.com/go/storage v1.21.0 7 | github.com/Azure/azure-pipeline-go v0.2.3 8 | github.com/Azure/azure-storage-blob-go v0.13.1-0.20210914164749-2d6cd3e07548 9 | github.com/Azure/azure-storage-file-go v0.6.1-0.20201111053559-3c1754dc00a5 10 | github.com/Azure/go-autorest/autorest/adal v0.9.18 11 | github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda 12 | github.com/danieljoos/wincred v1.1.2 13 | github.com/go-ini/ini v1.66.4 // indirect 14 | github.com/golang-jwt/jwt/v4 v4.3.0 // indirect 15 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da 16 | github.com/google/uuid v1.3.0 17 | github.com/hillu/go-ntdll v0.0.0-20220217145204-be7b5318100d 18 | github.com/kr/pretty v0.3.0 // indirect 19 | github.com/mattn/go-ieproxy v0.0.3 20 | github.com/minio/minio-go v6.0.14+incompatible 21 | github.com/pkg/errors v0.9.1 22 | github.com/spf13/cobra v1.3.0 23 | github.com/stretchr/objx v0.3.0 // indirect 24 | github.com/wastore/keychain v0.0.0-20180920053336-f2c902a3d807 25 | github.com/wastore/keyctl v0.3.1 26 | golang.org/x/crypto v0.0.0-20220214200702-86341886e292 27 | golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b 28 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c 29 | golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7 30 | google.golang.org/api v0.70.0 31 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c 32 | ) 33 | 34 | go 1.16 35 | -------------------------------------------------------------------------------- /sddl/sidTranslation_windows.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | // Copyright © Microsoft 4 | // 5 | // Permission is hereby granted, free of charge, to any person obtaining a copy 6 | // of this software and associated documentation files (the "Software"), to deal 7 | // in the Software without restriction, including without limitation the rights 8 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | // copies of the Software, and to permit persons to whom the Software is 10 | // furnished to do so, subject to the following conditions: 11 | // 12 | // The above copyright notice and this permission notice shall be included in 13 | // all copies or substantial portions of the Software. 14 | // 15 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | // THE SOFTWARE. 22 | 23 | package sddl 24 | 25 | import ( 26 | "golang.org/x/sys/windows" 27 | ) 28 | 29 | // Note that all usages of OSTranslateSID gracefully handle the error, rather than throwing the error. 30 | func OSTranslateSID(SID string) (string, error) { 31 | wsid, err := windows.StringToSid(SID) 32 | 33 | if err != nil { 34 | return "", err 35 | } 36 | 37 | return wsid.String(), nil 38 | } 39 | -------------------------------------------------------------------------------- /cmd/env.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2018 NAME HERE 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "fmt" 19 | "github.com/Azure/azure-storage-azcopy/v10/common" 20 | "github.com/spf13/cobra" 21 | ) 22 | 23 | var showSensitive = false 24 | 25 | // envCmd represents the env command 26 | var envCmd = &cobra.Command{ 27 | Use: "env", 28 | Short: envCmdShortDescription, 29 | Long: envCmdLongDescription, 30 | Run: func(cmd *cobra.Command, args []string) { 31 | for _, env := range common.VisibleEnvironmentVariables { 32 | val := glcm.GetEnvironmentVariable(env) 33 | if env.Hidden && !showSensitive { 34 | val = "REDACTED" 35 | } 36 | 37 | glcm.Info(fmt.Sprintf("Name: %s\nCurrent Value: %s\nDescription: %s\n", 38 | env.Name, val, env.Description)) 39 | } 40 | 41 | glcm.Exit(nil, common.EExitCode.Success()) 42 | }, 43 | } 44 | 45 | func init() { 46 | envCmd.PersistentFlags().BoolVar(&showSensitive, "show-sensitive", false, "Shows sensitive/secret environment variables.") 47 | rootCmd.AddCommand(envCmd) 48 | } 49 | -------------------------------------------------------------------------------- /e2etest/zt_overwrite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package e2etest 22 | 23 | // Purpose: Tests of overwrite logic 24 | 25 | // Might be able to use afterStart hook in these tests, since it can be used to send arbitrary text to the apps stdin 26 | // We currently use it to set "open" in the case where --await-open was on the command line, but the afterStart func in 27 | // execDebuggableWithOutput is not limited to that purpose. _Whatever_ it returns will be sent to AzCopy's stdin. 28 | -------------------------------------------------------------------------------- /common/credCacheModel.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package common 22 | 23 | // CredCacheOptions contains options could be used in different kinds of cred caches in different platform. 24 | type CredCacheOptions struct { 25 | // Used by credCache in Windows. 26 | DPAPIFilePath string 27 | 28 | // Used by credCacheSegmented in Windows, and keyring in Linux. 29 | KeyName string 30 | // Used by keychain in Mac OS, and gnome keyring in Linux. 31 | ServiceName string 32 | AccountName string 33 | } 34 | -------------------------------------------------------------------------------- /common/osOpen_windows.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | package common 4 | 5 | import ( 6 | "os" 7 | ) 8 | 9 | // NOTE: this is not safe to use on directories. It returns an os.File that points at a directory, but thinks it points to a file. 10 | // There is no way around that in the Go SDK as at Go 1.13, because the only way to make a valid windows.File that points to a directory 11 | // is private inside the Go SDK's Windows package. 12 | func OSOpenFile(name string, flag int, perm os.FileMode) (*os.File, error) { 13 | // use openwithwritethroughsetting with false writethrough, since it makes a windows syscall containing an ask 14 | // for backup privileges. This allows all of our file opening to go down one route of code. 15 | fd, err := OpenWithWriteThroughSetting(name, flag, uint32(perm), false) 16 | 17 | if err != nil { 18 | return nil, err 19 | } 20 | 21 | file := os.NewFile(uintptr(fd), name) 22 | if file == nil { 23 | return nil, os.ErrInvalid 24 | } 25 | 26 | return file, nil 27 | } 28 | 29 | // TODO: os.Stat(name string) already uses backup semantics. Can/should we just use that? 30 | // That would work around the issue where here we are using OSOpen, which technically returns a bad result for 31 | // directories (but which we seem to get away with in the os.Stat(handle) call - which goes down the "I am a file" code 32 | // path, but still returns a valid result for a directory. 33 | // See also todo on OsOpen, and edit or remove it 34 | func OSStat(name string) (os.FileInfo, error) { 35 | return os.Stat(name) // this is safe even with our --backup mode, because it uses FILE_FLAG_BACKUP_SEMANTICS (whereas os.File.Stat() does not) 36 | } 37 | -------------------------------------------------------------------------------- /cmd/jobs.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "github.com/spf13/cobra" 25 | ) 26 | 27 | // jobs command is used to encapsulate all sub-commands related to managing jobs 28 | // it is declared as a global variable here so that its sub-commands can add themselves to it 29 | // jobs command itself is not runnable 30 | var jobsCmd = &cobra.Command{ 31 | Use: "jobs", 32 | Short: jobsCmdShortDescription, 33 | Long: jobsCmdLongDescription, 34 | Example: jobsCmdExample, 35 | } 36 | 37 | func init() { 38 | // add jobs command as a top level command 39 | rootCmd.AddCommand(jobsCmd) 40 | } 41 | -------------------------------------------------------------------------------- /cmd/zt_sync_cmd_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "github.com/Azure/azure-storage-azcopy/v10/common" 25 | chk "gopkg.in/check.v1" 26 | ) 27 | 28 | //Test dfs endpoints are cooked to blob endpoints 29 | func (s *cmdIntegrationSuite) TestSyncS2SWithDFS(c *chk.C) { 30 | src := "https://myaccount1.dfs.core.windows.net/container1/" 31 | dst := "https://myaccount2.dfs.core.windows.net/container2/" 32 | 33 | raw := getDefaultSyncRawInput(src, dst) 34 | 35 | cooked, err := raw.cook() 36 | 37 | c.Assert(err, chk.IsNil) 38 | c.Assert(cooked.fromTo, chk.Equals, common.EFromTo.BlobBlob()) 39 | } 40 | -------------------------------------------------------------------------------- /common/credCacheGnomeKeyring_linux.go: -------------------------------------------------------------------------------- 1 | // +build se_integration 2 | // Note: This implementation is initially forked from https://github.com/tmc/keyring. 3 | // And Azcopy customized the code for its own usage. 4 | 5 | package common 6 | 7 | /* 8 | #cgo pkg-config: libsecret-1 glib-2.0 9 | #include 10 | #include "libsecret/secret.h" 11 | 12 | SecretSchema keyring_schema = 13 | { 14 | "org.freedesktop.Secret.Generic", 15 | SECRET_SCHEMA_NONE, 16 | { 17 | { "service", SECRET_SCHEMA_ATTRIBUTE_STRING }, 18 | { "account", SECRET_SCHEMA_ATTRIBUTE_STRING }, 19 | { NULL, 0 }, 20 | } 21 | }; 22 | 23 | // wrap the gnome calls because cgo can't deal with vararg functions 24 | gchar * gkr_get_password(gchar *service, gchar *account, GError **err) { 25 | return secret_password_lookup_sync( 26 | &keyring_schema, 27 | NULL, // Using gnome "default" keyring 28 | err, 29 | "service", service, 30 | "account", account, 31 | NULL); 32 | } 33 | */ 34 | import "C" 35 | 36 | import ( 37 | "fmt" 38 | "unsafe" 39 | ) 40 | 41 | type gnomeKeyring struct{} 42 | 43 | func (p gnomeKeyring) Get(service string, account string) (string, error) { 44 | var gErr *C.GError 45 | var pw *C.gchar 46 | 47 | cStrService := (*C.gchar)(C.CString(service)) 48 | cStrAccount := (*C.gchar)(C.CString(account)) 49 | 50 | defer C.free(unsafe.Pointer(cStrService)) 51 | defer C.free(unsafe.Pointer(cStrAccount)) 52 | 53 | pw = C.gkr_get_password(cStrService, cStrAccount, &gErr) 54 | defer func() { 55 | if gErr != nil { 56 | C.g_error_free(gErr) 57 | } 58 | }() 59 | defer C.secret_password_free((*C.gchar)(pw)) 60 | 61 | if pw == nil { 62 | return "", fmt.Errorf("GnomeKeyring failed to lookup: %+v", gErr) 63 | } 64 | return C.GoString((*C.char)(pw)), nil 65 | } 66 | -------------------------------------------------------------------------------- /common/nullHasher.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package common 22 | 23 | import "hash" 24 | 25 | // A hash.Hash implementation that does nothing 26 | type nullHasher struct{} 27 | 28 | func NewNullHasher() hash.Hash { 29 | return &nullHasher{} 30 | } 31 | 32 | func (*nullHasher) Write(p []byte) (n int, err error) { 33 | // noop 34 | return 0, nil 35 | } 36 | 37 | func (*nullHasher) Sum(b []byte) []byte { 38 | return make([]byte, 0) 39 | } 40 | 41 | func (*nullHasher) Reset() { 42 | // noop 43 | } 44 | 45 | func (*nullHasher) Size() int { 46 | return 0 47 | } 48 | 49 | func (*nullHasher) BlockSize() int { 50 | return 1 51 | } 52 | -------------------------------------------------------------------------------- /common/uuid_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package common 22 | 23 | import ( 24 | chk "gopkg.in/check.v1" 25 | "strings" 26 | ) 27 | 28 | type uuidTestSuite struct{} 29 | 30 | var _ = chk.Suite(&uuidTestSuite{}) 31 | 32 | func (s *uuidTestSuite) TestGUIDGenerationAndParsing(c *chk.C) { 33 | for i := 0; i < 100; i++ { 34 | uuid := NewUUID() 35 | 36 | // no space is allowed 37 | containsSpace := strings.Contains(uuid.String(), " ") 38 | c.Assert(containsSpace, chk.Equals, false) 39 | 40 | parsed, err := ParseUUID(uuid.String()) 41 | c.Assert(err, chk.IsNil) 42 | c.Assert(parsed, chk.DeepEquals, uuid) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /e2etest/zt_resume_windows_test.go: -------------------------------------------------------------------------------- 1 | package e2etest 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/Azure/azure-storage-azcopy/v10/common" 7 | ) 8 | 9 | func TestResume_FolderState(t *testing.T) { 10 | // Create a child file before the folder itself, then persist the properties of the folder upon resume, knowing that we created the folder. 11 | RunScenarios(t, eOperation.CopyAndSync()|eOperation.Resume(), eTestFromTo.Other(common.EFromTo.LocalFile(), common.EFromTo.FileFile(), common.EFromTo.FileLocal()), eValidate.Auto(), params{ 12 | recursive: true, 13 | preserveSMBInfo: true, 14 | debugSkipFiles: []string{ 15 | "a", 16 | }, 17 | }, nil, testFiles{ 18 | defaultSize: "1K", 19 | 20 | shouldTransfer: []interface{}{ 21 | folder(""), 22 | folder("a", with{smbAttributes: 2}), 23 | f("a/b"), 24 | }, 25 | }, EAccountType.Standard(), EAccountType.Standard(), "") 26 | } 27 | 28 | func TestResume_NoCreateFolder(t *testing.T) { 29 | // Don't create the folder "ourselves", and let AzCopy find that out on a resume. 30 | RunScenarios(t, eOperation.Copy()|eOperation.Resume(), eTestFromTo.Other(common.EFromTo.LocalFile(), common.EFromTo.FileFile(), common.EFromTo.FileLocal()), eValidate.Auto(), params{ 31 | recursive: true, 32 | preserveSMBInfo: true, 33 | debugSkipFiles: []string{ 34 | "a", 35 | "a/b", 36 | }, 37 | }, &hooks{ 38 | beforeResumeHook: func(h hookHelper) { 39 | // Create the folder in the middle of the transfer 40 | h.CreateFile(folder("a"), false) 41 | }, 42 | }, testFiles{ 43 | defaultSize: "1K", 44 | 45 | shouldTransfer: []interface{}{ 46 | folder(""), 47 | folder("a"), 48 | f("a/b"), 49 | f("c"), 50 | }, 51 | shouldSkip: []interface{}{ 52 | folder("a", with{smbAttributes: 2}), 53 | }, 54 | }, EAccountType.Standard(), EAccountType.Standard(), "") 55 | } 56 | -------------------------------------------------------------------------------- /azbfs/zc_pipeline.go: -------------------------------------------------------------------------------- 1 | package azbfs 2 | 3 | import ( 4 | "github.com/Azure/azure-pipeline-go/pipeline" 5 | ) 6 | 7 | // PipelineOptions is used to configure a request policy pipeline's retry policy and logging. 8 | type PipelineOptions struct { 9 | // Log configures the pipeline's logging infrastructure indicating what information is logged and where. 10 | Log pipeline.LogOptions 11 | 12 | // Retry configures the built-in retry policy behavior. 13 | Retry RetryOptions 14 | 15 | // RequestLog configures the built-in request logging policy. 16 | RequestLog RequestLogOptions 17 | 18 | // Telemetry configures the built-in telemetry policy behavior. 19 | Telemetry TelemetryOptions 20 | 21 | // HTTPSender configures the sender of HTTP requests 22 | HTTPSender pipeline.Factory 23 | } 24 | 25 | // NewPipeline creates a Pipeline using the specified credentials and options. 26 | func NewPipeline(c Credential, o PipelineOptions) pipeline.Pipeline { 27 | if c == nil { 28 | panic("c can't be nil") 29 | } 30 | 31 | // Closest to API goes first; closest to the wire goes last 32 | f := []pipeline.Factory{ 33 | NewTelemetryPolicyFactory(o.Telemetry), 34 | NewUniqueRequestIDPolicyFactory(), 35 | NewRetryPolicyFactory(o.Retry), 36 | } 37 | 38 | if _, ok := c.(*anonymousCredentialPolicyFactory); !ok { 39 | // For AnonymousCredential, we optimize out the policy factory since it doesn't do anything 40 | // NOTE: The credential's policy factory must appear close to the wire so it can sign any 41 | // changes made by other factories (like UniqueRequestIDPolicyFactory) 42 | f = append(f, c) 43 | } 44 | f = append(f, 45 | pipeline.MethodFactoryMarker(), // indicates at what stage in the pipeline the method factory is invoked 46 | NewRequestLogPolicyFactory_Deprecated(o.RequestLog)) 47 | 48 | return pipeline.NewPipeline(f, pipeline.Options{HTTPSender: o.HTTPSender, Log: o.Log}) 49 | } 50 | -------------------------------------------------------------------------------- /common/gcpURLParts_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | chk "gopkg.in/check.v1" 5 | "net/url" 6 | "strings" 7 | ) 8 | 9 | type gcpURLPartsTestSuite struct{} 10 | 11 | // This testsuite does not reach GCP service, and runs even with GCP_TESTS=FALSE 12 | var _ = chk.Suite(&gcpURLPartsTestSuite{}) 13 | 14 | func (s *gcpURLPartsTestSuite) TestGCPURLParse(c *chk.C) { 15 | u, _ := url.Parse("http://storage.cloud.google.com/bucket") 16 | p, err := NewGCPURLParts(*u) 17 | c.Assert(err, chk.IsNil) 18 | c.Assert(p.Host, chk.Equals, "storage.cloud.google.com") 19 | c.Assert(p.BucketName, chk.Equals, "bucket") 20 | c.Assert(p.ObjectKey, chk.Equals, "") 21 | c.Assert(p.String(), chk.Equals, "http://storage.cloud.google.com/bucket") 22 | 23 | u, _ = url.Parse("https://storage.cloud.google.com") 24 | p, err = NewGCPURLParts(*u) 25 | c.Assert(err, chk.IsNil) 26 | c.Assert(p.BucketName, chk.Equals, "") 27 | c.Assert(p.ObjectKey, chk.Equals, "") 28 | c.Assert(p.String(), chk.Equals, "https://storage.cloud.google.com") 29 | 30 | u, _ = url.Parse("http://storage.cloud.google.com/bucket/keyname/") 31 | p, err = NewGCPURLParts(*u) 32 | c.Assert(err, chk.IsNil) 33 | c.Assert(p.BucketName, chk.Equals, "bucket") 34 | c.Assert(p.ObjectKey, chk.Equals, "keyname/") 35 | c.Assert(p.String(), chk.Equals, "http://storage.cloud.google.com/bucket/keyname/") 36 | 37 | } 38 | 39 | func (s *gcpURLPartsTestSuite) TestGCPURLParseNegative(c *chk.C) { 40 | u, _ := url.Parse("https://storage.cloud.googly.com/bucket") 41 | _, err := NewGCPURLParts(*u) 42 | c.Assert(err, chk.NotNil) 43 | c.Assert(strings.Contains(err.Error(), invalidGCPURLErrorMessage), chk.Equals, true) 44 | 45 | u, _ = url.Parse("https://mcdheestorage.blob.core.windows.net") 46 | _, err = NewGCPURLParts(*u) 47 | c.Assert(err, chk.NotNil) 48 | c.Assert(strings.Contains(err.Error(), invalidGCPURLErrorMessage), chk.Equals, true) 49 | } 50 | -------------------------------------------------------------------------------- /azbfs/zc_policy_telemetry.go: -------------------------------------------------------------------------------- 1 | package azbfs 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "os" 8 | "runtime" 9 | 10 | "github.com/Azure/azure-pipeline-go/pipeline" 11 | ) 12 | 13 | // TelemetryOptions configures the telemetry policy's behavior. 14 | type TelemetryOptions struct { 15 | // Value is a string prepended to each request's User-Agent and sent to the service. 16 | // The service records the user-agent in logs for diagnostics and tracking of client requests. 17 | Value string 18 | } 19 | 20 | // NewTelemetryPolicyFactory creates a factory that can create telemetry policy objects 21 | // which add telemetry information to outgoing HTTP requests. 22 | func NewTelemetryPolicyFactory(o TelemetryOptions) pipeline.Factory { 23 | b := &bytes.Buffer{} 24 | b.WriteString(o.Value) 25 | if b.Len() > 0 { 26 | b.WriteRune(' ') 27 | } 28 | fmt.Fprintf(b, "Azure-Storage/%s %s", serviceLibVersion, platformInfo) 29 | telemetryValue := b.String() 30 | 31 | return pipeline.FactoryFunc(func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc { 32 | return func(ctx context.Context, request pipeline.Request) (pipeline.Response, error) { 33 | request.Header.Set("User-Agent", telemetryValue) 34 | return next.Do(ctx, request) 35 | } 36 | }) 37 | } 38 | 39 | // NOTE: the ONLY function that should write to this variable is this func 40 | var platformInfo = func() string { 41 | // Azure-Storage/version (runtime; os type and version)” 42 | // Azure-Storage/1.4.0 (NODE-VERSION v4.5.0; Windows_NT 10.0.14393)' 43 | operatingSystem := runtime.GOOS // Default OS string 44 | switch operatingSystem { 45 | case "windows": 46 | operatingSystem = os.Getenv("OS") // Get more specific OS information 47 | case "linux": // accept default OS info 48 | case "freebsd": // accept default OS info 49 | } 50 | return fmt.Sprintf("(%s; %s)", runtime.Version(), operatingSystem) 51 | }() 52 | -------------------------------------------------------------------------------- /cmd/zt_remove_copy_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | chk "gopkg.in/check.v1" 5 | "strings" 6 | ) 7 | 8 | func (s *cmdIntegrationSuite) TestCopyBlobsWithDirectoryStubsS2S(c *chk.C) { 9 | c.Skip("Enable after setting Account to non-HNS") 10 | bsu := getBSU() 11 | vdirName := "vdir1/" 12 | 13 | // create container and dest container 14 | srcContainerURL, srcContainerName := createNewContainer(c, bsu) 15 | dstContainerURL, dstContainerName := createNewContainer(c, bsu) 16 | dstBlobName := "testcopyblobswithdirectorystubs" + generateBlobName() 17 | defer deleteContainer(c, srcContainerURL) 18 | defer deleteContainer(c, dstContainerURL) 19 | 20 | blobAndDirStubsList := scenarioHelper{}.generateCommonRemoteScenarioForWASB(c, srcContainerURL, vdirName) 21 | c.Assert(srcContainerURL, chk.NotNil) 22 | c.Assert(len(blobAndDirStubsList), chk.Not(chk.Equals), 0) 23 | 24 | // set up interceptor 25 | mockedRPC := interceptor{} 26 | Rpc = mockedRPC.intercept 27 | mockedRPC.init() 28 | 29 | // construct the raw input to simulate user input 30 | rawSrcBlobWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, srcContainerName, vdirName) 31 | rawDstBlobWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, dstContainerName, dstBlobName) 32 | raw := getDefaultCopyRawInput(rawSrcBlobWithSAS.String(), rawDstBlobWithSAS.String()) 33 | raw.recursive = true 34 | raw.includeDirectoryStubs = true 35 | 36 | runCopyAndVerify(c, raw, func(err error) { 37 | c.Assert(err, chk.IsNil) 38 | 39 | // validate that the right number of transfers were scheduled 40 | c.Assert(len(mockedRPC.transfers), chk.Equals, len(blobAndDirStubsList)) 41 | 42 | // validate that the right transfers were sent 43 | expectedTransfers := scenarioHelper{}.shaveOffPrefix(blobAndDirStubsList, strings.TrimSuffix(vdirName, "/")) 44 | validateCopyTransfersAreScheduled(c, true, true, vdirName, "/vdir1",expectedTransfers, mockedRPC) 45 | }) 46 | } 47 | -------------------------------------------------------------------------------- /common/prologueState.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package common 22 | 23 | type cutdownJptm interface { 24 | ResourceDstData(dataFileToXfer []byte) (headers ResourceHTTPHeaders, metadata Metadata, blobTags BlobTags, cpkOptions CpkOptions) 25 | } 26 | 27 | // PrologueState contains info necessary for different sending operations' prologue. 28 | type PrologueState struct { 29 | // Leading bytes are the early bytes of the file, to be used 30 | // for mime-type detection (or nil if file is empty or the bytes code 31 | // not be read). 32 | LeadingBytes []byte 33 | } 34 | 35 | func (ps PrologueState) GetInferredContentType(jptm cutdownJptm) string { 36 | headers, _, _, _ := jptm.ResourceDstData(ps.LeadingBytes) 37 | return headers.ContentType 38 | } 39 | -------------------------------------------------------------------------------- /e2etest/zt_enumeration_other_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package e2etest 22 | 23 | import ( 24 | "github.com/Azure/azure-storage-azcopy/v10/common" 25 | "testing" 26 | ) 27 | 28 | // Purpose: Other tests for enumeration of sources, NOT including filtering 29 | 30 | func TestEnumeration_DirectoryStubsAreNotDownloaded(t *testing.T) { 31 | RunScenarios(t, eOperation.Copy(), eTestFromTo.Other(common.EFromTo.BlobLocal()), eValidate.Auto(), params{ 32 | recursive: true, 33 | }, nil, testFiles{ 34 | defaultSize: "1K", 35 | shouldIgnore: []interface{}{ 36 | f("dir", withDirStubMetadata{}), 37 | }, 38 | shouldTransfer: []interface{}{ 39 | "filea", 40 | folder("dir"), 41 | "dir/fileb", 42 | }, 43 | }, EAccountType.Standard(), EAccountType.Standard(), "") 44 | } 45 | -------------------------------------------------------------------------------- /ste/putListNeed.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package ste 22 | 23 | import "sync/atomic" 24 | 25 | const ( 26 | putListNotNeeded = -1 27 | putListNeedUnknown = 0 28 | putListNeeded = 1 29 | ) 30 | 31 | // TODO: do we want to keep using atomic, just to be sure this is safe no matter how or where its used? 32 | // Or do we want to rely on the assumption, which is correct as at Jan 2018, that its only 33 | // used in the single-threaded chunk creation loop and in the epilogue. For now, we are erring on the side of safety. 34 | func setPutListNeed(target *int32, value int32) { 35 | previous := atomic.SwapInt32(target, value) 36 | if previous != putListNeedUnknown && previous != value { 37 | panic("'put list' need cannot be set twice") 38 | } 39 | } 40 | 41 | func getPutListNeed(storage *int32) int32 { 42 | return atomic.LoadInt32(storage) 43 | } 44 | -------------------------------------------------------------------------------- /cmd/load.go: -------------------------------------------------------------------------------- 1 | // +build linux 2 | 3 | // Copyright © Microsoft 4 | // 5 | // Permission is hereby granted, free of charge, to any person obtaining a copy 6 | // of this software and associated documentation files (the "Software"), to deal 7 | // in the Software without restriction, including without limitation the rights 8 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | // copies of the Software, and to permit persons to whom the Software is 10 | // furnished to do so, subject to the following conditions: 11 | // 12 | // The above copyright notice and this permission notice shall be included in 13 | // all copies or substantial portions of the Software. 14 | // 15 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | // THE SOFTWARE. 22 | 23 | package cmd 24 | 25 | import ( 26 | "github.com/spf13/cobra" 27 | ) 28 | 29 | // loadCmd represents the load command 30 | var loadCmd = &cobra.Command{ 31 | Use: "load", 32 | // Note: the help messages are kept in this file on purpose, to limit the footprint of the change (easier migration later) 33 | Short: "Sub-commands related to transferring data in specific formats", 34 | Long: "Sub-commands related to transferring data in specific formats, such as Microsoft's Avere Cloud FileSystem (CLFS) format.", 35 | Example: ` 36 | Load an entire directory to a container with a SAS in CLFS format: 37 | - azcopy load clfs "/path/to/dir" "https://[account].blob.core.windows.net/[container]?[SAS]" --state-path="/path/to/state/path" 38 | `, 39 | } 40 | 41 | func init() { 42 | rootCmd.AddCommand(loadCmd) 43 | } 44 | -------------------------------------------------------------------------------- /cmd/zc_attr_filter_notwin.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | // Copyright © Microsoft 4 | // 5 | // Permission is hereby granted, free of charge, to any person obtaining a copy 6 | // of this software and associated documentation files (the "Software"), to deal 7 | // in the Software without restriction, including without limitation the rights 8 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | // copies of the Software, and to permit persons to whom the Software is 10 | // furnished to do so, subject to the following conditions: 11 | // 12 | // The above copyright notice and this permission notice shall be included in 13 | // all copies or substantial portions of the Software. 14 | // 15 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | // THE SOFTWARE. 22 | 23 | package cmd 24 | 25 | type attrFilter struct{} 26 | 27 | func (f *attrFilter) DoesSupportThisOS() (msg string, supported bool) { 28 | msg = "'include-attributes' and 'exclude-attributes' are not supported on this OS. Abort." 29 | supported = false 30 | return 31 | } 32 | 33 | func (f *attrFilter) AppliesOnlyToFiles() bool { 34 | return true 35 | } 36 | 37 | func (f *attrFilter) DoesPass(storedObject StoredObject) bool { 38 | // ignore this option on Unix systems 39 | return true 40 | } 41 | 42 | func buildAttrFilters(attributes []string, fullPath string, resultIfMatch bool) []ObjectFilter { 43 | // ignore this option on Unix systems 44 | filters := make([]ObjectFilter, 0) 45 | if len(attributes) > 0 { 46 | filters = append(filters, &attrFilter{}) 47 | } 48 | return filters 49 | } 50 | -------------------------------------------------------------------------------- /azbfs/zc_credential_anonymous.go: -------------------------------------------------------------------------------- 1 | package azbfs 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/Azure/azure-pipeline-go/pipeline" 7 | ) 8 | 9 | // Credential represent any credential type; it is used to create a credential policy Factory. 10 | type Credential interface { 11 | pipeline.Factory 12 | credentialMarker() 13 | } 14 | 15 | type credentialFunc pipeline.FactoryFunc 16 | 17 | func (f credentialFunc) New(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.Policy { 18 | return f(next, po) 19 | } 20 | 21 | // credentialMarker is a package-internal method that exists just to satisfy the Credential interface. 22 | func (credentialFunc) credentialMarker() {} 23 | 24 | ////////////////////////////// 25 | 26 | // NewAnonymousCredential creates an anonymous credential for use with HTTP(S) requests that read public resource 27 | // or for use with Shared Access Signatures (SAS). 28 | func NewAnonymousCredential() Credential { 29 | return anonymousCredentialFactory 30 | } 31 | 32 | var anonymousCredentialFactory Credential = &anonymousCredentialPolicyFactory{} // Singleton 33 | 34 | // anonymousCredentialPolicyFactory is the credential's policy factory. 35 | type anonymousCredentialPolicyFactory struct { 36 | } 37 | 38 | // New creates a credential policy object. 39 | func (f *anonymousCredentialPolicyFactory) New(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.Policy { 40 | return &anonymousCredentialPolicy{next: next} 41 | } 42 | 43 | // credentialMarker is a package-internal method that exists just to satisfy the Credential interface. 44 | func (*anonymousCredentialPolicyFactory) credentialMarker() {} 45 | 46 | // anonymousCredentialPolicy is the credential's policy object. 47 | type anonymousCredentialPolicy struct { 48 | next pipeline.Policy 49 | } 50 | 51 | // Do implements the credential's policy interface. 52 | func (p anonymousCredentialPolicy) Do(ctx context.Context, request pipeline.Request) (pipeline.Response, error) { 53 | // For anonymous credentials, this is effectively a no-op 54 | return p.next.Do(ctx, request) 55 | } 56 | -------------------------------------------------------------------------------- /common/azError.go: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package common 22 | 23 | // AzError is to handle AzCopy internal errors in a fine way 24 | type AzError struct { 25 | code uint64 26 | msg string 27 | additonalInfo string 28 | } 29 | 30 | // NewAzError composes an AzError with given code and messgae 31 | func NewAzError(base AzError, additionalInfo string) AzError { 32 | base.additonalInfo = additionalInfo 33 | return base 34 | } 35 | 36 | func (err AzError) ErrorCode() uint64 { 37 | return err.code 38 | } 39 | 40 | func (lhs AzError) Equals(rhs AzError) bool { 41 | return lhs.code == rhs.code 42 | } 43 | 44 | func (err AzError) Error() string { 45 | return err.msg + err.additonalInfo 46 | } 47 | 48 | var EAzError AzError 49 | 50 | func (err AzError) LoginCredMissing() AzError { 51 | return AzError{uint64(1), "Login Credentials missing. ", ""} 52 | } 53 | 54 | func (err AzError) InvalidBlobName() AzError { 55 | return AzError {uint64(2), "Invalid Blob Name.", ""} 56 | } -------------------------------------------------------------------------------- /azbfs/zc_uuid.go: -------------------------------------------------------------------------------- 1 | package azbfs 2 | 3 | import ( 4 | "crypto/rand" 5 | "fmt" 6 | "strconv" 7 | ) 8 | 9 | // The UUID reserved variants. 10 | const ( 11 | reservedNCS byte = 0x80 12 | reservedRFC4122 byte = 0x40 13 | reservedMicrosoft byte = 0x20 14 | reservedFuture byte = 0x00 15 | ) 16 | 17 | // A UUID representation compliant with specification in RFC 4122 document. 18 | type uuid [16]byte 19 | 20 | // NewUUID returns a new uuid using RFC 4122 algorithm. 21 | func newUUID() (u uuid) { 22 | u = uuid{} 23 | // Set all bits to randomly (or pseudo-randomly) chosen values. 24 | _, err := rand.Read(u[:]) 25 | if err != nil { 26 | panic("ran.Read failed") 27 | } 28 | u[8] = (u[8] | reservedRFC4122) & 0x7F // u.setVariant(ReservedRFC4122) 29 | 30 | var version byte = 4 31 | u[6] = (u[6] & 0xF) | (version << 4) // u.setVersion(4) 32 | return 33 | } 34 | 35 | // String returns an unparsed version of the generated UUID sequence. 36 | func (u uuid) String() string { 37 | return fmt.Sprintf("%x-%x-%x-%x-%x", u[0:4], u[4:6], u[6:8], u[8:10], u[10:]) 38 | } 39 | 40 | // ParseUUID parses a string formatted as "003020100-0504-0706-0809-0a0b0c0d0e0f" 41 | // or "{03020100-0504-0706-0809-0a0b0c0d0e0f}" into a UUID. 42 | func parseUUID(uuidStr string) uuid { 43 | char := func(hexString string) byte { 44 | i, _ := strconv.ParseUint(hexString, 16, 8) 45 | return byte(i) 46 | } 47 | if uuidStr[0] == '{' { 48 | uuidStr = uuidStr[1:] // Skip over the '{' 49 | } 50 | // 03020100 - 05 04 - 07 06 - 08 09 - 0a 0b 0c 0d 0e 0f 51 | // 1 11 1 11 11 1 12 22 2 22 22 22 33 33 33 52 | // 01234567 8 90 12 3 45 67 8 90 12 3 45 67 89 01 23 45 53 | uuidVal := uuid{ 54 | char(uuidStr[0:2]), 55 | char(uuidStr[2:4]), 56 | char(uuidStr[4:6]), 57 | char(uuidStr[6:8]), 58 | 59 | char(uuidStr[9:11]), 60 | char(uuidStr[11:13]), 61 | 62 | char(uuidStr[14:16]), 63 | char(uuidStr[16:18]), 64 | 65 | char(uuidStr[19:21]), 66 | char(uuidStr[21:23]), 67 | 68 | char(uuidStr[24:26]), 69 | char(uuidStr[26:28]), 70 | char(uuidStr[28:30]), 71 | char(uuidStr[30:32]), 72 | char(uuidStr[32:34]), 73 | char(uuidStr[34:36]), 74 | } 75 | return uuidVal 76 | } 77 | 78 | func (u uuid) bytes() []byte { 79 | return u[:] 80 | } 81 | -------------------------------------------------------------------------------- /e2etest/azcopyDebugHelper.go: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package e2etest 22 | 23 | import ( 24 | "io" 25 | ) 26 | 27 | func beginAzCopyDebugging(stdin io.Writer) { 28 | // to debug AzCopy from within a running test, you must 29 | // 0. Make sure that gops is available on your PATH. (i.e. running "gops" will run it). See https://github.com/google/gops 30 | // Gops is needed by TestRunner.isLaunchedByDebugger(), and it's also needed by GoLand's Attach To Process (step 2 below) 31 | // then 32 | // 1. Set a breakpoint on the line below 33 | // 2. When the breakpoint is hit, use Attach To Process to manually attach your debugger to the running AzCopy process 34 | // 3. Then you MUST continue past the breakpoint that you set here (i.e. on the line below). Any breakpoints you've set in AzCopy will then be hit. 35 | // (Note that some IDEs, e.g. GoLand, will give you two separate debug tabs/contexts, one for the tests exe, and one for AzCopy. So make sure you hit run/continue in the right tab) 36 | _, _ = stdin.Write([]byte("continue\n")) // tell AzCopy to start its work 37 | } 38 | -------------------------------------------------------------------------------- /cmd/zc_traverser_local_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | 6 | chk "gopkg.in/check.v1" 7 | ) 8 | 9 | type localTraverserTestSuite struct{} 10 | 11 | var _ = chk.Suite(&localTraverserTestSuite{}) 12 | 13 | func (s *localTraverserTestSuite) TestCleanLocalPath(c *chk.C) { 14 | testCases := map[string]string{ 15 | "/user/foo/bar": "/user/foo/bar", // regular unix path with no change 16 | "/user/foo/bar/": "/user/foo/bar", // regular unix path with extra slash 17 | "/user//foo//bar/": "/user/foo/bar", // regular unix path with double slashes 18 | "./foo/bar": "foo/bar", // relative unix path 19 | "../foo/bar": "../foo/bar", // relative unix path with parent dir 20 | "foo/bar": "foo/bar", // shorthand relative unix path 21 | } 22 | 23 | for orig, expected := range testCases { 24 | c.Assert(cleanLocalPath(orig), chk.Equals, expected) 25 | } 26 | } 27 | 28 | func (s *localTraverserTestSuite) TestCleanLocalPathForWindows(c *chk.C) { 29 | // ignore these tests when not running on Windows 30 | // as the path cleaning behavior depends on the platform 31 | if os.PathSeparator != '\\' { 32 | c.Skip("not running since the test applies to Windows only") 33 | } 34 | 35 | // Paths on Windows get consolidated to backwards-slash typically. 36 | testCases := map[string]string{ 37 | `C:\foo\bar`: `C:\foo\bar`, // regular windows path with no change 38 | `C:\foo\bar\`: `C:\foo\bar`, // regular windows path with extra slash 39 | `.\foo\bar`: `foo\bar`, // relative windows path 40 | `..\foo\bar`: `..\foo\bar`, // relative windows path with parent dir 41 | `foo\bar`: `foo\bar`, // shorthand relative windows path 42 | `\\foo\bar\`: `\\foo\bar`, // network share 43 | `C:\`: `C:\`, // special case, the slash after colon is actually required 44 | `D:`: `D:\`, // special case, the slash after colon is actually required 45 | `c:\`: `c:\`, // special case, the slash after colon is actually required 46 | `c:`: `c:\`, // special case, the slash after colon is actually required 47 | } 48 | 49 | for orig, expected := range testCases { 50 | c.Assert(cleanLocalPath(orig), chk.Equals, expected) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /ste/pacer-nullAutoPacer.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package ste 22 | 23 | import ( 24 | "context" 25 | "sync/atomic" 26 | ) 27 | 28 | // nullAutoPacer is a no-op auto pacer. For use in code which may or may not need pacing. When not needed, 29 | // just use an instance of this type 30 | type nullAutoPacer struct { 31 | atomicGrandTotal int64 32 | } 33 | 34 | func newNullAutoPacer() *nullAutoPacer { 35 | return &nullAutoPacer{} 36 | } 37 | 38 | func (a *nullAutoPacer) Close() error { 39 | return nil 40 | } 41 | 42 | func (a *nullAutoPacer) RetryCallback() { 43 | // noop 44 | } 45 | 46 | func (a *nullAutoPacer) RequestTrafficAllocation(ctx context.Context, byteCount int64) error { 47 | atomic.AddInt64(&a.atomicGrandTotal, byteCount) // we track total aggregate throughput, even though we don't do any actual pacing 48 | return nil 49 | } 50 | 51 | func (a *nullAutoPacer) UndoRequest(byteCount int64) { 52 | atomic.AddInt64(&a.atomicGrandTotal, -byteCount) 53 | } 54 | 55 | func (a *nullAutoPacer) GetTotalTraffic() int64 { 56 | return atomic.LoadInt64(&a.atomicGrandTotal) 57 | } 58 | -------------------------------------------------------------------------------- /testSuite/cmd/common.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | gcpUtils "cloud.google.com/go/storage" 5 | "context" 6 | "fmt" 7 | "net/http" 8 | "os" 9 | "strings" 10 | 11 | "github.com/Azure/azure-storage-blob-go/azblob" 12 | minio "github.com/minio/minio-go" 13 | 14 | "github.com/Azure/azure-storage-azcopy/v10/common" 15 | ) 16 | 17 | // validateString compares the two strings. 18 | func validateString(expected string, actual string) bool { 19 | if strings.Compare(expected, actual) != 0 { 20 | return false 21 | } 22 | return true 23 | } 24 | 25 | type createS3ResOptions struct { 26 | Location string 27 | } 28 | 29 | func createS3ClientWithMinio(o createS3ResOptions) *minio.Client { 30 | lcm := common.GetLifecycleMgr() 31 | accessKeyID := lcm.GetEnvironmentVariable(common.EEnvironmentVariable.AWSAccessKeyID()) 32 | secretAccessKey := lcm.GetEnvironmentVariable(common.EEnvironmentVariable.AWSSecretAccessKey()) 33 | 34 | if accessKeyID == "" || secretAccessKey == "" { 35 | fmt.Println("AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY should be set before creating the S3 client") 36 | os.Exit(1) 37 | } 38 | 39 | s3Client, err := minio.NewWithRegion("s3.amazonaws.com", accessKeyID, secretAccessKey, true, o.Location) 40 | if err != nil { 41 | fmt.Println(err) 42 | os.Exit(1) 43 | } 44 | return s3Client 45 | } 46 | 47 | func createGCPClientWithGCSSDK() (*gcpUtils.Client, error) { 48 | jsonKey := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") 49 | if jsonKey == "" { 50 | return nil, fmt.Errorf("GOOGLE_APPLICATION_CREDENTIALS should be set before creating the GCP Client") 51 | } 52 | projectID := os.Getenv("GOOGLE_CLOUD_PROJECT") 53 | if projectID == "" { 54 | return nil, fmt.Errorf("GOOGLE_CLOUD_PROJECT should be set before creating GCP Client for testing") 55 | } 56 | ctx := context.Background() 57 | gcpClient, err := gcpUtils.NewClient(ctx) 58 | if err != nil { 59 | return nil, err 60 | } 61 | return gcpClient, nil 62 | } 63 | 64 | func ignoreStorageConflictStatus(err error) error { 65 | if err != nil { 66 | // Skip the error, when resource already exists. 67 | if stgErr, ok := err.(azblob.StorageError); !ok || 68 | (stgErr.Response().StatusCode != http.StatusConflict) { 69 | return err 70 | } 71 | } 72 | 73 | return nil 74 | } 75 | -------------------------------------------------------------------------------- /cmd/zt_block_size_to_bytes_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | chk "gopkg.in/check.v1" 25 | ) 26 | 27 | type blockSizeFilterSuite struct{} 28 | 29 | var _ = chk.Suite(&blockSizeFilterSuite{}) 30 | 31 | func (s *genericFilterSuite) TestConversions(c *chk.C) { 32 | 33 | testData := []struct { 34 | floatMiB float64 35 | expectedBytes int64 36 | expectedErrorMsg string 37 | }{ 38 | {100, 100 * 1024 * 1024, ""}, 39 | {1, 1024 * 1024, ""}, 40 | {0.25, 256 * 1024, ""}, 41 | {0.000030517578125, 32, ""}, // 32 bytes, extremely small case 42 | {-1, 0, "negative block size not allowed"}, 43 | {0.333, 0, "while fractional numbers of MiB are allowed as the block size, the fraction must result to a whole number of bytes. 0.333000000000 MiB resolves to 349175.808 bytes"}, 44 | } 45 | 46 | for _, d := range testData { 47 | actualBytes, err := blockSizeInBytes(d.floatMiB) 48 | if d.expectedErrorMsg != "" { 49 | c.Check(err.Error(), chk.Equals, d.expectedErrorMsg) 50 | } else { 51 | c.Check(actualBytes, chk.Equals, d.expectedBytes) 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /cmd/logout.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "fmt" 25 | 26 | "github.com/spf13/cobra" 27 | ) 28 | 29 | func init() { 30 | logoutCmdArgs := logoutCmdArgs{} 31 | 32 | // logoutCmd represents the logout command 33 | logoutCmd := &cobra.Command{ 34 | Use: "logout", 35 | SuggestFor: []string{"logout"}, 36 | Short: logoutCmdShortDescription, 37 | Long: logoutCmdLongDescription, 38 | Args: func(cmd *cobra.Command, args []string) error { 39 | return nil 40 | }, 41 | RunE: func(cmd *cobra.Command, args []string) error { 42 | err := logoutCmdArgs.process() 43 | if err != nil { 44 | return fmt.Errorf("failed to perform logout command, %v", err) 45 | } 46 | return nil 47 | }, 48 | } 49 | 50 | rootCmd.AddCommand(logoutCmd) 51 | } 52 | 53 | type logoutCmdArgs struct{} 54 | 55 | func (lca logoutCmdArgs) process() error { 56 | uotm := GetUserOAuthTokenManagerInstance() 57 | if err := uotm.RemoveCachedToken(); err != nil { 58 | return err 59 | } 60 | 61 | // For MSI login, info success message to user. 62 | glcm.Info("Logout succeeded.") 63 | 64 | return nil 65 | } 66 | -------------------------------------------------------------------------------- /cmd/zt_scenario_helpers_for_windows_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | chk "gopkg.in/check.v1" 25 | "path/filepath" 26 | "strings" 27 | "syscall" 28 | ) 29 | 30 | // set file attributes to test file 31 | func (scenarioHelper) setAttributesForLocalFile(filePath string, attrList []string) error { 32 | lpFilePath, err := syscall.UTF16PtrFromString(filePath) 33 | if err != nil { 34 | return err 35 | } 36 | 37 | fileAttributeMap := map[string]uint32{ 38 | "R": 1, 39 | "A": 32, 40 | "S": 4, 41 | "H": 2, 42 | "C": 2048, 43 | "N": 128, 44 | "E": 16384, 45 | "T": 256, 46 | "O": 4096, 47 | "I": 8192, 48 | } 49 | var attrs uint32 50 | for _, attribute := range attrList { 51 | attrs |= fileAttributeMap[strings.ToUpper(attribute)] 52 | } 53 | err = syscall.SetFileAttributes(lpFilePath, attrs) 54 | return err 55 | } 56 | 57 | func (s scenarioHelper) setAttributesForLocalFiles(c *chk.C, dirPath string, fileList []string, attrList []string) { 58 | for _, fileName := range fileList { 59 | err := s.setAttributesForLocalFile(filepath.Join(dirPath, fileName), attrList) 60 | c.Assert(err, chk.IsNil) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /cmd/zc_traverser_local_windows.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | package cmd 4 | 5 | import ( 6 | "os" 7 | "syscall" 8 | "time" 9 | "unsafe" 10 | 11 | "github.com/hillu/go-ntdll" 12 | "golang.org/x/sys/windows" 13 | ) 14 | 15 | // Override the modtime from the OS stat 16 | type folderStatWrapper struct { 17 | os.FileInfo 18 | changeTime time.Time 19 | } 20 | 21 | func (f *folderStatWrapper) ModTime() time.Time { 22 | return f.changeTime 23 | } 24 | 25 | func WrapFolder(fullpath string, stat os.FileInfo) (os.FileInfo, error) { 26 | srcPtr, err := syscall.UTF16PtrFromString(fullpath) 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | // custom open call, because must specify FILE_FLAG_BACKUP_SEMANTICS to make --backup mode work properly (i.e. our use of SeBackupPrivilege) 32 | fd, err := windows.CreateFile(srcPtr, 33 | windows.GENERIC_READ, windows.FILE_SHARE_READ, nil, 34 | windows.OPEN_EXISTING, windows.FILE_FLAG_BACKUP_SEMANTICS, 0) 35 | if err != nil { 36 | return nil, err 37 | } 38 | defer windows.Close(fd) 39 | 40 | buf := make([]byte, 1024) 41 | var rLen = uint32(unsafe.Sizeof(ntdll.FileBasicInformationT{})) 42 | if st := ntdll.CallWithExpandingBuffer(func() ntdll.NtStatus { 43 | var stat ntdll.IoStatusBlock 44 | return ntdll.NtQueryInformationFile(ntdll.Handle(fd), &stat, &buf[0], uint32(len(buf)), ntdll.FileBasicInformation) 45 | }, &buf, &rLen); st.IsSuccess() { 46 | ntdllTime := time.Date(1601, time.January, 1, 0, 0, 0, 0, time.UTC) 47 | 48 | // do a nasty unsafe thing and tell go that our []byte is actually a FileStandardInformationT. 49 | fi := (*ntdll.FileBasicInformationT)(unsafe.Pointer(&buf[0])) 50 | // ntdll returns times that are incremented by 100-nanosecond "instants" past the beginning of 1601. 51 | // time.Duration is a 64 bit integer, starting at nanoseconds. 52 | // It cannot hold more than 290 years. You can't add any kind of arbitrary precision number of nanoseconds either. 53 | // However, time.Time can handle such things on it's own. 54 | // So, we just add our changetime 100x. It's a little cheesy, but it does work. 55 | for i := 0; i < 100; i++ { 56 | ntdllTime = ntdllTime.Add(time.Duration(fi.ChangeTime)) 57 | } 58 | 59 | return &folderStatWrapper{ 60 | stat, 61 | ntdllTime, 62 | }, nil 63 | } else { 64 | return nil, st.Error() 65 | } 66 | } -------------------------------------------------------------------------------- /cmd/zt_parseSize_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | chk "gopkg.in/check.v1" 25 | ) 26 | 27 | type parseSizeSuite struct{} 28 | 29 | var _ = chk.Suite(&parseSizeSuite{}) 30 | 31 | func (s *parseSizeSuite) TestParseSize(c *chk.C) { 32 | b, _ := ParseSizeString("123K", "x") 33 | c.Assert(b, chk.Equals, int64(123*1024)) 34 | 35 | b, _ = ParseSizeString("456m", "x") 36 | c.Assert(b, chk.Equals, int64(456*1024*1024)) 37 | 38 | b, _ = ParseSizeString("789G", "x") 39 | c.Assert(b, chk.Equals, int64(789*1024*1024*1024)) 40 | 41 | expectedError := "foo-bar must be a number immediately followed by K, M or G. E.g. 12k or 200G" 42 | 43 | _, err := ParseSizeString("123", "foo-bar") 44 | c.Assert(err.Error(), chk.Equals, expectedError) 45 | 46 | _, err = ParseSizeString("123 K", "foo-bar") 47 | c.Assert(err.Error(), chk.Equals, expectedError) 48 | 49 | _, err = ParseSizeString("123KB", "foo-bar") 50 | c.Assert(err.Error(), chk.Equals, expectedError) 51 | 52 | _, err = ParseSizeString("123T", "foo-bar") // we don't support terabytes 53 | c.Assert(err.Error(), chk.Equals, expectedError) 54 | 55 | _, err = ParseSizeString("abcK", "foo-bar") 56 | c.Assert(err.Error(), chk.Equals, expectedError) 57 | 58 | } 59 | -------------------------------------------------------------------------------- /ste/sender_pageBlobFromURL_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package ste 22 | 23 | import ( 24 | "github.com/Azure/azure-storage-blob-go/azblob" 25 | 26 | chk "gopkg.in/check.v1" 27 | ) 28 | 29 | type pageBlobFromURLSuite struct{} 30 | 31 | var _ = chk.Suite(&pageBlobFromURLSuite{}) 32 | 33 | func (s *pageBlobFromURLSuite) TestRangeWorthTransferring(c *chk.C) { 34 | // Arrange 35 | copier := pageRangeOptimizer{} 36 | copier.srcPageList = &azblob.PageList{ 37 | PageRange: []azblob.PageRange{ 38 | {Start: 512, End: 1023}, 39 | {Start: 2560, End: 4095}, 40 | {Start: 7168, End: 8191}, 41 | }, 42 | } 43 | 44 | testCases := map[azblob.PageRange]bool{ 45 | {Start: 512, End: 1023}: true, // fully included 46 | {Start: 2048, End: 3071}: true, // overlapping 47 | {Start: 3071, End: 4606}: true, // overlapping 48 | {Start: 0, End: 511}: false, // before all ranges 49 | {Start: 1536, End: 2559}: false, // in between ranges 50 | {Start: 15360, End: 15871}: false, // all the way out 51 | } 52 | 53 | // Action & Assert 54 | for testRange, expectedResult := range testCases { 55 | doesContainData := copier.doesRangeContainData(testRange) 56 | c.Assert(doesContainData, chk.Equals, expectedResult) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /ste/sourceInfoProvider-Benchmark.go: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package ste 22 | 23 | import ( 24 | "time" 25 | 26 | "github.com/Azure/azure-storage-azcopy/v10/common" 27 | ) 28 | 29 | type benchmarkSourceInfoProvider struct { 30 | jptm IJobPartTransferMgr 31 | } 32 | 33 | func newBenchmarkSourceInfoProvider(jptm IJobPartTransferMgr) (ISourceInfoProvider, error) { 34 | return &benchmarkSourceInfoProvider{jptm}, nil 35 | } 36 | 37 | func (b benchmarkSourceInfoProvider) Properties() (*SrcProperties, error) { 38 | return &SrcProperties{ 39 | SrcHTTPHeaders: common.ResourceHTTPHeaders{}, 40 | SrcMetadata: common.Metadata{}, 41 | SrcBlobTags: common.BlobTags{}, 42 | }, nil 43 | } 44 | 45 | func (b benchmarkSourceInfoProvider) IsLocal() bool { 46 | return true 47 | } 48 | 49 | func (b benchmarkSourceInfoProvider) OpenSourceFile() (common.CloseableReaderAt, error) { 50 | return common.NewRandomDataGenerator(b.jptm.Info().SourceSize), nil 51 | } 52 | 53 | func (b benchmarkSourceInfoProvider) GetFreshFileLastModifiedTime() (time.Time, error) { 54 | return common.BenchmarkLmt, nil 55 | } 56 | 57 | func (b benchmarkSourceInfoProvider) EntityType() common.EntityType { 58 | return common.EEntityType.File() // no folders in benchmark 59 | } 60 | -------------------------------------------------------------------------------- /e2etest/scenario_os_helpers.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | // Copyright © Microsoft 3 | // 4 | // Permission is hereby granted, free of charge, to any person obtaining a copy 5 | // of this software and associated documentation files (the "Software"), to deal 6 | // in the Software without restriction, including without limitation the rights 7 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | // copies of the Software, and to permit persons to whom the Software is 9 | // furnished to do so, subject to the following conditions: 10 | // 11 | // The above copyright notice and this permission notice shall be included in 12 | // all copies or substantial portions of the Software. 13 | // 14 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | // THE SOFTWARE. 21 | 22 | // TODO this file was forked from the cmd package, it needs to cleaned to keep only the necessary part 23 | 24 | package e2etest 25 | 26 | import ( 27 | "time" 28 | ) 29 | 30 | type osScenarioHelper struct{} 31 | 32 | // set file attributes to test file 33 | //nolint 34 | func (osScenarioHelper) setAttributesForLocalFile() error { 35 | panic("should never be called") 36 | } 37 | 38 | //nolint 39 | func (osScenarioHelper) setAttributesForLocalFiles(c asserter, dirPath string, fileList []string, attrList []string) { 40 | panic("should never be called") 41 | } 42 | 43 | //nolint 44 | func (osScenarioHelper) getFileDates(c asserter, filePath string) (createdTime, lastWriteTime time.Time) { 45 | panic("should never be called") 46 | } 47 | 48 | //nolint 49 | func (osScenarioHelper) getFileAttrs(c asserter, filepath string) *uint32 { 50 | var ret uint32 51 | return &ret 52 | } 53 | 54 | //nolint 55 | func (osScenarioHelper) getFileSDDLString(c asserter, filepath string) *string { 56 | ret := "" 57 | return &ret 58 | } 59 | 60 | //nolint 61 | func (osScenarioHelper) setFileSDDLString(c asserter, filepath string, sddldata string) { 62 | panic("should never be called") 63 | } 64 | -------------------------------------------------------------------------------- /common/uuid.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "crypto/rand" 5 | "encoding/json" 6 | "fmt" 7 | "unsafe" 8 | ) 9 | 10 | // The JobID reserved variants. 11 | const ( 12 | reservedNCS byte = 0x80 13 | reservedRFC4122 byte = 0x40 14 | reservedMicrosoft byte = 0x20 15 | reservedFuture byte = 0x00 16 | guidFormat = "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x" 17 | ) 18 | 19 | // A UUID representation compliant with specification in RFC 4122 document. 20 | type UUID struct { 21 | D1 uint32 22 | D2 uint16 23 | D3 uint16 24 | D4 [8]uint8 25 | } 26 | 27 | // NewUUID returns a new UUID using RFC 4122 algorithm. 28 | func NewUUID() (u UUID) { 29 | u = UUID{} 30 | // Set all bits to randomly (or pseudo-randomly) chosen values. 31 | uuid := (*[16]byte)(unsafe.Pointer(&u))[:] 32 | _, err := rand.Reader.Read(uuid) 33 | if err != nil { 34 | panic("rand.Read failed") 35 | } 36 | uuid[8] = (uuid[8] | reservedRFC4122) & 0x7F // u.setVariant(ReservedRFC4122) 37 | 38 | var version byte = 4 39 | uuid[6] = (uuid[6] & 0xF) | (version << 4) // u.setVersion(4) 40 | return 41 | } 42 | 43 | // String returns an unparsed version of the generated UUID sequence. 44 | func (u UUID) String() string { 45 | return fmt.Sprintf(guidFormat, u.D1, u.D2, u.D3, u.D4[0], u.D4[1], u.D4[2], u.D4[3], u.D4[4], u.D4[5], u.D4[6], u.D4[7]) 46 | } 47 | 48 | // Implementing MarshalJSON() method for type UUID 49 | func (u UUID) MarshalJSON() ([]byte, error) { 50 | s := u.String() 51 | return json.Marshal(s) 52 | } 53 | 54 | // Implementing UnmarshalJSON() method for type UUID 55 | func (u *UUID) UnmarshalJSON(b []byte) error { 56 | var s string 57 | if err := json.Unmarshal(b, &s); err != nil { 58 | return err 59 | } 60 | uuid, err := ParseUUID(s) 61 | if err != nil { 62 | return err 63 | } 64 | *u = uuid 65 | return nil 66 | } 67 | 68 | // ParseUUID parses a string formatted as "003020100-0504-0706-0809-0a0b0c0d0e0f" 69 | // or "{03020100-0504-0706-0809-0a0b0c0d0e0f}" into a UUID. 70 | func ParseUUID(uuidStr string) (UUID, error) { 71 | if uuidStr[0] == '{' { 72 | uuidStr = uuidStr[1:] // Skip over the '{' 73 | } 74 | uuid := UUID{} 75 | _, err := fmt.Sscanf(uuidStr, guidFormat, 76 | &uuid.D1, &uuid.D2, &uuid.D3, 77 | &uuid.D4[0], &uuid.D4[1], &uuid.D4[2], &uuid.D4[3], &uuid.D4[4], &uuid.D4[5], &uuid.D4[6], &uuid.D4[7]) 78 | if err != nil { 79 | return UUID{}, err 80 | } 81 | return uuid, nil 82 | } 83 | -------------------------------------------------------------------------------- /e2etest/zt_naming_file_and_folder_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package e2etest 22 | 23 | import ( 24 | "github.com/Azure/azure-storage-azcopy/v10/common" 25 | "testing" 26 | ) 27 | 28 | // Upload, Download, S2S transfer of folders/files with special characters. Required for avoiding regression. 29 | func TestNaming_ShareFileFoldersSpecialChar(t *testing.T) { 30 | files := []string{"file1.txt", "fi,le2.pdf", "fil%e3.mp3", "file 4.jpg", "file;a5.csv", "file_a6.cpp", "file+a7.mp4"} 31 | folders := []string{";", ";;", "%", "_", "+", "test%folder1", "test+folder2", "test,folder3", "test folder4", "test_folder5", "test;folder6"} 32 | transfers := make([]interface{}, 0) 33 | transfers = append(transfers, folder("")) 34 | for i := 0; i < len(folders); i++ { 35 | transfers = append(transfers, folder(folders[i])) 36 | for j := 0; j < len(files); j++ { 37 | transfers = append(transfers, f(folders[i]+"/"+files[j])) 38 | } 39 | } 40 | RunScenarios(t, eOperation.CopyAndSync(), eTestFromTo.Other(common.EFromTo.FileFile(), common.EFromTo.FileLocal(), common.EFromTo.LocalFile()), eValidate.Auto(), params{ 41 | recursive: true, 42 | }, nil, testFiles{ 43 | defaultSize: "1K", 44 | shouldTransfer: transfers, 45 | }, EAccountType.Standard(), EAccountType.Standard(), "") 46 | } 47 | -------------------------------------------------------------------------------- /cmd/copyEnumeratorHelper_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "github.com/Azure/azure-storage-azcopy/v10/common" 25 | chk "gopkg.in/check.v1" 26 | ) 27 | 28 | type copyEnumeratorHelperTestSuite struct{} 29 | 30 | var _ = chk.Suite(©EnumeratorHelperTestSuite{}) 31 | 32 | func newLocalRes(path string) common.ResourceString { 33 | return common.ResourceString{Value: path} 34 | } 35 | 36 | func newRemoteRes(url string) common.ResourceString { 37 | r, err := SplitResourceString(url, common.ELocation.Blob()) 38 | if err != nil { 39 | panic("can't parse resource string") 40 | } 41 | return r 42 | } 43 | 44 | func (s *copyEnumeratorHelperTestSuite) TestAddTransferPathRootsTrimmed(c *chk.C) { 45 | // setup 46 | request := common.CopyJobPartOrderRequest{ 47 | SourceRoot: newLocalRes("a/b/"), 48 | DestinationRoot: newLocalRes("y/z/"), 49 | } 50 | 51 | transfer := common.CopyTransfer{ 52 | Source: "a/b/c.txt", 53 | Destination: "y/z/c.txt", 54 | } 55 | 56 | // execute 57 | err := addTransfer(&request, transfer, &CookedCopyCmdArgs{}) 58 | 59 | // assert 60 | c.Assert(err, chk.IsNil) 61 | c.Assert(request.Transfers.List[0].Source, chk.Equals, "c.txt") 62 | c.Assert(request.Transfers.List[0].Destination, chk.Equals, "c.txt") 63 | } 64 | -------------------------------------------------------------------------------- /common/emptyChunkReader.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package common 22 | 23 | import ( 24 | "errors" 25 | "hash" 26 | "io" 27 | ) 28 | 29 | // emptyChunkReader satisfies the SingleChunkReader interface for empty chunks (i.e. the first and only chunk of an empty file) 30 | type emptyChunkReader struct { 31 | } 32 | 33 | func (cr *emptyChunkReader) BlockingPrefetch(fileReader io.ReaderAt, isRetry bool) error { 34 | return nil 35 | } 36 | 37 | func (cr *emptyChunkReader) Seek(offset int64, whence int) (int64, error) { 38 | if whence == io.SeekEnd && offset > 0 || offset < 0 { 39 | return 0, errors.New("cannot seek to before beginning") 40 | } 41 | return 0, nil 42 | } 43 | 44 | func (cr *emptyChunkReader) Read(p []byte) (n int, err error) { 45 | return 0, io.EOF 46 | } 47 | 48 | func (cr *emptyChunkReader) Close() error { 49 | return nil 50 | } 51 | 52 | func (cr *emptyChunkReader) GetPrologueState() PrologueState { 53 | return PrologueState{} 54 | } 55 | 56 | func (cr *emptyChunkReader) HasPrefetchedEntirelyZeros() bool { 57 | return false // we don't have any zeros (or anything else for that matter) 58 | } 59 | 60 | func (cr *emptyChunkReader) Length() int64 { 61 | return 0 62 | } 63 | 64 | func (cr *emptyChunkReader) WriteBufferTo(h hash.Hash) { 65 | return // no content to write 66 | } 67 | -------------------------------------------------------------------------------- /cmd/jobsList_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "time" 25 | 26 | "github.com/Azure/azure-storage-azcopy/v10/common" 27 | 28 | chk "gopkg.in/check.v1" 29 | ) 30 | 31 | type jobsListTestSuite struct{} 32 | 33 | var _ = chk.Suite(&jobsListTestSuite{}) 34 | 35 | func (s *jobsListTestSuite) TestSortJobs(c *chk.C) { 36 | // setup 37 | job2 := common.JobIDDetails{ 38 | JobId: common.NewJobID(), 39 | StartTime: time.Now().UnixNano(), 40 | CommandString: "dummy2", 41 | } 42 | 43 | // sleep for a bit so that the time stamp is different 44 | time.Sleep(time.Millisecond) 45 | job1 := common.JobIDDetails{ 46 | JobId: common.NewJobID(), 47 | StartTime: time.Now().UnixNano(), 48 | CommandString: "dummy1", 49 | } 50 | 51 | // sleep for a bit so that the time stamp is different 52 | time.Sleep(time.Millisecond) 53 | job0 := common.JobIDDetails{ 54 | JobId: common.NewJobID(), 55 | StartTime: time.Now().UnixNano(), 56 | CommandString: "dummy0", 57 | } 58 | jobsList := []common.JobIDDetails{job2, job1, job0} 59 | 60 | // act 61 | sortJobs(jobsList) 62 | 63 | // verify 64 | c.Assert(jobsList[0], chk.DeepEquals, job0) 65 | c.Assert(jobsList[1], chk.DeepEquals, job1) 66 | c.Assert(jobsList[2], chk.DeepEquals, job2) 67 | } 68 | -------------------------------------------------------------------------------- /tool_distributed_mutex.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import time 3 | import random 4 | 5 | # note that the track 2 python SDK is being used here 6 | from azure.storage.blob import ( 7 | BlobClient, 8 | LeaseClient, 9 | ) 10 | 11 | from azure.core import ( 12 | HttpResponseError, 13 | ) 14 | 15 | LOCK = 'lock' 16 | UNLOCK = 'unlock' 17 | 18 | 19 | def get_raw_input(): 20 | parser = argparse.ArgumentParser(description='Lock/unlock a distributed mutex (implemented with blob lease)') 21 | parser.add_argument('action', help='can be "lock" (attempt to acquire lease) or "unlock" (break lease)') 22 | parser.add_argument('mutex_url', help='points to a blob url (SAS included)') 23 | args = parser.parse_args() 24 | 25 | if args.action not in [LOCK, UNLOCK]: 26 | raise ValueError('invalid action, can only be "lock" or "unlock"') 27 | return args.action, args.mutex_url 28 | 29 | 30 | def process(): 31 | action, mutex_url = get_raw_input() 32 | 33 | # check whether the blob exists, if not quit right away to avoid wasting time 34 | blob_client = BlobClient(mutex_url) 35 | try: 36 | blob_client.get_blob_properties() 37 | print("INFO: validated mutex url") 38 | except HttpResponseError as e: 39 | raise ValueError('please provide an existing and valid blob URL, failed to get properties with error: ' + e) 40 | 41 | # get a handle on the lease 42 | lease_client = LeaseClient(blob_client) 43 | if action == UNLOCK: 44 | # make the lease free as soon as possible 45 | lease_client.break_lease(lease_break_period=1) 46 | print(f"INFO: successfully unlocked the mutex!") 47 | return 48 | 49 | # action is lock, attempt to acquire the lease continuously 50 | while True: 51 | # try to acquire and infinite lease 52 | try: 53 | lease_client.acquire(lease_duration=-1) 54 | 55 | # if we get here, the acquire call succeeded 56 | # if we don't get here it stalls forever, as expected 57 | print(f"INFO: successfully locked the mutex!") 58 | return 59 | except HttpResponseError: 60 | # failed to acquire lease, another agent holds the mutex 61 | # sleep a bit (randomly) and try again 62 | sleep_period = random.randint(1, 5) 63 | print(f"INFO: failed to lock mutex, wait for {sleep_period} and try again") 64 | time.sleep(sleep_period) 65 | 66 | 67 | if __name__ == '__main__': 68 | process() 69 | -------------------------------------------------------------------------------- /cmd/zt_user_input_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | chk "gopkg.in/check.v1" 25 | ) 26 | 27 | func (s *cmdIntegrationSuite) TestCPKEncryptionInputTest(c *chk.C) { 28 | mockedRPC := interceptor{} 29 | Rpc = mockedRPC.intercept 30 | mockedRPC.init() 31 | 32 | dirPath := "this/is/a/dummy/path" 33 | rawDFSEndpointWithSAS := scenarioHelper{}.getRawAdlsServiceURLWithSAS(c) 34 | raw := getDefaultRawCopyInput(dirPath, rawDFSEndpointWithSAS.String()) 35 | raw.recursive = true 36 | raw.cpkInfo = true 37 | 38 | runCopyAndVerify(c, raw, func(err error) { 39 | c.Assert(err, chk.NotNil) 40 | c.Assert(err.Error(), StringContains, "client provided keys (CPK) based encryption is only supported with blob endpoints (blob.core.windows.net)") 41 | }) 42 | 43 | mockedRPC.reset() 44 | raw.cpkInfo = false 45 | raw.cpkScopeInfo = "dummyscope" 46 | runCopyAndVerify(c, raw, func(err error) { 47 | c.Assert(err, chk.NotNil) 48 | c.Assert(err.Error(), StringContains, "client provided keys (CPK) based encryption is only supported with blob endpoints (blob.core.windows.net)") 49 | }) 50 | 51 | rawContainerURL := scenarioHelper{}.getContainerURL(c, "testcpkcontainer") 52 | raw2 := getDefaultRawCopyInput(dirPath, rawContainerURL.String()) 53 | raw2.recursive = true 54 | raw2.cpkInfo = true 55 | 56 | _, err := raw2.cook() 57 | c.Assert(err, chk.IsNil) 58 | } 59 | -------------------------------------------------------------------------------- /common/zt_multiSliceBytePooler_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package common 22 | 23 | import ( 24 | "math" 25 | 26 | chk "gopkg.in/check.v1" 27 | ) 28 | 29 | type multiSliceBytePoolerSuite struct{} 30 | 31 | var _ = chk.Suite(&multiSliceBytePoolerSuite{}) 32 | 33 | func (s *multiSliceBytePoolerSuite) TestMultiSliceSlotInfo(c *chk.C) { 34 | eightMB := 8 * 1024 * 1024 35 | 36 | cases := []struct { 37 | size int 38 | expectedSlotIndex int 39 | expectedMaxCapInSlot int 40 | }{ 41 | {1, 0, 1}, 42 | {2, 1, 2}, 43 | {3, 2, 4}, 44 | {4, 2, 4}, 45 | {5, 3, 8}, 46 | {8, 3, 8}, 47 | {9, 4, 16}, 48 | {eightMB - 1, 23, eightMB}, 49 | {eightMB, 23, eightMB}, 50 | {eightMB + 1, 24, eightMB * 2}, 51 | {100 * 1024 * 1024, 27, 128 * 1024 * 1024}, 52 | } 53 | 54 | for _, x := range cases { 55 | logBase2 := math.Log2(float64(x.size)) 56 | roundedLogBase2 := int(math.Round(logBase2 + 0.49999999999999)) // rounds up unless already exact(ish) 57 | 58 | // now lets see if the pooler is working as we expect 59 | slotIndex, maxCap := getSlotInfo(int64(x.size)) 60 | 61 | c.Assert(slotIndex, chk.Equals, roundedLogBase2) // this what, mathematically, we expect 62 | c.Assert(slotIndex, chk.Equals, x.expectedSlotIndex) // this what our test case said (should be same) 63 | 64 | c.Assert(maxCap, chk.Equals, x.expectedMaxCapInSlot) 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /azbfs/zc_util_validate.go: -------------------------------------------------------------------------------- 1 | package azbfs 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "strconv" 8 | ) 9 | 10 | const ( 11 | // CountToEnd indicates a flag for count parameter. It means the count of bytes 12 | // from start offset to the end of file. 13 | CountToEnd = 0 14 | ) 15 | 16 | // httpRange defines a range of bytes within an HTTP resource, starting at offset and 17 | // ending at offset+count-1 inclusively. 18 | // An httpRange which has a zero-value offset, and a count with value CountToEnd indicates the entire resource. 19 | // An httpRange which has a non zero-value offset but a count with value CountToEnd indicates from the offset to the resource's end. 20 | type httpRange struct { 21 | offset int64 22 | count int64 23 | } 24 | 25 | func (r httpRange) pointers() *string { 26 | if r.offset == 0 && r.count == CountToEnd { // Do common case first for performance 27 | return nil // No specified range 28 | } 29 | if r.offset < 0 { 30 | panic("The range offset must be >= 0") 31 | } 32 | if r.count <= 0 && r.count != CountToEnd { 33 | panic("The range count must be either equal to CountToEnd (0) or > 0") 34 | } 35 | 36 | return toRange(r.offset, r.count) 37 | } 38 | 39 | // toRange makes range string adhere to REST API. 40 | // A count with value CountToEnd means count of bytes from offset to the end of file. 41 | // For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-the-range-header-for-file-service-operations. 42 | func toRange(offset int64, count int64) *string { 43 | // No additional validation by design. API can validate parameter by case, and use this method. 44 | endRange := "" 45 | if count != CountToEnd { 46 | endRange = strconv.FormatInt(offset+count-1, 10) 47 | } 48 | r := fmt.Sprintf("bytes=%d-%s", offset, endRange) 49 | return &r 50 | } 51 | 52 | func validateSeekableStreamAt0AndGetCount(body io.ReadSeeker) int64 { 53 | if body == nil { // nil body's are "logically" seekable to 0 and are 0 bytes long 54 | return 0 55 | } 56 | validateSeekableStreamAt0(body) 57 | count, err := body.Seek(0, io.SeekEnd) 58 | if err != nil { 59 | panic("failed to seek stream") 60 | } 61 | body.Seek(0, io.SeekStart) 62 | return count 63 | } 64 | 65 | func validateSeekableStreamAt0(body io.ReadSeeker) { 66 | if body == nil { // nil body's are "logically" seekable to 0 67 | return 68 | } 69 | if pos, err := body.Seek(0, io.SeekCurrent); pos != 0 || err != nil { 70 | if err != nil { 71 | panic(err) 72 | } 73 | panic(errors.New("stream must be set to position 0")) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /ste/remoteObjectExists.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package ste 22 | 23 | import ( 24 | "net/http" 25 | "time" 26 | ) 27 | 28 | // an error with an HTTP Response 29 | type responseError interface { 30 | Response() *http.Response 31 | } 32 | 33 | type lastModifiedTimerProvider interface { 34 | LastModified() time.Time 35 | } 36 | 37 | // remoteObjectExists takes the error returned when trying to access a remote object, sees whether is 38 | // a "not found" error. If the object exists (i.e. error is nil) it returns (true, nil). If the 39 | // error is a "not found" error, it returns (false, nil). Else it returns false and the original error. 40 | // The initial, dummy, parameter, is to allow callers to conveniently call it with functions that return a tuple 41 | // - even though we only need the error. 42 | func remoteObjectExists(props lastModifiedTimerProvider, errWhenAccessingRemoteObject error) (bool, time.Time, error) { 43 | 44 | if typedErr, ok := errWhenAccessingRemoteObject.(responseError); ok && typedErr.Response().StatusCode == http.StatusNotFound { 45 | return false, time.Time{}, nil // 404 error, so it does NOT exist 46 | } else if errWhenAccessingRemoteObject != nil { 47 | return false, time.Time{}, errWhenAccessingRemoteObject // some other error happened, so we return it 48 | } else { 49 | return true, props.LastModified(), nil // If err equals nil, the file exists 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /testSuite/cmd/root.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "fmt" 25 | "os" 26 | 27 | "github.com/spf13/cobra" 28 | ) 29 | 30 | var cfgFile string 31 | 32 | // rootCmd represents the base command when called without any subcommands 33 | var rootCmd = &cobra.Command{ 34 | Use: "validator", 35 | Short: "validator is a cli based test framework for azcopy.", 36 | Long: `validator is a cli based test framework for azcopy.`, 37 | // Uncomment the following line if your bare application 38 | // has an action associated with it: 39 | // Run: func(cmd *cobra.Command, args []string) { }, 40 | } 41 | 42 | // Execute adds all child commands to the root command and sets flags appropriately. 43 | // This is called by main.main(). It only needs to happen once to the rootCmd. 44 | func Execute() { 45 | if err := rootCmd.Execute(); err != nil { 46 | fmt.Println(err) 47 | os.Exit(1) 48 | } 49 | } 50 | 51 | func init() { 52 | // Here you will define your flags and configuration settings. 53 | // Cobra supports persistent flags, which, if defined here, 54 | // will be global for your application. 55 | rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.validator.yaml)") 56 | 57 | // Cobra also supports local flags, which will only run 58 | // when this action is called directly. 59 | rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") 60 | } 61 | -------------------------------------------------------------------------------- /common/writeThoughFile_linux.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package common 22 | 23 | import ( 24 | "os" 25 | "syscall" 26 | ) 27 | 28 | func CreateFileOfSizeWithWriteThroughOption(destinationPath string, fileSize int64, writeThrough bool, t FolderCreationTracker, forceIfReadOnly bool) (*os.File, error) { 29 | // forceIfReadOnly is not used on this OS 30 | 31 | err := CreateParentDirectoryIfNotExist(destinationPath, t) 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | flags := os.O_RDWR | os.O_CREATE | os.O_TRUNC 37 | if writeThrough { 38 | // TODO: conduct further testing of this code path, on Linux 39 | flags = flags | os.O_SYNC // technically, O_DSYNC may be very slightly faster, but its not exposed in the os package 40 | } 41 | f, err := os.OpenFile(destinationPath, flags, DEFAULT_FILE_PERM) 42 | if err != nil { 43 | return nil, err 44 | } 45 | 46 | if fileSize == 0 { 47 | return f, err 48 | } 49 | 50 | err = syscall.Fallocate(int(f.Fd()), 0, 0, fileSize) 51 | if err != nil { 52 | // To solve the case that Fallocate cannot work well with cifs/smb3. 53 | if err == syscall.ENOTSUP { 54 | if truncateError := f.Truncate(fileSize); truncateError != nil { 55 | return nil, truncateError 56 | } 57 | } else { 58 | return nil, err 59 | } 60 | } 61 | return f, nil 62 | } 63 | 64 | func SetBackupMode(enable bool, fromTo FromTo) error { 65 | // n/a on this platform 66 | return nil 67 | } 68 | -------------------------------------------------------------------------------- /common/writeThroughFile_darwin.go: -------------------------------------------------------------------------------- 1 | // +build linux darwin 2 | 3 | // Copyright © 2017 Microsoft 4 | // 5 | // Permission is hereby granted, free of charge, to any person obtaining a copy 6 | // of this software and associated documentation files (the "Software"), to deal 7 | // in the Software without restriction, including without limitation the rights 8 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | // copies of the Software, and to permit persons to whom the Software is 10 | // furnished to do so, subject to the following conditions: 11 | // 12 | // The above copyright notice and this permission notice shall be included in 13 | // all copies or substantial portions of the Software. 14 | // 15 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | // THE SOFTWARE. 22 | 23 | package common 24 | 25 | import ( 26 | "os" 27 | ) 28 | 29 | func CreateFileOfSizeWithWriteThroughOption(destinationPath string, fileSize int64, writeThrough bool, t FolderCreationTracker, forceIfReadOnly bool) (*os.File, error) { 30 | // forceIfReadOnly is not used on this OS 31 | 32 | err := CreateParentDirectoryIfNotExist(destinationPath, t) 33 | if err != nil { 34 | return nil, err 35 | } 36 | 37 | // TODO: currently writeThrough is ignored here on darwin. Review, and use it if we have an implementation we are comfortable with 38 | // A quick internet search returned conflicting opinions on whether MacOS suppose O_SYNC or uses a different flag with the same meaning. 39 | // If different with same meaning, can we just use O_SYNC here? That's what we need to find out before implementing. 40 | 41 | f, err := os.OpenFile(destinationPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, DEFAULT_FILE_PERM) 42 | if err != nil { 43 | return nil, err 44 | } 45 | //err = syscall.Fallocate(int(f.Fd()), 0, 0, fileSize) 46 | //if err != nil { 47 | // return nil, err 48 | //} 49 | // TODO: Need to appropriate fallocate api for darwin 50 | if truncateError := f.Truncate(fileSize); truncateError != nil { 51 | return nil, truncateError 52 | } 53 | return f, nil 54 | } 55 | 56 | func SetBackupMode(enable bool, fromTo FromTo) error { 57 | // n/a on this platform 58 | return nil 59 | } 60 | -------------------------------------------------------------------------------- /cmd/loadCLFS_test.go: -------------------------------------------------------------------------------- 1 | // +build linux 2 | 3 | // Copyright © Microsoft 4 | // 5 | // Permission is hereby granted, free of charge, to any person obtaining a copy 6 | // of this software and associated documentation files (the "Software"), to deal 7 | // in the Software without restriction, including without limitation the rights 8 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | // copies of the Software, and to permit persons to whom the Software is 10 | // furnished to do so, subject to the following conditions: 11 | // 12 | // The above copyright notice and this permission notice shall be included in 13 | // all copies or substantial portions of the Software. 14 | // 15 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | // THE SOFTWARE. 22 | 23 | package cmd 24 | 25 | import ( 26 | "fmt" 27 | "os" 28 | "path/filepath" 29 | 30 | chk "gopkg.in/check.v1" 31 | ) 32 | 33 | type loadCmdTestSuite struct{} 34 | 35 | var _ = chk.Suite(&loadCmdTestSuite{}) 36 | 37 | func (s *loadCmdTestSuite) TestParsingRawArgs(c *chk.C) { 38 | sampleAccountName := "accountname" 39 | sampleContainerName := "cont" 40 | sampleSAS := "st=2019-09-23T07%3A06%3A55Z&se=2019-09-24T07%3A06%3A55Z&sp=racwdl&sv=2018-03-28&sr=c&sig=FGDUz8Jb1F%2Fsh%2FLKdX1IBYQR5n5LKfk46GRAeVDYeW4%3D" 41 | raw := rawLoadCmdArgs{ 42 | src: os.TempDir(), 43 | dst: fmt.Sprintf("https://%s.blob.core.windows.net/%s?%s", sampleAccountName, sampleContainerName, sampleSAS), 44 | newSession: true, 45 | statePath: "/usr/dir/state", 46 | } 47 | 48 | cooked, err := raw.cook() 49 | c.Assert(err, chk.IsNil) 50 | 51 | // validate the straightforward args 52 | cleanedSrc, err := filepath.Abs(raw.src) 53 | c.Assert(err, chk.IsNil) 54 | 55 | c.Assert(cooked.src, chk.Equals, cleanedSrc) 56 | c.Assert(cooked.newSession, chk.Equals, raw.newSession) 57 | c.Assert(cooked.statePath, chk.Equals, raw.statePath) 58 | 59 | // check the dst related args 60 | c.Assert(cooked.dstAccount, chk.Equals, sampleAccountName) 61 | c.Assert(cooked.dstContainer, chk.Equals, sampleContainerName) 62 | 63 | // the order of sas query params gets mingled 64 | c.Assert(len(cooked.dstSAS), chk.Equals, len(sampleSAS)) 65 | } 66 | -------------------------------------------------------------------------------- /ste/zt_ste_misc_windows_test.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | // Copyright © Microsoft 3 | // 4 | // Permission is hereby granted, free of charge, to any person obtaining a copy 5 | // of this software and associated documentation files (the "Software"), to deal 6 | // in the Software without restriction, including without limitation the rights 7 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | // copies of the Software, and to permit persons to whom the Software is 9 | // furnished to do so, subject to the following conditions: 10 | // 11 | // The above copyright notice and this permission notice shall be included in 12 | // all copies or substantial portions of the Software. 13 | // 14 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | // THE SOFTWARE. 21 | 22 | package ste 23 | 24 | import ( 25 | chk "gopkg.in/check.v1" 26 | ) 27 | 28 | type steMiscSuite struct{} 29 | 30 | var _ = chk.Suite(&steMiscSuite{}) 31 | 32 | func (s *concurrencyTunerSuite) Test_IsParentShareRoot(c *chk.C) { 33 | d := azureFilesDownloader{} 34 | 35 | c.Assert(d.parentIsShareRoot("https://a.file.core.windows.net/share"), chk.Equals, false) // THIS is the share root, not the parent of this 36 | c.Assert(d.parentIsShareRoot("https://a.file.core.windows.net/share/"), chk.Equals, false) 37 | c.Assert(d.parentIsShareRoot("https://a.file.core.windows.net/share?aaa/bbb"), chk.Equals, false) 38 | c.Assert(d.parentIsShareRoot("https://a.file.core.windows.net/share/?aaa/bbb"), chk.Equals, false) 39 | 40 | c.Assert(d.parentIsShareRoot("https://a.file.core.windows.net/share/foo"), chk.Equals, true) 41 | c.Assert(d.parentIsShareRoot("https://a.file.core.windows.net/share/foo/"), chk.Equals, true) 42 | c.Assert(d.parentIsShareRoot("https://a.file.core.windows.net/share/foo/?x/y"), chk.Equals, true) 43 | c.Assert(d.parentIsShareRoot("https://a.file.core.windows.net/share/foo?x/y"), chk.Equals, true) 44 | 45 | c.Assert(d.parentIsShareRoot("https://a.file.core.windows.net/share/foo/bar"), chk.Equals, false) 46 | c.Assert(d.parentIsShareRoot("https://a.file.core.windows.net/share/foo/bar/"), chk.Equals, false) 47 | c.Assert(d.parentIsShareRoot("https://a.file.core.windows.net/share/foo/bar?nethe"), chk.Equals, false) 48 | } 49 | -------------------------------------------------------------------------------- /azbfs/zz_generated_responder_policy.go: -------------------------------------------------------------------------------- 1 | package azbfs 2 | 3 | // Code generated by Microsoft (R) AutoRest Code Generator. 4 | // Changes may cause incorrect behavior and will be lost if the code is regenerated. 5 | 6 | import ( 7 | "bytes" 8 | "context" 9 | "encoding/json" 10 | "github.com/Azure/azure-pipeline-go/pipeline" 11 | "io/ioutil" 12 | ) 13 | 14 | type responder func(resp pipeline.Response) (result pipeline.Response, err error) 15 | 16 | // ResponderPolicyFactory is a Factory capable of creating a responder pipeline. 17 | type responderPolicyFactory struct { 18 | responder responder 19 | } 20 | 21 | // New creates a responder policy factory. 22 | func (arpf responderPolicyFactory) New(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.Policy { 23 | return responderPolicy{next: next, responder: arpf.responder} 24 | } 25 | 26 | type responderPolicy struct { 27 | next pipeline.Policy 28 | responder responder 29 | } 30 | 31 | // Do sends the request to the service and validates/deserializes the HTTP response. 32 | func (arp responderPolicy) Do(ctx context.Context, request pipeline.Request) (pipeline.Response, error) { 33 | resp, err := arp.next.Do(ctx, request) 34 | if err != nil { 35 | return resp, err 36 | } 37 | return arp.responder(resp) 38 | } 39 | 40 | // validateResponse checks an HTTP response's status code against a legal set of codes. 41 | // If the response code is not legal, then validateResponse reads all of the response's body 42 | // (containing error information) and returns a response error. 43 | func validateResponse(resp pipeline.Response, successStatusCodes ...int) error { 44 | if resp == nil { 45 | return NewResponseError(nil, nil, "nil response") 46 | } 47 | responseCode := resp.Response().StatusCode 48 | for _, i := range successStatusCodes { 49 | if i == responseCode { 50 | return nil 51 | } 52 | } 53 | // only close the body in the failure case. in the 54 | // success case responders will close the body as required. 55 | defer resp.Response().Body.Close() 56 | b, err := ioutil.ReadAll(resp.Response().Body) 57 | if err != nil { 58 | return err 59 | } 60 | // the service code, description and details will be populated during unmarshalling 61 | responseError := NewResponseError(nil, resp.Response(), resp.Response().Status) 62 | if len(b) > 0 { 63 | if err = json.Unmarshal(b, &responseError); err != nil { 64 | return NewResponseError(err, resp.Response(), "failed to unmarshal response body") 65 | } 66 | } 67 | return responseError 68 | } 69 | 70 | // removes any BOM from the byte slice 71 | func removeBOM(b []byte) []byte { 72 | // UTF8 73 | return bytes.TrimPrefix(b, []byte("\xef\xbb\xbf")) 74 | } 75 | -------------------------------------------------------------------------------- /ste/sender_blockBlob_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package ste 22 | 23 | import ( 24 | "fmt" 25 | 26 | chk "gopkg.in/check.v1" 27 | ) 28 | 29 | type blockBlobSuite struct{} 30 | 31 | var _ = chk.Suite(&blockBlobSuite{}) 32 | 33 | func (s *blockBlobSuite) TestGetVerifiedChunkParams(c *chk.C) { 34 | // Mock required params 35 | transferInfo := TransferInfo{ 36 | BlockSize: 4195352576, // 4001MiB 37 | Source: "tmpSrc", 38 | SourceSize: 8389656576, // 8001MiB 39 | } 40 | 41 | //Verify memory limit 42 | memLimit := int64(2097152000) // 2000Mib 43 | expectedErr := fmt.Sprintf("Cannot use a block size of 3.91GiB. AzCopy is limited to use only 1.95GiB of memory") 44 | _, _, err := getVerifiedChunkParams(transferInfo, memLimit) 45 | c.Assert(err.Error(), chk.Equals, expectedErr) 46 | 47 | // Verify large block Size 48 | memLimit = int64(8388608000) // 8000MiB 49 | expectedErr = fmt.Sprintf("block size of 3.91GiB for file tmpSrc of size 7.81GiB exceeds maxmimum allowed block size for a BlockBlob") 50 | _, _, err = getVerifiedChunkParams(transferInfo, memLimit) 51 | c.Assert(err.Error(), chk.Equals, expectedErr) 52 | 53 | // High block count 54 | transferInfo.SourceSize = 2147483648 //16GiB 55 | transferInfo.BlockSize = 2048 // 2KiB 56 | expectedErr = fmt.Sprintf("Block size 2048 for source of size 2147483648 is not correct. Number of blocks will exceed the limit") 57 | _, _, err = getVerifiedChunkParams(transferInfo, memLimit) 58 | c.Assert(err.Error(), chk.Equals, expectedErr) 59 | 60 | } 61 | -------------------------------------------------------------------------------- /ste/mgr-JobPartMgr_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package ste 22 | 23 | import ( 24 | chk "gopkg.in/check.v1" 25 | "strings" 26 | "testing" 27 | ) 28 | 29 | // Hookup to the testing framework 30 | func Test(t *testing.T) { chk.TestingT(t) } 31 | 32 | type jobPartMgrTestSuite struct{} 33 | 34 | var _ = chk.Suite(&jobPartMgrTestSuite{}) 35 | 36 | func (s *jobPartMgrTestSuite) TestInferContentType(c *chk.C) { 37 | // Arrange 38 | partMgr := jobPartMgr{} 39 | 40 | // the goal is to make sure content type detection at least works for static websites 41 | testCases := map[string]string{ 42 | "/usr/foo/bla.txt": "text/plain", 43 | "/usr/foo/bla.html": "text/html", 44 | "/usr/foo/bla.css": "text/css", 45 | "/usr/foo/bla.js": "application/javascript", 46 | "/usr/foo/bla.json": "application/json", 47 | "/usr/foo/bla.jpeg": "image/jpeg", 48 | "/usr/foo/bla.png": "image/png", 49 | "/usr/foo/bla.multiple.dot.js": "application/javascript", 50 | "/usr/foo/no/extension": "application/octet-stream", 51 | "/usr/foo/bla.HTML": "text/html", 52 | } 53 | 54 | // Action & Assert 55 | for testPath, expectedType := range testCases { 56 | contentType := partMgr.inferContentType(testPath, make([]byte, 5)) 57 | 58 | // make sure the inferred type is correct 59 | // we use Contains to check because charset is also in contentType 60 | c.Assert(strings.Contains(contentType, expectedType), chk.Equals, true) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /cmd/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "fmt" 25 | "os" 26 | 27 | "github.com/spf13/cobra" 28 | "github.com/spf13/cobra/doc" 29 | ) 30 | 31 | var docCmdInput = struct { 32 | outputLocation string 33 | }{} 34 | 35 | // docCmd represents the doc command 36 | var docCmd = &cobra.Command{ 37 | Use: "doc", 38 | Short: docCmdShortDescription, 39 | Long: docCmdLongDescription, 40 | Run: func(cmd *cobra.Command, args []string) { 41 | // verify the output location 42 | f, err := os.Stat(docCmdInput.outputLocation) 43 | if err != nil && os.IsNotExist(err) { 44 | // create the output location if it does not exist yet 45 | if err = os.MkdirAll(docCmdInput.outputLocation, os.ModePerm); err != nil { 46 | glcm.Error("Unable to create output location due to error: " + err.Error()) 47 | } 48 | } else if err != nil { 49 | glcm.Error("Cannot access the output location due to error: " + err.Error()) 50 | } else if !f.IsDir() { 51 | glcm.Error("The output location is invalid as it is pointing to a file.") 52 | } 53 | 54 | // dump the entire command tree's doc into the folder 55 | // it will include this command too, which is intended 56 | err = doc.GenMarkdownTree(rootCmd, docCmdInput.outputLocation) 57 | if err != nil { 58 | glcm.Error(fmt.Sprintf("Cannot generate doc due to error %s, please contact the dev team.", err)) 59 | } 60 | }, 61 | } 62 | 63 | func init() { 64 | rootCmd.AddCommand(docCmd) 65 | docCmd.PersistentFlags().StringVar(&docCmdInput.outputLocation, "output-location", "./doc", 66 | "where to put the generated markdown files") 67 | } 68 | -------------------------------------------------------------------------------- /testSuite/scripts/test_azcopy_operations.py: -------------------------------------------------------------------------------- 1 | import utility as util 2 | import json 3 | import unittest 4 | from collections import namedtuple 5 | import sys 6 | import os 7 | 8 | class Azcopy_Operation_User_Scenario(unittest.TestCase): 9 | def setUp(self): 10 | cmd = util.Command("login").add_arguments("--service-principal").add_flags("application-id", os.environ['ACTIVE_DIRECTORY_APPLICATION_ID']) 11 | cmd.execute_azcopy_copy_command() 12 | 13 | def tearDown(self): 14 | cmd = util.Command("logout") 15 | cmd.execute_azcopy_copy_command() 16 | 17 | 18 | # test_remove_virtual_directory creates a virtual directory, removes the virtual directory created 19 | # and then verifies the contents of virtual directory. 20 | def test_remove_virtual_directory(self): 21 | # create dir dir_10_files and 1 kb files inside the dir. 22 | dir_name = "dir_" + str(10) + "_files_rm" 23 | dir_n_files_path = util.create_test_n_files(1024, 10, dir_name) 24 | 25 | # execute azcopy command 26 | result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ 27 | add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command() 28 | self.assertTrue(result) 29 | 30 | destination = util.get_resource_sas(dir_name) 31 | result = util.Command("rm").add_arguments(destination).add_flags("recursive", "true").execute_azcopy_copy_command() 32 | self.assertTrue(result) 33 | 34 | result = util.Command("list").add_arguments(destination).add_flags("resource-num", "0").execute_azcopy_verify() 35 | self.assertTrue(result) 36 | 37 | def test_remove_virtual_directory_oauth(self): 38 | # create dir dir_10_files and 1 kb files inside the dir. 39 | dir_name = "dir_" + str(10) + "_files_rm_oauth" 40 | dir_n_files_path = util.create_test_n_files(1024, 10, dir_name) 41 | 42 | # execute azcopy command 43 | result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_oauth_container_url). \ 44 | add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command() 45 | self.assertTrue(result) 46 | 47 | destination = util.get_object_without_sas(util.test_oauth_container_url, dir_name) 48 | result = util.Command("rm").add_arguments(destination).add_flags("recursive", "true").execute_azcopy_copy_command() 49 | self.assertTrue(result) 50 | 51 | destination_with_sas = util.get_object_sas(util.test_oauth_container_validate_sas_url, dir_name) 52 | result = util.Command("list").add_arguments(destination_with_sas).add_flags("resource-num", "0").execute_azcopy_verify() 53 | self.assertTrue(result) 54 | -------------------------------------------------------------------------------- /cmd/bytesizetostring_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | chk "gopkg.in/check.v1" 5 | ) 6 | 7 | type byteSizeToStringTestSuite struct{} 8 | 9 | var _ = chk.Suite(&byteSizeToStringTestSuite{}) 10 | 11 | func (s *byteSizeToStringTestSuite) TestBToString(c *chk.C) { 12 | inputs := []int64{50, 100, 125} 13 | expects := []string{"50.00 B", "100.00 B", "125.00 B"} 14 | 15 | for k, v := range inputs { 16 | output := byteSizeToString(v) 17 | c.Assert(output, chk.Equals, expects[k]) 18 | } 19 | } 20 | 21 | func (s *byteSizeToStringTestSuite) TestKiBToString(c *chk.C) { 22 | inputs := []int64{1024, 51200, 128000, 5632, 5376} 23 | expects := []string{"1.00 KiB", "50.00 KiB", "125.00 KiB", "5.50 KiB", "5.25 KiB"} 24 | 25 | for k, v := range inputs { 26 | output := byteSizeToString(v) 27 | c.Assert(output, chk.Equals, expects[k]) 28 | } 29 | } 30 | 31 | func (s *byteSizeToStringTestSuite) TestMiBToString(c *chk.C) { 32 | inputs := []int64{1048576, 52428800, 131072000, 5767168, 5505024} 33 | expects := []string{"1.00 MiB", "50.00 MiB", "125.00 MiB", "5.50 MiB", "5.25 MiB"} 34 | 35 | for k, v := range inputs { 36 | output := byteSizeToString(v) 37 | c.Assert(output, chk.Equals, expects[k]) 38 | } 39 | } 40 | 41 | func (s *byteSizeToStringTestSuite) TestGiBToString(c *chk.C) { 42 | inputs := []int64{1073741824, 53687091200, 134217728000, 5905580032, 5637144576} 43 | expects := []string{"1.00 GiB", "50.00 GiB", "125.00 GiB", "5.50 GiB", "5.25 GiB"} 44 | 45 | for k, v := range inputs { 46 | output := byteSizeToString(v) 47 | c.Assert(output, chk.Equals, expects[k]) 48 | } 49 | } 50 | 51 | func (s *byteSizeToStringTestSuite) TestTiBToString(c *chk.C) { 52 | inputs := []int64{1099511627776, 54975581388800, 137438953472000, 6047313952768, 5772436045824} 53 | expects := []string{"1.00 TiB", "50.00 TiB", "125.00 TiB", "5.50 TiB", "5.25 TiB"} 54 | 55 | for k, v := range inputs { 56 | output := byteSizeToString(v) 57 | c.Assert(output, chk.Equals, expects[k]) 58 | } 59 | } 60 | 61 | func (s *byteSizeToStringTestSuite) TestPiBToString(c *chk.C) { 62 | inputs := []int64{1125899906842624, 56294995342131200, 140737488355328000, 6192449487634432, 5910974510923776} 63 | expects := []string{"1.00 PiB", "50.00 PiB", "125.00 PiB", "5.50 PiB", "5.25 PiB"} 64 | 65 | for k, v := range inputs { 66 | output := byteSizeToString(v) 67 | c.Assert(output, chk.Equals, expects[k]) 68 | } 69 | } 70 | 71 | func (s *byteSizeToStringTestSuite) TestEiBToString(c *chk.C) { 72 | inputs := []int64{1152921504606846976, 6341068275337658368, 6052837899185946624} 73 | expects := []string{"1.00 EiB", "5.50 EiB", "5.25 EiB"} //50 & 125 aren't present Because they overflow int64 74 | 75 | for k, v := range inputs { 76 | output := byteSizeToString(v) 77 | c.Assert(output, chk.Equals, expects[k]) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /cmd/zt_pathUtils_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "github.com/Azure/azure-storage-azcopy/v10/common" 25 | chk "gopkg.in/check.v1" 26 | ) 27 | 28 | type pathUtilsSuite struct{} 29 | 30 | var _ = chk.Suite(&pathUtilsSuite{}) 31 | 32 | func (s *pathUtilsSuite) TestStripQueryFromSaslessUrl(c *chk.C) { 33 | tests := []struct { 34 | full string 35 | isRemote bool 36 | expectedMain string 37 | expectedQuery string 38 | }{ 39 | // remote urls 40 | {"http://example.com/abc?foo=bar", true, "http://example.com/abc", "foo=bar"}, 41 | {"http://example.com/abc", true, "http://example.com/abc", ""}, 42 | {"http://example.com/abc?", true, "http://example.com/abc", ""}, // no query string if ? is at very end 43 | 44 | // things that are not URLs, or not to be interpreted as such 45 | {"http://foo/bar?eee", false, "http://foo/bar?eee", ""}, // note isRemote == false 46 | {`c:\notUrl`, false, `c:\notUrl`, ""}, 47 | {`\\?\D:\longStyle\Windows\path`, false, `\\?\D:\longStyle\Windows\path`, ""}, 48 | } 49 | 50 | for _, t := range tests { 51 | loc := common.ELocation.Local() 52 | if t.isRemote { 53 | loc = common.ELocation.File() 54 | } 55 | m, q := splitQueryFromSaslessResource(t.full, loc) 56 | c.Assert(m, chk.Equals, t.expectedMain) 57 | c.Assert(q, chk.Equals, t.expectedQuery) 58 | } 59 | } 60 | 61 | func (s *pathUtilsSuite) TestToReversedString(c *chk.C) { 62 | t := &benchmarkTraverser{} 63 | c.Assert("1", chk.Equals, t.toReversedString(1)) 64 | c.Assert("01", chk.Equals, t.toReversedString(10)) 65 | c.Assert("54321", chk.Equals, t.toReversedString(12345)) 66 | } 67 | -------------------------------------------------------------------------------- /cmd/removeProcessor.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "github.com/Azure/azure-storage-azcopy/v10/common" 25 | ) 26 | 27 | // extract the right info from cooked arguments and instantiate a generic copy transfer processor from it 28 | func newRemoveTransferProcessor(cca *CookedCopyCmdArgs, numOfTransfersPerPart int, fpo common.FolderPropertyOption) *copyTransferProcessor { 29 | copyJobTemplate := &common.CopyJobPartOrderRequest{ 30 | JobID: cca.jobID, 31 | CommandString: cca.commandString, 32 | FromTo: cca.FromTo, 33 | Fpo: fpo, 34 | SourceRoot: cca.Source.CloneWithConsolidatedSeparators(), // TODO: why do we consolidate here, but not in "copy"? Is it needed in both places or neither? Or is copy just covering the same need differently? 35 | CredentialInfo: cca.credentialInfo, 36 | ForceIfReadOnly: cca.ForceIfReadOnly, 37 | 38 | // flags 39 | LogLevel: cca.LogVerbosity, 40 | BlobAttributes: common.BlobTransferAttributes{DeleteSnapshotsOption: cca.deleteSnapshotsOption, PermanentDeleteOption: cca.permanentDeleteOption}, 41 | } 42 | 43 | reportFirstPart := func(jobStarted bool) { 44 | if jobStarted { 45 | cca.waitUntilJobCompletion(false) 46 | } 47 | } 48 | reportFinalPart := func() { cca.isEnumerationComplete = true } 49 | 50 | // note that the source and destination, along with the template are given to the generic processor's constructor 51 | // this means that given an object with a relative path, this processor already knows how to schedule the right kind of transfers 52 | return newCopyTransferProcessor(copyJobTemplate, numOfTransfersPerPart, cca.Source, cca.Destination, 53 | reportFirstPart, reportFinalPart, false, cca.dryrunMode) 54 | } 55 | -------------------------------------------------------------------------------- /testSuite/scripts/test_blobfs_upload_SAS.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import utility as util 4 | 5 | class BlobFs_Upload_SAS_User_Scenarios(unittest.TestCase): 6 | def setUp(self): 7 | cmd = util.Command("login").add_arguments("--service-principal").add_flags("application-id", os.environ['ACTIVE_DIRECTORY_APPLICATION_ID']) 8 | cmd.execute_azcopy_copy_command() 9 | self.cachedAzCopyAccountKey = os.environ['ACCOUNT_KEY'] 10 | os.environ['ACCOUNT_KEY'] = '' 11 | 12 | def tearDown(self): 13 | cmd = util.Command("logout") 14 | cmd.execute_azcopy_copy_command() 15 | os.environ['ACCOUNT_KEY'] = self.cachedAzCopyAccountKey 16 | 17 | def test_blobfs_sas_upload_1Kb_file(self): 18 | # Create file of size 1KB 19 | filename = "test_blobfs_sas_u_1kb_file.txt" 20 | file_path = util.create_test_file(filename, 1024) 21 | # Upload the file using azCopy 22 | result = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_sas_account_url). \ 23 | add_flags("log-level", "Info").execute_azcopy_copy_command() 24 | self.assertTrue(result) 25 | 26 | # Validate the uploaded file 27 | file_url = util.get_resource_sas_from_bfs(filename) 28 | result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(file_url).execute_azcopy_verify() 29 | self.assertTrue(result) 30 | 31 | def test_blobfs_sas_upload_64MB_file(self): 32 | # Create file of size 1KB 33 | filename = "test_blobfs_sas_u_64MB_file.txt" 34 | file_path = util.create_test_file(filename, 64 * 1024 * 1024) 35 | # Upload the file using azCopy 36 | result = util.Command("copy").add_arguments(file_path).add_arguments(util.test_bfs_sas_account_url). \ 37 | add_flags("log-level", "Info").execute_azcopy_copy_command() 38 | self.assertTrue(result) 39 | 40 | # Validate the uploaded file 41 | file_url = util.get_resource_sas_from_bfs(filename) 42 | result = util.Command("testBlobFS").add_arguments(file_path).add_arguments(file_url).execute_azcopy_verify() 43 | self.assertTrue(result) 44 | 45 | def test_blobfs_sas_upload_100_1kb_file(self): 46 | # Create dir with 100 1kb files inside it 47 | dir_name = "dir_blobfs_sas_u_100_1K" 48 | dir_n_file_path = util.create_test_n_files(1024, 100, dir_name) 49 | 50 | # Upload the directory with 100 files inside it 51 | result = util.Command("copy").add_arguments(dir_n_file_path).add_arguments(util.test_bfs_sas_account_url). \ 52 | add_flags("log-level", "Info").add_flags("recursive", "true").execute_azcopy_copy_command() 53 | self.assertTrue(result) 54 | 55 | # Validate the uploaded directory 56 | dirUrl = util.get_resource_sas_from_bfs(dir_name) 57 | result = util.Command("testBlobFS").add_arguments(dir_n_file_path).add_arguments(dirUrl). \ 58 | add_flags("is-object-dir", "true").execute_azcopy_verify() 59 | self.assertTrue(result) 60 | -------------------------------------------------------------------------------- /common/zt_exclusiveStringMap_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package common 22 | 23 | import ( 24 | chk "gopkg.in/check.v1" 25 | ) 26 | 27 | type exclusiveStringMapSuite struct{} 28 | 29 | var _ = chk.Suite(&exclusiveStringMapSuite{}) 30 | 31 | func (s *exclusiveStringMapSuite) TestExclusiveStringMap(c *chk.C) { 32 | var m *ExclusiveStringMap 33 | 34 | addShouldWork := func(v string) { 35 | err := m.Add(v) 36 | c.Assert(err, chk.IsNil) 37 | } 38 | 39 | addShouldErrorOut := func(v string) { 40 | err := m.Add(v) 41 | c.Assert(err, chk.Equals, exclusiveStringMapCollisionError) 42 | } 43 | 44 | // case sensitive 45 | m = NewExclusiveStringMap(EFromTo.BlobLocal(), "linux") 46 | addShouldWork("cat") 47 | addShouldWork("dog") 48 | addShouldWork("doG") 49 | addShouldErrorOut("dog") // collision 50 | m.Remove("dog") // remove and try again 51 | addShouldWork("dog") 52 | 53 | // case insensitive 54 | m = NewExclusiveStringMap(EFromTo.BlobLocal(), "windows") 55 | addShouldWork("cat") 56 | addShouldWork("dog") 57 | addShouldErrorOut("doG") // collision 58 | m.Remove("dog") // remove and try again 59 | addShouldWork("doG") 60 | 61 | } 62 | 63 | func (s *exclusiveStringMapSuite) TestChooseRightCaseSensitivity(c *chk.C) { 64 | test := func(fromTo FromTo, goos string, shouldBeSensitive bool) { 65 | m := NewExclusiveStringMap(fromTo, goos) 66 | c.Assert(m.caseSensitive, chk.Equals, shouldBeSensitive) 67 | } 68 | 69 | test(EFromTo.BlobLocal(), "linux", true) 70 | test(EFromTo.BlobLocal(), "windows", false) 71 | test(EFromTo.BlobLocal(), "darwin", false) // default MacOS behaviour is case INsensitive, so assume we are running under that default 72 | 73 | test(EFromTo.LocalFile(), "linux", false) // anything ToFile should be INsensitive 74 | test(EFromTo.BlobFile(), "linux", false) // anything ToFile should be INsensitive 75 | test(EFromTo.BlobBlob(), "windows", true) 76 | } 77 | -------------------------------------------------------------------------------- /azbfs/zt_url_filesystem_test.go: -------------------------------------------------------------------------------- 1 | package azbfs_test 2 | 3 | import ( 4 | "context" 5 | "os" 6 | 7 | "net/http" 8 | "net/url" 9 | 10 | "github.com/Azure/azure-storage-azcopy/v10/azbfs" 11 | chk "gopkg.in/check.v1" 12 | ) 13 | 14 | type FileSystemURLSuite struct{} 15 | 16 | var _ = chk.Suite(&FileSystemURLSuite{}) 17 | 18 | func delFileSystem(c *chk.C, fs azbfs.FileSystemURL) { 19 | resp, err := fs.Delete(context.Background()) 20 | c.Assert(err, chk.IsNil) 21 | c.Assert(resp.Response().StatusCode, chk.Equals, http.StatusAccepted) 22 | } 23 | 24 | func (s *FileSystemURLSuite) TestFileSystemCreateRootDirectoryURL(c *chk.C) { 25 | fsu := getBfsServiceURL() 26 | testURL := fsu.NewFileSystemURL(fileSystemPrefix).NewRootDirectoryURL() 27 | 28 | correctURL := "https://" + os.Getenv("ACCOUNT_NAME") + ".dfs.core.windows.net/" + fileSystemPrefix 29 | temp := testURL.URL() 30 | c.Assert(temp.String(), chk.Equals, correctURL) 31 | } 32 | 33 | func (s *FileSystemURLSuite) TestFileSystemCreateDirectoryURL(c *chk.C) { 34 | fsu := getBfsServiceURL() 35 | testURL := fsu.NewFileSystemURL(fileSystemPrefix).NewDirectoryURL(directoryPrefix) 36 | 37 | correctURL := "https://" + os.Getenv("ACCOUNT_NAME") + ".dfs.core.windows.net/" + fileSystemPrefix + "/" + directoryPrefix 38 | temp := testURL.URL() 39 | c.Assert(temp.String(), chk.Equals, correctURL) 40 | c.Assert(testURL.String(), chk.Equals, correctURL) 41 | } 42 | 43 | func (s *FileSystemURLSuite) TestFileSystemNewFileSystemURLNegative(c *chk.C) { 44 | c.Assert(func() { azbfs.NewFileSystemURL(url.URL{}, nil) }, chk.Panics, "p can't be nil") 45 | } 46 | 47 | func (s *FileSystemURLSuite) TestFileSystemCreateDelete(c *chk.C) { 48 | fsu := getBfsServiceURL() 49 | fileSystemURL, _ := getFileSystemURL(c, fsu) 50 | 51 | _, err := fileSystemURL.Create(ctx) 52 | defer delFileSystem(c, fileSystemURL) 53 | c.Assert(err, chk.IsNil) 54 | 55 | // Test get properties 56 | resp, err := fileSystemURL.GetProperties(ctx) 57 | c.Assert(resp.StatusCode(), chk.Equals, http.StatusOK) 58 | c.Assert(err, chk.IsNil) 59 | } 60 | 61 | func (s *FileSystemURLSuite) TestFileSystemList(c *chk.C) { 62 | fsu := getBfsServiceURL() 63 | fileSystemURL, _ := getFileSystemURL(c, fsu) 64 | 65 | _, err := fileSystemURL.Create(ctx) 66 | defer delFileSystem(c, fileSystemURL) 67 | c.Assert(err, chk.IsNil) 68 | 69 | // List Setup 70 | dirUrl, dirName := getDirectoryURLFromFileSystem(c, fileSystemURL) 71 | dirUrl.Create(context.Background(), true) 72 | 73 | fileUrl, fileName := getFileURLFromFileSystem(c, fileSystemURL) 74 | fileUrl.Create(context.Background(), azbfs.BlobFSHTTPHeaders{}, azbfs.BlobFSAccessControl{}) 75 | 76 | // List 77 | paths, err := fileSystemURL.ListPaths(context.Background(), azbfs.ListPathsFilesystemOptions{Recursive: false}) 78 | c.Assert(err, chk.IsNil) 79 | c.Assert(paths.Paths, chk.NotNil) 80 | c.Assert(len(paths.Paths), chk.Equals, 2) 81 | dirPath := paths.Paths[0] 82 | c.Assert(*dirPath.Name, chk.Equals, dirName) 83 | c.Assert(*dirPath.IsDirectory, chk.Equals, true) 84 | filePath := paths.Paths[1] 85 | c.Assert(*filePath.Name, chk.Equals, fileName) 86 | c.Assert(filePath.IsDirectory, chk.IsNil) 87 | } 88 | -------------------------------------------------------------------------------- /azbfs/zc_storage_error.go: -------------------------------------------------------------------------------- 1 | package azbfs 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "net/http" 7 | "sort" 8 | 9 | "github.com/Azure/azure-pipeline-go/pipeline" 10 | ) 11 | 12 | func init() { 13 | // wire up our custom error handling constructor 14 | responseErrorFactory = newStorageError 15 | } 16 | 17 | // ServiceCodeType is a string identifying a storage service error. 18 | // For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/status-and-error-codes2 19 | type ServiceCodeType string 20 | 21 | // StorageError identifies a responder-generated network or response parsing error. 22 | type StorageError interface { 23 | // ResponseError implements error's Error(), net.Error's Temporary() and Timeout() methods & Response(). 24 | ResponseError 25 | 26 | // ServiceCode returns a service error code. Your code can use this to make error recovery decisions. 27 | ServiceCode() ServiceCodeType 28 | } 29 | 30 | // storageError is the internal struct that implements the public StorageError interface. 31 | type storageError struct { 32 | responseError 33 | serviceCode ServiceCodeType 34 | details map[string]string 35 | } 36 | 37 | // newStorageError creates an error object that implements the error interface. 38 | func newStorageError(cause error, response *http.Response, description string) error { 39 | return &storageError{ 40 | responseError: responseError{ 41 | ErrorNode: pipeline.ErrorNode{}.Initialize(cause, 3), 42 | response: response, 43 | description: description, 44 | }, 45 | serviceCode: ServiceCodeType(response.Header.Get("X-Ms-Error-Code")), 46 | } 47 | } 48 | 49 | // ServiceCode returns service-error information. The caller may examine these values but should not modify any of them. 50 | func (e *storageError) ServiceCode() ServiceCodeType { return e.serviceCode } 51 | 52 | // Error implements the error interface's Error method to return a string representation of the error. 53 | func (e *storageError) Error() string { 54 | b := &bytes.Buffer{} 55 | fmt.Fprintf(b, "===== RESPONSE ERROR (ServiceCode=%s) =====\n", e.serviceCode) 56 | fmt.Fprintf(b, "Description=%s, Details: ", e.description) 57 | if len(e.details) == 0 { 58 | b.WriteString("(none)\n") 59 | } else { 60 | b.WriteRune('\n') 61 | keys := make([]string, 0, len(e.details)) 62 | // Alphabetize the details 63 | for k := range e.details { 64 | keys = append(keys, k) 65 | } 66 | sort.Strings(keys) 67 | for _, k := range keys { 68 | fmt.Fprintf(b, " %s: %+v\n", k, e.details[k]) 69 | } 70 | } 71 | req := pipeline.Request{Request: e.response.Request}.Copy() // Make a copy of the response's request 72 | pipeline.WriteRequestWithResponse(b, prepareRequestForLogging(req), e.response, nil) 73 | return e.ErrorNode.Error(b.String()) 74 | } 75 | 76 | // Temporary returns true if the error occurred due to a temporary condition (including an HTTP status of 500 or 503). 77 | func (e *storageError) Temporary() bool { 78 | if e.response != nil { 79 | if (e.response.StatusCode == http.StatusInternalServerError) || (e.response.StatusCode == http.StatusServiceUnavailable) { 80 | return true 81 | } 82 | } 83 | return e.ErrorNode.Temporary() 84 | } 85 | -------------------------------------------------------------------------------- /main_windows.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package main 22 | 23 | import ( 24 | "math" 25 | "net/http" 26 | "os" 27 | "os/exec" 28 | "path" 29 | "strings" 30 | "syscall" 31 | 32 | "github.com/minio/minio-go" 33 | 34 | "github.com/Azure/azure-storage-azcopy/v10/common" 35 | ) 36 | 37 | func osModifyProcessCommand(cmd *exec.Cmd) *exec.Cmd { 38 | // On Windows, create the child process in new process group to avoid receiving signals 39 | // (Ctrl+C, Ctrl+Break) from the console 40 | cmd.SysProcAttr = &syscall.SysProcAttr{ 41 | CreationFlags: syscall.CREATE_NEW_PROCESS_GROUP, 42 | } 43 | return cmd 44 | } 45 | 46 | // ProcessOSSpecificInitialization chnages the soft limit for filedescriptor for process 47 | // return the filedescriptor limit for process. If the function fails with some, it returns 48 | // the error 49 | // TODO: this api is implemented for windows as well but not required because Windows 50 | // does not default to a precise low limit like Linux does 51 | func ProcessOSSpecificInitialization() (int, error) { 52 | 53 | // this exaggerates what's possible, but is accurate enough for our purposes, in which our goal is simply to apply no specific limit on Windows 54 | const effectivelyUnlimited = math.MaxInt32 55 | 56 | return effectivelyUnlimited, nil 57 | } 58 | 59 | // GetAzCopyAppPath returns the path of Azcopy in local appdata. 60 | func GetAzCopyAppPath() string { 61 | lcm := common.GetLifecycleMgr() 62 | userProfile := lcm.GetEnvironmentVariable(common.EEnvironmentVariable.UserDir()) 63 | azcopyAppDataFolder := strings.ReplaceAll(path.Join(userProfile, ".azcopy"), "/", `\`) 64 | if err := os.Mkdir(azcopyAppDataFolder, os.ModeDir); err != nil && !os.IsExist(err) { 65 | return "" 66 | } 67 | return azcopyAppDataFolder 68 | } 69 | 70 | func init() { 71 | //Catch everything that uses http.DefaultTransport with ieproxy.GetProxyFunc() 72 | http.DefaultTransport.(*http.Transport).Proxy = common.GlobalProxyLookup 73 | minio.DefaultTransport.(*http.Transport).Proxy = common.GlobalProxyLookup 74 | } 75 | -------------------------------------------------------------------------------- /ste/sourceInfoProvider-Local.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package ste 22 | 23 | import ( 24 | "os" 25 | "time" 26 | 27 | "github.com/Azure/azure-storage-azcopy/v10/common" 28 | ) 29 | 30 | // Source info provider for local files 31 | type localFileSourceInfoProvider struct { 32 | jptm IJobPartTransferMgr 33 | transferInfo TransferInfo 34 | } 35 | 36 | func newLocalSourceInfoProvider(jptm IJobPartTransferMgr) (ISourceInfoProvider, error) { 37 | return &localFileSourceInfoProvider{jptm, jptm.Info()}, nil 38 | } 39 | 40 | func (f localFileSourceInfoProvider) Properties() (*SrcProperties, error) { 41 | // create simulated headers, to represent what we want to propagate to the destination based on 42 | // this file 43 | 44 | headers, metadata, blobTags, _ := f.jptm.ResourceDstData(nil) // we don't have a known MIME type yet, so pass nil for the sniffed content of thefile 45 | 46 | return &SrcProperties{ 47 | SrcHTTPHeaders: common.ResourceHTTPHeaders{ 48 | ContentType: headers.ContentType, 49 | ContentEncoding: headers.ContentEncoding, 50 | ContentLanguage: headers.ContentLanguage, 51 | ContentDisposition: headers.ContentDisposition, 52 | CacheControl: headers.CacheControl, 53 | }, 54 | SrcMetadata: metadata, 55 | SrcBlobTags: blobTags, 56 | }, nil 57 | } 58 | 59 | func (f localFileSourceInfoProvider) IsLocal() bool { 60 | return true 61 | } 62 | 63 | func (f localFileSourceInfoProvider) OpenSourceFile() (common.CloseableReaderAt, error) { 64 | path := f.jptm.Info().Source 65 | 66 | if custom, ok := interface{}(f).(ICustomLocalOpener); ok { 67 | return custom.Open(path) 68 | } 69 | return os.Open(path) 70 | } 71 | 72 | func (f localFileSourceInfoProvider) GetFreshFileLastModifiedTime() (time.Time, error) { 73 | i, err := common.OSStat(f.jptm.Info().Source) 74 | if err != nil { 75 | return time.Time{}, err 76 | } 77 | return i.ModTime(), nil 78 | } 79 | 80 | func (f localFileSourceInfoProvider) EntityType() common.EntityType { 81 | return f.transferInfo.EntityType 82 | } 83 | -------------------------------------------------------------------------------- /common/s3Models.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package common 22 | 23 | import ( 24 | "encoding/base64" 25 | "strings" 26 | 27 | minio "github.com/minio/minio-go" 28 | ) 29 | 30 | type ObjectInfoExtension struct { 31 | ObjectInfo minio.ObjectInfo 32 | } 33 | 34 | func (oie *ObjectInfoExtension) ContentType() string { 35 | return oie.ObjectInfo.ContentType 36 | } 37 | 38 | // CacheControl returns the value for header Cache-Control. 39 | func (oie *ObjectInfoExtension) CacheControl() string { 40 | return oie.ObjectInfo.Metadata.Get("Cache-Control") 41 | } 42 | 43 | // ContentDisposition returns the value for header Content-Disposition. 44 | func (oie *ObjectInfoExtension) ContentDisposition() string { 45 | return oie.ObjectInfo.Metadata.Get("Content-Disposition") 46 | } 47 | 48 | // ContentEncoding returns the value for header Content-Encoding. 49 | func (oie *ObjectInfoExtension) ContentEncoding() string { 50 | return oie.ObjectInfo.Metadata.Get("Content-Encoding") 51 | } 52 | 53 | // ContentLanguage returns the value for header Content-Language. 54 | func (oie *ObjectInfoExtension) ContentLanguage() string { 55 | return oie.ObjectInfo.Metadata.Get("Content-Language") 56 | } 57 | 58 | // ContentMD5 returns the value for header Content-MD5. 59 | func (oie *ObjectInfoExtension) ContentMD5() []byte { 60 | s := oie.ObjectInfo.Metadata.Get("Content-MD5") 61 | if s == "" { 62 | return nil 63 | } 64 | b, err := base64.StdEncoding.DecodeString(s) 65 | if err != nil { 66 | b = nil 67 | } 68 | return b 69 | } 70 | 71 | const s3MetadataPrefix = "x-amz-meta-" 72 | 73 | const s3MetadataPrefixLen = len(s3MetadataPrefix) 74 | 75 | // NewMetadata returns user-defined key/value pairs. 76 | func (oie *ObjectInfoExtension) NewCommonMetadata() Metadata { 77 | md := Metadata{} 78 | for k, v := range oie.ObjectInfo.Metadata { 79 | if len(k) > s3MetadataPrefixLen { 80 | if prefix := k[0:s3MetadataPrefixLen]; strings.EqualFold(prefix, s3MetadataPrefix) { 81 | md[k[s3MetadataPrefixLen:]] = v[0] 82 | } 83 | } 84 | } 85 | return md 86 | } 87 | -------------------------------------------------------------------------------- /cmd/loginStatus.go: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "context" 25 | "fmt" 26 | "github.com/Azure/azure-storage-azcopy/v10/common" 27 | "github.com/Azure/azure-storage-azcopy/v10/ste" 28 | "github.com/spf13/cobra" 29 | ) 30 | 31 | func init() { 32 | type loginStatus struct { 33 | tenantID bool 34 | endpoint bool 35 | } 36 | commandLineInput := loginStatus{} 37 | 38 | lgStatus := &cobra.Command{ 39 | Use: "status", 40 | Short: loginStatusShortDescription, 41 | Long: loginStatusLongDescription, 42 | Args: func(cmd *cobra.Command, args []string) error { 43 | if len(args) > 0 { 44 | return fmt.Errorf("login status does not require any argument") 45 | } 46 | return nil 47 | }, 48 | Run: func(cmd *cobra.Command, args []string) { 49 | // getting current token info and refreshing it with GetTokenInfo() 50 | ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) 51 | uotm := GetUserOAuthTokenManagerInstance() 52 | tokenInfo, err := uotm.GetTokenInfo(ctx) 53 | 54 | if err == nil && !tokenInfo.IsExpired() { 55 | glcm.Info("You have successfully refreshed your token. Your login session is still active") 56 | 57 | if commandLineInput.tenantID { 58 | glcm.Info(fmt.Sprintf("Tenant ID: %v", tokenInfo.Tenant)) 59 | } 60 | 61 | if commandLineInput.endpoint { 62 | glcm.Info(fmt.Sprintf("Active directory endpoint: %v", tokenInfo.ActiveDirectoryEndpoint)) 63 | } 64 | 65 | glcm.Exit(nil, common.EExitCode.Success()) 66 | } 67 | 68 | glcm.Info("You are currently not logged in. Please login using 'azcopy login'") 69 | glcm.Exit(nil, common.EExitCode.Error()) 70 | }, 71 | } 72 | 73 | lgCmd.AddCommand(lgStatus) 74 | lgStatus.PersistentFlags().BoolVar(&commandLineInput.tenantID, "tenant", false, "Prints the Azure Active Directory tenant ID that is currently being used in session.") 75 | lgStatus.PersistentFlags().BoolVar(&commandLineInput.endpoint, "endpoint", false, "Prints the Azure Active Directory endpoint that is being used in the current session.") 76 | } 77 | -------------------------------------------------------------------------------- /azbfs/url_service.go: -------------------------------------------------------------------------------- 1 | package azbfs 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "net/url" 7 | "path" 8 | 9 | "github.com/Azure/azure-pipeline-go/pipeline" 10 | ) 11 | 12 | // A ServiceURL represents a URL to the Azure Storage File service allowing you to manipulate file shares. 13 | type ServiceURL struct { 14 | client filesystemClient 15 | } 16 | 17 | // NewServiceURL creates a ServiceURL object using the specified URL and request policy pipeline. 18 | func NewServiceURL(url url.URL, p pipeline.Pipeline) ServiceURL { 19 | if p == nil { 20 | panic("p can't be nil") 21 | } 22 | client := filesystemClient{newManagementClient(url, p)} 23 | return ServiceURL{client: client} 24 | } 25 | 26 | func (s ServiceURL) ListFilesystemsSegment(ctx context.Context, marker *string) (*FilesystemList, error) { 27 | return s.client.List(ctx, nil, marker, nil, nil, nil, nil) 28 | } 29 | 30 | // URL returns the URL endpoint used by the ServiceURL object. 31 | func (s ServiceURL) URL() url.URL { 32 | return s.client.URL() 33 | } 34 | 35 | // String returns the URL as a string. 36 | func (s ServiceURL) String() string { 37 | u := s.URL() 38 | return u.String() 39 | } 40 | 41 | // WithPipeline creates a new ServiceURL object identical to the source but with the specified request policy pipeline. 42 | func (s ServiceURL) WithPipeline(p pipeline.Pipeline) ServiceURL { 43 | return NewServiceURL(s.URL(), p) 44 | } 45 | 46 | // NewFileSystemURL creates a new ShareURL object by concatenating shareName to the end of 47 | // ServiceURL's URL. The new ShareURL uses the same request policy pipeline as the ServiceURL. 48 | // To change the pipeline, create the ShareURL and then call its WithPipeline method passing in the 49 | // desired pipeline object. Or, call this package's NewFileSystemURL instead of calling this object's 50 | // NewFileSystemURL method. 51 | func (s ServiceURL) NewFileSystemURL(fileSystemName string) FileSystemURL { 52 | fileSystemURL := appendToURLPath(s.URL(), fileSystemName) 53 | return NewFileSystemURL(fileSystemURL, s.client.Pipeline()) 54 | } 55 | 56 | // appendToURLPath appends a string to the end of a URL's path (prefixing the string with a '/' if required) 57 | func appendToURLPath(u url.URL, name string) url.URL { 58 | // e.g. "https://ms.com/a/b/?k1=v1&k2=v2#f" 59 | // When you call url.Parse() this is what you'll get: 60 | // Scheme: "https" 61 | // Opaque: "" 62 | // User: nil 63 | // Host: "ms.com" 64 | // Path: "/a/b/" This should start with a / and it might or might not have a trailing slash 65 | // RawPath: "" 66 | // ForceQuery: false 67 | // RawQuery: "k1=v1&k2=v2" 68 | // Fragment: "f" 69 | if len(u.Path) == 0 || u.Path[len(u.Path)-1] != '/' { 70 | u.Path += "/" // Append "/" to end before appending name 71 | } 72 | u.Path += name 73 | return u 74 | } 75 | 76 | func removeLastSectionOfPath(u url.URL) (url.URL, error) { 77 | if len(u.Path) == 0 { 78 | return url.URL{}, errors.New("cannot remove from path because it is empty") 79 | } 80 | 81 | trimmedPath := path.Dir(u.Path) 82 | if trimmedPath == "." { 83 | trimmedPath = "" // should never happen, given what we pass in, but just in case, let's not return the file-system-ish dot 84 | } 85 | 86 | u.Path = trimmedPath 87 | return u, nil 88 | } 89 | -------------------------------------------------------------------------------- /cmd/versionChecker_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | chk "gopkg.in/check.v1" 25 | ) 26 | 27 | type versionCheckerTestSuite struct{} 28 | 29 | var _ = chk.Suite(&versionCheckerTestSuite{}) 30 | 31 | func (s *versionCheckerTestSuite) TestVersionEquality(c *chk.C) { 32 | // simple equal 33 | v1, _ := NewVersion("10.0.0") 34 | v2, _ := NewVersion("10.0.0") 35 | c.Assert(v1.compare(*v2), chk.Equals, 0) 36 | 37 | // preview version equal 38 | v1, _ = NewVersion("10.0.0-preview") 39 | v2, _ = NewVersion("10.0.0-preview") 40 | c.Assert(v1.compare(*v2), chk.Equals, 0) 41 | 42 | // future version equal 43 | v1, _ = NewVersion("10.0.0-preview") 44 | v2, _ = NewVersion("10.0.0-beta5") 45 | c.Assert(v1.compare(*v2), chk.Equals, 0) 46 | } 47 | 48 | func (s *versionCheckerTestSuite) TestVersionSuperiority(c *chk.C) { 49 | // major version bigger 50 | v1, _ := NewVersion("11.3.0") 51 | v2, _ := NewVersion("10.8.3") 52 | c.Assert(v1.compare(*v2), chk.Equals, 1) 53 | 54 | // minor version bigger 55 | v1, _ = NewVersion("15.5.6") 56 | v2, _ = NewVersion("15.3.5") 57 | c.Assert(v1.compare(*v2), chk.Equals, 1) 58 | 59 | // patch version bigger 60 | v1, _ = NewVersion("15.5.6") 61 | v2, _ = NewVersion("15.5.5") 62 | c.Assert(v1.compare(*v2), chk.Equals, 1) 63 | 64 | // preview bigger 65 | v1, _ = NewVersion("15.5.5") 66 | v2, _ = NewVersion("15.5.5-preview") 67 | c.Assert(v1.compare(*v2), chk.Equals, 1) 68 | } 69 | 70 | func (s *versionCheckerTestSuite) TestVersionInferiority(c *chk.C) { 71 | // major version smaller 72 | v1, _ := NewVersion("10.5.6") 73 | v2, _ := NewVersion("11.8.3") 74 | c.Assert(v1.compare(*v2), chk.Equals, -1) 75 | 76 | // minor version smaller 77 | v1, _ = NewVersion("15.3.6") 78 | v2, _ = NewVersion("15.5.5") 79 | c.Assert(v1.compare(*v2), chk.Equals, -1) 80 | 81 | // patch version smaller 82 | v1, _ = NewVersion("15.5.5") 83 | v2, _ = NewVersion("15.5.6") 84 | c.Assert(v1.compare(*v2), chk.Equals, -1) 85 | 86 | // preview smaller 87 | v1, _ = NewVersion("15.5.5-preview") 88 | v2, _ = NewVersion("15.5.5") 89 | c.Assert(v1.compare(*v2), chk.Equals, -1) 90 | } 91 | -------------------------------------------------------------------------------- /cmd/pause.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package cmd 22 | 23 | import ( 24 | "errors" 25 | 26 | "github.com/Azure/azure-storage-azcopy/v10/common" 27 | "github.com/spf13/cobra" 28 | ) 29 | 30 | // TODO should this command be removed? Previously AzCopy was supposed to have an independent backend (out of proc) 31 | // TODO but that's not the plan anymore 32 | func init() { 33 | var commandLineInput = "" 34 | 35 | // pauseCmd represents the pause command 36 | pauseCmd := &cobra.Command{ 37 | Use: "pause", 38 | SuggestFor: []string{"pase", "ause", "paue"}, 39 | Short: "Pause the existing job with the given Job Id", 40 | Long: `Pause the existing job with the given Job Id`, 41 | Args: func(cmd *cobra.Command, args []string) error { 42 | // the pause command requires necessarily to have an argument 43 | // pause jobId -- pause all the parts of an existing job for given jobId 44 | 45 | // If no argument is passed then it is not valid 46 | if len(args) != 1 { 47 | return errors.New("this command only requires jobId") 48 | } 49 | commandLineInput = (args[0]) 50 | return nil 51 | }, 52 | Run: func(cmd *cobra.Command, args []string) { 53 | HandlePauseCommand(commandLineInput) 54 | glcm.Exit(nil, common.EExitCode.Success()) 55 | }, 56 | // hide features not relevant to BFS 57 | // TODO remove after preview release 58 | Hidden: true, 59 | } 60 | rootCmd.AddCommand(pauseCmd) 61 | } 62 | 63 | // handles the pause command 64 | // dispatches the pause Job order to the storage engine 65 | func HandlePauseCommand(jobIdString string) { 66 | 67 | // parsing the given JobId to validate its format correctness 68 | jobID, err := common.ParseJobID(jobIdString) 69 | if err != nil { 70 | // If parsing gives an error, hence it is not a valid JobId format 71 | glcm.Error("invalid jobId string passed. Failed while parsing string to jobId") 72 | } 73 | 74 | var pauseJobResponse common.CancelPauseResumeResponse 75 | Rpc(common.ERpcCmd.PauseJob(), jobID, &pauseJobResponse) 76 | glcm.Exit(func(format common.OutputFormat) string { 77 | return "Job " + jobID.String() + " paused successfully" 78 | }, common.EExitCode.Success()) 79 | } 80 | -------------------------------------------------------------------------------- /ste/pacedReadSeeker.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package ste 22 | 23 | import ( 24 | "context" 25 | "io" 26 | ) 27 | 28 | // pacedReadSeeker implements read/seek/close with pacing. (Formerly in file pacer-lite) 29 | type pacedReadSeeker struct { 30 | 31 | // Although storing ctx in a struct is generally considered an anti-patten, this particular 32 | // struct happens to be fairly-short lived (from its users point of view). Basically just for 33 | // as long as a takes to read or send a request body. That's not terribly long, but it is 34 | // long enough that we might need to cancel during it, hence the ctx. 35 | ctx context.Context 36 | 37 | body io.Reader // Seeking is required to support retries 38 | p pacer 39 | } 40 | 41 | func newPacedRequestBody(ctx context.Context, requestBody io.ReadSeeker, p pacer) io.ReadSeeker { 42 | if p == nil { 43 | panic("p must not be nil") 44 | } 45 | return &pacedReadSeeker{ctx: ctx, body: requestBody, p: p} 46 | } 47 | 48 | func newPacedResponseBody(ctx context.Context, responseBody io.ReadCloser, p pacer) io.ReadCloser { 49 | if p == nil { 50 | panic("p must not be nil") 51 | } 52 | return &pacedReadSeeker{ctx: ctx, body: responseBody, p: p} 53 | } 54 | 55 | func (prs *pacedReadSeeker) Read(p []byte) (int, error) { 56 | requestedCount := len(p) 57 | 58 | // blocks until we are allowed to process the bytes 59 | err := prs.p.RequestTrafficAllocation(prs.ctx, int64(requestedCount)) 60 | if err != nil { 61 | return 0, err 62 | } 63 | 64 | // process them 65 | n, err := prs.body.Read(p) 66 | 67 | // "return" any unused tokens to the pacer (e.g. if we hit eof before the end of our buffer p) 68 | excess := requestedCount - n 69 | prs.p.UndoRequest(int64(excess)) 70 | 71 | return n, err 72 | } 73 | 74 | // Seeking is required to support retries 75 | func (prs *pacedReadSeeker) Seek(offset int64, whence int) (offsetFromStart int64, err error) { 76 | return prs.body.(io.ReadSeeker).Seek(offset, whence) 77 | } 78 | 79 | // pacedReadSeeker supports Close but the underlying stream may not; if it does, Close will close it. 80 | func (prs *pacedReadSeeker) Close() error { 81 | if c, ok := prs.body.(io.Closer); ok { 82 | return c.Close() 83 | } 84 | return nil 85 | } 86 | -------------------------------------------------------------------------------- /ste/downloader.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2017 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package ste 22 | 23 | import ( 24 | "github.com/Azure/azure-pipeline-go/pipeline" 25 | "github.com/Azure/azure-storage-azcopy/v10/common" 26 | ) 27 | 28 | // Abstraction of the methods needed to download files/blobs from a remote location 29 | type downloader interface { 30 | // Prologue does any necessary first-time setup 31 | Prologue(jptm IJobPartTransferMgr, srcPipeline pipeline.Pipeline) 32 | 33 | // GenerateDownloadFunc returns a func() that will download the specified portion of the remote file into dstFile 34 | // Instead of taking destination file as a parameter, it takes a helper that will write to the file. That keeps details of 35 | // file IO out out the download func, and lets that func concentrate only on the details of the remote endpoint 36 | GenerateDownloadFunc(jptm IJobPartTransferMgr, srcPipeline pipeline.Pipeline, writer common.ChunkedFileWriter, id common.ChunkID, length int64, pacer pacer) chunkFunc 37 | 38 | // Epilogue does cleanup. MAY be the only method that gets called (in error cases). So must not fail simply because 39 | // Prologue has not yet been called 40 | Epilogue() 41 | } 42 | 43 | // folderDownloader is a downloader that can also process folder properties 44 | type folderDownloader interface { 45 | downloader 46 | SetFolderProperties(jptm IJobPartTransferMgr) error 47 | } 48 | 49 | // smbPropertyAwareDownloader is a windows-triggered interface. 50 | // Code outside of windows-specific files shouldn't implement this ever. 51 | type smbPropertyAwareDownloader interface { 52 | PutSDDL(sip ISMBPropertyBearingSourceInfoProvider, txInfo TransferInfo) error 53 | 54 | PutSMBProperties(sip ISMBPropertyBearingSourceInfoProvider, txInfo TransferInfo) error 55 | } 56 | 57 | type downloaderFactory func() downloader 58 | 59 | func createDownloadChunkFunc(jptm IJobPartTransferMgr, id common.ChunkID, body func()) chunkFunc { 60 | // If uploading, we set the chunk status to done as soon as the chunkFunc completes. 61 | // But we don't do that for downloads, since for those the chunk is not "done" until its flushed out 62 | // by the ChunkedFileWriter. (The ChunkedFileWriter will set the status to done at that time.) 63 | return createChunkFunc(false, jptm, id, body) 64 | } 65 | -------------------------------------------------------------------------------- /e2etest/zt_error_handling_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package e2etest 22 | 23 | // Purpose: Tests for how we respond to errors. Maybe also resume? 24 | 25 | // TODO: include how we clean up destination files/blobs after errors 26 | 27 | // This test runs a transfer partially, then cancel the job, then resumes it 28 | /* 29 | * 30 | func TestError_CanResume(t *testing.T) { 31 | // TODO: as at 1 July 2020, this test doesn't pass, but if you run it, it will tell you why (all to do with missing parts of the framework) 32 | // It's here as one possible approach for testing resume. 33 | // TODO: the current test data volume might be a bit big 34 | RunScenarios( 35 | t, 36 | eOperation.CopyAndSync(), 37 | eTestFromTo.AllSourcesToOneDest(), // TODO: or should we make this AllPairs? 38 | eValidate.AutoPlusContent(), // important to validate file content after a resume // TODO: as at 1 July 2020, content validation isn't supported yet, so this test will fail 39 | params{ 40 | recursive: true, 41 | capMbps: 50, // at this speed the payload should take about 50 seconds to move 42 | cancelFromStdin: true, // needed to use hookHelper.CancelAndResume() 43 | }, 44 | &hooks{beforeRunJob: func(h hookHelper) { 45 | go func() { 46 | // wait a while, until we are probably somewhere the middle of the transfer, and then 47 | // kill AzCopy and resume it 48 | // Must do this in a separate GoRoutine, since we are (ab)using the beforeRunJob hook, and the job won't start 49 | // unit our hook func returns 50 | time.Sleep(20 * time.Second) 51 | h.CancelAndResume() 52 | }() 53 | }}, 54 | testFiles{ 55 | // Make a payload this is big enough to last for a minute or so, at our capped speed as per params above. 56 | // Make the files in it big enough to have multiple chunks, since that's the more interesting case for resume. 57 | defaultSize: "50M", 58 | shouldTransfer: []interface{}{ 59 | folder(""), 60 | "a", 61 | "b", 62 | folder("fold1"), 63 | "fold1/f", 64 | "fold1/g", 65 | folder("fold1/fold2"), 66 | "fold1/j", 67 | "fold1/k", 68 | }, 69 | }) 70 | } 71 | 72 | 73 | */ 74 | ///* Go fmt is messing things 75 | // "fold1/k", 76 | // }, 77 | // }) 78 | //} 79 | --------------------------------------------------------------------------------