├── .gitignore ├── .travis.yml ├── .vscode └── launch.json ├── README ├── azurecopy ├── azurecopy.go ├── handlers │ ├── AzureHandler.go │ ├── CloudHandlerInterface.go │ ├── DropboxHandler.go │ ├── FTPHandler.go │ ├── FilesystemHandler.go │ └── S3Handler.go ├── models │ ├── constants.go │ ├── simpleblob.go │ └── simplecontainer.go └── utils │ ├── blobutils │ └── blobutils.go │ ├── containerutils │ └── containerutils.go │ ├── handlerutils.go │ ├── helpers │ ├── azurehelper.go │ ├── dropboxhelper.go │ └── ftphelper.go │ └── misc │ ├── config.go │ └── pathutils.go └── azurecopycommand ├── .gitignore ├── .vscode └── launch.json ├── main.go └── res /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | *.dll 6 | *.cmd 7 | 8 | # Folders 9 | _obj 10 | _test 11 | 12 | # Architecture specific extensions/prefixes 13 | *.[568vq] 14 | [568vq].out 15 | 16 | *.cgo1.go 17 | *.cgo2.c 18 | _cgo_defun.c 19 | _cgo_gotypes.go 20 | _cgo_export.* 21 | 22 | _testmain.go 23 | 24 | notes 25 | *.exe 26 | *.test 27 | *.prof 28 | 29 | # Output of the go coverage tool, specifically when used with LiteIDE 30 | *.out 31 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.8.x 5 | - master 6 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "name": "Launch", 6 | "type": "go", 7 | "request": "launch", 8 | "mode": "debug", 9 | "remotePath": "", 10 | "port": 2345, 11 | "host": "127.0.0.1", 12 | "program": "${workspaceRoot}/azurecopycommand/", 13 | "env": {}, 14 | "args": [], 15 | "showLog": true 16 | } 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | AzureCopy (Go version) 2 | 3 | This is a reimplementation of the AzureCopy project (github.com/kpfaulkner/azurecopy) in the GO language. 4 | It is primarily to allow cross platform (Linux,*BSD, MacOS and Windows) versions of it. 5 | 6 | 7 | - Copy to/from Filesystem (done) 8 | - Copy to/from Azure Blob Storage (done) 9 | - Copy to/from S3 (done) 10 | - Copy to/from Dropbox (done) 11 | - Add CopyBlob flag for Azure destination (huge bandwidth savings) 12 | - Copy to/from Onedrive 13 | - Copy to/from Google Storage 14 | - Copy to/from Azure File Storage 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /azurecopy/azurecopy.go: -------------------------------------------------------------------------------- 1 | package azurecopy 2 | 3 | import ( 4 | "azurecopy/azurecopy/handlers" 5 | "azurecopy/azurecopy/models" 6 | "azurecopy/azurecopy/utils" 7 | "os" 8 | 9 | // "azurecopy/azurecopy/utils/helpers" 10 | "azurecopy/azurecopy/utils/misc" 11 | "fmt" 12 | "regexp" 13 | "strings" 14 | 15 | "sync" 16 | 17 | log "github.com/Sirupsen/logrus" 18 | ) 19 | 20 | var wg sync.WaitGroup 21 | 22 | // AzureCopy main client class. 23 | // Have one instance of this PER cloud env. 24 | // ie one for source and one for destination. 25 | type AzureCopy struct { 26 | config misc.CloudConfig 27 | 28 | // source/destination URLS 29 | sourceURL string 30 | destURL string 31 | 32 | // cloud types 33 | sourceCloudType models.CloudType 34 | destCloudType models.CloudType 35 | 36 | // handlers 37 | sourceHandler handlers.CloudHandlerInterface 38 | destHandler handlers.CloudHandlerInterface 39 | } 40 | 41 | // NewAzureCopy factory time! 42 | // want to know source/dest up front. 43 | func NewAzureCopy(config misc.CloudConfig) *AzureCopy { 44 | ac := AzureCopy{} 45 | ac.config = config 46 | 47 | // technically duped from config, but just easier to reference. 48 | ac.destURL = config.Configuration[misc.Dest] 49 | ac.sourceURL = config.Configuration[misc.Source] 50 | 51 | ac.sourceCloudType, _ = ac.getCloudType(ac.sourceURL) 52 | ac.destCloudType, _ = ac.getCloudType(ac.destURL) 53 | 54 | ac.sourceHandler = ac.GetHandlerForURL(ac.sourceURL, true, true) 55 | ac.destHandler = ac.GetHandlerForURL(ac.destURL, false, true) 56 | 57 | return &ac 58 | } 59 | 60 | // Get Cloud Type... 61 | // Should pre-compile all of these regexs 62 | func (ac *AzureCopy) getCloudType(url string) (cloudType models.CloudType, isEmulator bool) { 63 | lowerURL := strings.ToLower(url) 64 | 65 | // Azure 66 | match, _ := regexp.MatchString("blob.core.windows.net", lowerURL) 67 | if match { 68 | return models.Azure, false 69 | } 70 | 71 | // Dropbox 72 | match, _ = regexp.MatchString("dropbox.com", lowerURL) 73 | if match { 74 | return models.DropBox, false 75 | } 76 | 77 | // Azure emulator 78 | match, _ = regexp.MatchString("127.0.0.1:10000", lowerURL) 79 | if match { 80 | return models.Azure, true 81 | } 82 | 83 | // S3 84 | // need to think about S3 compatible devices. TODO(kpfaulkner) 85 | match, _ = regexp.MatchString("amazonaws.com", lowerURL) 86 | if match { 87 | return models.S3, false 88 | } 89 | 90 | return models.Filesystem, false 91 | } 92 | 93 | // ListContainer lists containers/blobs in URL 94 | func (ac *AzureCopy) ListContainer() (*models.SimpleContainer, error) { 95 | log.Debugf("Listing contents of %s", ac.sourceURL) 96 | 97 | container, err := ac.sourceHandler.GetSpecificSimpleContainer(ac.sourceURL) 98 | if err != nil { 99 | log.Fatal("ListContainer failed ", err) 100 | } 101 | 102 | // get the blobs for the deepest vdir which is part of the URL. 103 | ac.sourceHandler.GetContainerContents(container) 104 | return container, nil 105 | } 106 | 107 | // CreateContainer lists containers/blobs in URL 108 | func (ac *AzureCopy) CreateContainer(containerName string) error { 109 | log.Debugf("CreateContainer %s", containerName) 110 | 111 | _, err := ac.sourceHandler.CreateContainer(containerName) 112 | if err != nil { 113 | log.Fatal("CreateContainer failed ", err) 114 | } 115 | 116 | return nil 117 | } 118 | 119 | // CopyBlobByURL copy a blob from one URL to another. 120 | func (ac *AzureCopy) CopyBlobByURL(replaceExisting bool, useCopyBlobFlag bool) error { 121 | 122 | log.Debugf("CopyBlobByURL sourceURL %s", ac.sourceURL) 123 | var err error 124 | if misc.GetLastChar(ac.sourceURL) == "/" || misc.GetLastChar(ac.sourceURL) == "\\" { 125 | // copying a directory/vdir worth of stuff.... 126 | err = ac.CopyContainerByURL(ac.sourceURL, ac.destURL, replaceExisting, useCopyBlobFlag) 127 | } else { 128 | err = ac.CopySingleBlobByURL(ac.sourceURL, ac.destURL, replaceExisting, useCopyBlobFlag) 129 | } 130 | 131 | if err != nil { 132 | log.Fatal("CopyBlobByUrl error ", err) 133 | } 134 | return nil 135 | } 136 | 137 | // CopySingleBlobByURL copies a single blob referenced by URL to a destination URL 138 | // useCopyBlobFlag currently unused!! TODO(kpfaulkner) 139 | func (ac *AzureCopy) CopySingleBlobByURL(sourceURL string, destURL string, replaceExisting bool, useCopyBlobFlag bool) error { 140 | fmt.Printf("Copying single blob %s to %s\n", sourceURL, destURL) 141 | 142 | simpleSourceBlob, err := ac.sourceHandler.GetSpecificSimpleBlob(sourceURL) 143 | if err != nil { 144 | return err 145 | } 146 | 147 | // prune destination to just be last element of blobname. 148 | // dest name is the "fake" name excluding any "/" prefix (ie fake vdirs) 149 | sp := strings.Split(simpleSourceBlob.BlobCloudName, "/") 150 | simpleSourceBlob.DestName = sp[len(sp)-1] 151 | 152 | log.Debugf("single blob is %v", simpleSourceBlob) 153 | destContainer, err := ac.destHandler.GetSpecificSimpleContainer(destURL) 154 | if err != nil { 155 | return err 156 | } 157 | 158 | copyChannel := make(chan models.SimpleBlob, 1) 159 | 160 | copyChannel <- *simpleSourceBlob 161 | 162 | // launch go routines for copying. 163 | ac.launchCopyGoRoutines(destContainer, replaceExisting, copyChannel, useCopyBlobFlag) 164 | 165 | // finished copying contents to channel. Close now? 166 | close(copyChannel) 167 | 168 | // wait for all copying to be done. 169 | wg.Wait() 170 | return nil 171 | } 172 | 173 | // CopyContainerByURL copies blobs/containers from a URL to a destination URL. 174 | // This CURRENTLY uses a different method of generating blobs and containers. 175 | // The plan is to consolidate both listing and copying into using the same methods, but for now 176 | // want to make sure copying at least is able to start copying blobs before the listing is finished. 177 | // So will use GoRoutines to concurrently retrieve list of blobs and another for writing to destination. 178 | func (ac *AzureCopy) CopyContainerByURL(sourceURL string, destURL string, replaceExisting bool, useCopyBlobFlag bool) error { 179 | log.Debugf("CopyContainerByURL %s to %s", sourceURL, destURL ) 180 | deepestContainer, err := ac.sourceHandler.GetSpecificSimpleContainer(sourceURL) 181 | if err != nil { 182 | log.Fatal("CopyContainerByURL failed source: ", err) 183 | } 184 | log.Debugf("deepest source container is %s", deepestContainer.Name) 185 | 186 | deepestDestinationContainer, err := ac.destHandler.GetSpecificSimpleContainer(destURL) 187 | if err != nil { 188 | log.Fatal("CopyContainerByURL failed dest: ", err) 189 | } 190 | log.Debugf("deepest dest container %s", deepestDestinationContainer.Name) 191 | 192 | // make channel for reading from cloud. 193 | readChannel := make(chan models.SimpleContainer, 1000) 194 | 195 | // channel for individual blobs (not containers) which will be read and copied. 196 | copyChannel := make(chan models.SimpleBlob, 1000) 197 | 198 | // launch go routines for copying. 199 | ac.launchCopyGoRoutines(deepestDestinationContainer, replaceExisting, copyChannel, useCopyBlobFlag) 200 | 201 | // get container contents over channel. 202 | // get the blobs for the deepest vdir which is part of the URL. 203 | // The readChannel will be populated with containers that are populated from the "REAL" cloud container. ie Azure Container or S3 bucket. 204 | go ac.sourceHandler.GetContainerContentsOverChannel(*deepestContainer, readChannel) 205 | if err != nil { 206 | log.Fatalf("CopyContainerByURL err %s", err) 207 | } 208 | 209 | for { 210 | // get data read. 211 | containerDetails, ok := <-readChannel 212 | if !ok { 213 | // channel closed. We're done now. 214 | break 215 | } 216 | 217 | containerDetails.DisplayContainer("") 218 | 219 | // populate the copyChannel with individual blobs. 220 | ac.populateCopyChannel(&containerDetails, "", copyChannel) 221 | } 222 | 223 | // finished copying contents to channel. Close now? 224 | close(copyChannel) 225 | 226 | // wait for all copying to be done. 227 | // wait for all copying to be done. 228 | wg.Wait() 229 | return nil 230 | } 231 | 232 | // launchCopyGoRoutines starts a number of Go Routines used for copying contents. 233 | func (ac *AzureCopy) launchCopyGoRoutines(destContainer *models.SimpleContainer, replaceExisting bool, copyChannel chan models.SimpleBlob, useCopyBlobFlag bool) { 234 | 235 | ac.config.ConcurrentCount = 1 236 | 237 | log.Debugf("launching %d goroutines", ac.config.ConcurrentCount) 238 | for i := 0; i < int(ac.config.ConcurrentCount); i++ { 239 | wg.Add(1) 240 | 241 | if useCopyBlobFlag { 242 | go ac.copyBlobFromChannelUsingCopyBlobFlag(destContainer, replaceExisting, copyChannel) 243 | } else { 244 | go ac.copyBlobFromChannel(destContainer, replaceExisting, copyChannel) 245 | } 246 | } 247 | } 248 | 249 | // populateCopyChannel copies blobs into channel for later copying. 250 | func (ac *AzureCopy) populateCopyChannel(sourceContainer *models.SimpleContainer, prefix string, copyChannel chan models.SimpleBlob) error { 251 | 252 | log.Debugf("sourcecontainer blobslice size %d", len(sourceContainer.BlobSlice)) 253 | log.Debugf("populateCopyChannel ContainerSlice size %d", len(sourceContainer.ContainerSlice)) 254 | 255 | if len(sourceContainer.BlobSlice) > 0 { 256 | log.Debugf("first blob is %s", sourceContainer.BlobSlice[0].Name) 257 | } 258 | 259 | if len(sourceContainer.ContainerSlice) > 0 { 260 | log.Debugf("first container is %s", sourceContainer.ContainerSlice[0].Name) 261 | } 262 | 263 | log.Debugf("populateCopyChannel prefix %s", prefix) 264 | log.Debugf("populateCopyChannel container %s", sourceContainer.Name) 265 | 266 | // copy all blobs 267 | for _, blob := range sourceContainer.BlobSlice { 268 | 269 | log.Debugf("populateCopyChannel blobname %s", blob.Name) 270 | if prefix != "" { 271 | blob.DestName = prefix + "/" + blob.Name 272 | } else { 273 | blob.DestName = blob.Name 274 | } 275 | 276 | log.Debugf("changing destname %s", blob.DestName) 277 | log.Debugf("Adding blob %s to channel", blob.URL) 278 | copyChannel <- *blob 279 | } 280 | 281 | log.Debugf("DB populateCopyChannel name %s", sourceContainer.Name) 282 | log.Debugf("populateCopyChannel containerSlice size %d", len(sourceContainer.ContainerSlice)) 283 | 284 | for _, container := range sourceContainer.ContainerSlice { 285 | log.Debugf("container name is %s", container.Name) 286 | var newPrefix string 287 | if prefix != "" { 288 | newPrefix = prefix + "/" + container.Name 289 | } else { 290 | newPrefix = container.Name 291 | } 292 | 293 | err := ac.populateCopyChannel(container, newPrefix, copyChannel) 294 | if err != nil { 295 | log.Fatal(err) 296 | } 297 | } 298 | 299 | return nil 300 | } 301 | 302 | // copyBlobFromChannel reads blob from channel and copies it to destinationContainer 303 | func (ac *AzureCopy) copyBlobFromChannel(destContainer *models.SimpleContainer, replaceExisting bool, copyChannel chan models.SimpleBlob) { 304 | 305 | defer wg.Done() 306 | 307 | for { 308 | blob, ok := <-copyChannel 309 | if !ok { 310 | log.Debugf("Closing channel for copyBlobFromChannel") 311 | // closed... so all writing is done? Or what? 312 | return 313 | } 314 | 315 | // check if we need to skip it. 316 | if !replaceExisting { 317 | exists, err := ac.destHandler.BlobExists(*destContainer, blob.DestName) 318 | if err != nil { 319 | log.Debugf("Unable to copy %s\n", blob.URL) 320 | continue 321 | } 322 | 323 | if exists { 324 | fmt.Printf("Skipping %s\n", blob.URL) 325 | continue 326 | } 327 | } 328 | 329 | log.Debugf("Read blob %s", blob.URL) 330 | ac.ReadBlob(&blob) 331 | 332 | // rename name for destination. HACK! 333 | blob.Name = blob.DestName 334 | 335 | ac.WriteBlob(destContainer, &blob) 336 | } 337 | } 338 | 339 | // copyBlobFromChannelUsingCopyBlobFlag reads blob from channel, makes presigned URL (based on source blob) then triggers Azure CopyBlob operation. 340 | func (ac *AzureCopy) copyBlobFromChannelUsingCopyBlobFlag(destContainer *models.SimpleContainer, replaceExisting bool, copyChannel chan models.SimpleBlob) { 341 | 342 | defer wg.Done() 343 | 344 | //azureAccountName, azureAccountKey := utils.GetAzureCredentials(false, ac.config) 345 | // azureHelper := helpers.NewAzureHelper(azureAccountName, azureAccountKey) 346 | 347 | for { 348 | blob, ok := <-copyChannel 349 | if !ok { 350 | // closed... so all writing is done? Or what? 351 | return 352 | } 353 | 354 | // check if we need to skip it. 355 | if !replaceExisting { 356 | exists, err := ac.destHandler.BlobExists(*destContainer, blob.DestName) 357 | if err != nil { 358 | log.Debugf("Unable to copy %s", blob.URL) 359 | continue 360 | } 361 | 362 | if exists { 363 | fmt.Printf("Skipping %s", blob.URL) 364 | continue 365 | } 366 | } 367 | 368 | // generate presigned URL 369 | url, err := ac.sourceHandler.GeneratePresignedURL(&blob) 370 | if err != nil { 371 | log.Errorf("Unable to generate presigned URL %s", blob.URL) 372 | continue 373 | } 374 | 375 | fmt.Printf("displaying url just for the fun of it %s\n", url) 376 | fmt.Printf("Copying %s to %s\n", blob.Name, destContainer.Name+"/"+blob.DestName) 377 | //azureHelper.DoCopyBlobUsingAzureCopyBlobFlag(url, destContainer, blob.DestName) 378 | } 379 | } 380 | 381 | // GetHandlerForURL returns the appropriate handler for a given cloud type. 382 | func (ac *AzureCopy) GetHandlerForURL(url string, isSource bool, cacheToDisk bool) handlers.CloudHandlerInterface { 383 | cloudType, isEmulator := ac.getCloudType(url) 384 | handler := utils.GetHandler(cloudType, isSource, ac.config, cacheToDisk, isEmulator) 385 | return handler 386 | } 387 | 388 | func (ac *AzureCopy) GetSourceRootContainer() models.SimpleContainer { 389 | rootContainer := ac.sourceHandler.GetRootContainer() 390 | return rootContainer 391 | } 392 | 393 | func (ac *AzureCopy) GetDestRootContainer() models.SimpleContainer { 394 | rootContainer := ac.destHandler.GetRootContainer() 395 | return rootContainer 396 | } 397 | 398 | // GetContainerContents populates the container with data. 399 | func (ac *AzureCopy) GetContainerContents(container *models.SimpleContainer) { 400 | 401 | // check where container came from. 402 | if container.IsSource { 403 | ac.sourceHandler.GetContainerContents(container) 404 | } else { 405 | ac.destHandler.GetContainerContents(container) 406 | } 407 | } 408 | 409 | // GetDestContainerContents populates the container with data. 410 | func (ac *AzureCopy) GetDestContainerContents(container *models.SimpleContainer) { 411 | ac.destHandler.GetContainerContents(container) 412 | } 413 | 414 | // ReadBlob reads a blob and keeps it in memory OR caches to disk. 415 | // (or in the special case of azure copyblob flag it will do something tricky, once I get to that part) 416 | func (ac *AzureCopy) ReadBlob(blob *models.SimpleBlob) { 417 | 418 | log.Debugf("ReadBlob %s", blob.URL) 419 | err := ac.sourceHandler.PopulateBlob(blob) 420 | 421 | if err != nil { 422 | log.Fatal(err) 423 | } 424 | } 425 | 426 | // doesDestinationBlobExist checks if the destination blob exists 427 | func (ac *AzureCopy) doesDestinationBlobExist(destContainer *models.SimpleContainer, sourceBlob *models.SimpleBlob) (bool, error) { 428 | 429 | if destContainer == nil { 430 | log.Debugf("dest container is nil") 431 | } else { 432 | log.Debugf("check dest, write dest loc %s ", destContainer.URL) 433 | } 434 | 435 | if err := ac.destHandler.WriteBlob(destContainer, sourceBlob); err != nil { 436 | log.Fatal("WriteBlob kaboom ", err) 437 | } 438 | return false, nil 439 | } 440 | 441 | // WriteBlob writes a source blob (can be from anywhere) to a destination container (can and probably will be a different cloud platform) 442 | func (ac *AzureCopy) WriteBlob(destContainer *models.SimpleContainer, sourceBlob *models.SimpleBlob) error { 443 | 444 | if destContainer == nil { 445 | log.Debugf("dest container is nil\n") 446 | } else { 447 | log.Debugf("write dest loc %s\n", destContainer.URL) 448 | } 449 | 450 | if err := ac.destHandler.WriteBlob(destContainer, sourceBlob); err != nil { 451 | log.Fatalf("WriteBlob kaboom %s\n", err) 452 | } 453 | 454 | // if cached delete the cache. 455 | // make sure dont delete if just simply read from local filesystem (due to source being local file) 456 | if !sourceBlob.BlobInMemory && ac.config.Command != misc.CommandCopyBlob && sourceBlob.Origin != models.Filesystem { 457 | log.Debugf("About to delete cache file %s", sourceBlob.DataCachedAtPath) 458 | err := os.Remove(sourceBlob.DataCachedAtPath) 459 | if err != nil { 460 | log.Errorf("Unable to delete cache file %s", err) 461 | } 462 | log.Debugf("deleted cache file %s", sourceBlob.DataCachedAtPath) 463 | } 464 | 465 | return nil 466 | } 467 | -------------------------------------------------------------------------------- /azurecopy/handlers/AzureHandler.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "azurecopy/azurecopy/models" 5 | "azurecopy/azurecopy/utils/containerutils" 6 | "azurecopy/azurecopy/utils/misc" 7 | "encoding/base64" 8 | "errors" 9 | "fmt" 10 | "io/ioutil" 11 | "net/url" 12 | "os" 13 | "regexp" 14 | "strings" 15 | 16 | log "github.com/Sirupsen/logrus" 17 | "github.com/satori/uuid" 18 | 19 | storage "github.com/azure/azure-storage-blob-go/2016-05-31/azblob" 20 | "time" 21 | "context" 22 | 23 | "bytes" 24 | ) 25 | 26 | type AzureHandler struct { 27 | serviceURL storage.ServiceURL 28 | 29 | // determine if we're caching the blob to disk during copy operations. 30 | // or if we're keeping it in memory 31 | cacheToDisk bool 32 | cacheLocation string 33 | 34 | // is this handler for the source or dest? 35 | IsSource bool 36 | 37 | // dealing with emulator. 38 | IsEmulator bool 39 | } 40 | 41 | // NewAzureHandler factory to create new one. Evil? 42 | func NewAzureHandler(accountName string, accountKey string, isSource bool, cacheToDisk bool, isEmulator bool) (*AzureHandler, error) { 43 | ah := new(AzureHandler) 44 | 45 | ah.cacheToDisk = cacheToDisk 46 | dir, err := ioutil.TempDir("", "azurecopy") 47 | if err != nil { 48 | log.Fatalf("Unable to create temp directory %s", err) 49 | } 50 | 51 | ah.cacheLocation = dir 52 | ah.IsSource = isSource 53 | ah.IsEmulator = isEmulator 54 | 55 | if isEmulator || (accountName == "" && accountKey == "") { 56 | // set emulator accountName and accountKey 57 | } 58 | 59 | credential := storage.NewSharedKeyCredential(accountName, accountKey) 60 | p := storage.NewPipeline(credential, storage.PipelineOptions{}) 61 | u, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", accountName)) 62 | serviceURL := storage.NewServiceURL(*u, p) 63 | 64 | if err != nil { 65 | 66 | // indicate error somehow.. still trying to figure that out with GO. 67 | return nil, err 68 | } 69 | 70 | ah.serviceURL = serviceURL 71 | return ah, nil 72 | } 73 | 74 | // GetRootContainer gets root container of Azure. In reality there isn't a root container, but this would basically be a SimpleContainer 75 | // that has the containerSlice populated with the real Azure containers. 76 | func (ah *AzureHandler) GetRootContainer() models.SimpleContainer { 77 | 78 | ctx, cancel := context.WithTimeout(context.Background(), 600*time.Second) 79 | defer cancel() 80 | containerResponse, _:= ah.serviceURL.ListContainers( ctx, storage.Marker{}, storage.ListContainersOptions{} ) 81 | rootContainer := models.NewSimpleContainer() 82 | 83 | for _, c := range containerResponse.Containers { 84 | sc := models.NewSimpleContainer() 85 | sc.Name = c.Name 86 | sc.Origin = models.Azure 87 | 88 | rootContainer.ContainerSlice = append(rootContainer.ContainerSlice, sc) 89 | } 90 | 91 | return *rootContainer 92 | } 93 | 94 | // BlobExists checks if blob exists 95 | func (ah *AzureHandler) BlobExists(container models.SimpleContainer, blobName string) (bool, error) { 96 | 97 | azureContainerName, _ := ah.getContainerAndBlobNames(&container, blobName) 98 | containerURL := ah.serviceURL.NewContainerURL(azureContainerName) 99 | ctx, cancel := context.WithTimeout(context.Background(), 600*time.Second) 100 | defer cancel() 101 | 102 | // must be a better way surely? 103 | resp, err := containerURL.ListBlobs(ctx, storage.Marker{}, storage.ListBlobsOptions{Prefix: blobName}) 104 | if err != nil { 105 | return false, err 106 | } 107 | 108 | for _,bn := range resp.Blobs.Blob { 109 | if bn.Name == blobName { 110 | return true, nil 111 | } 112 | } 113 | 114 | return false, nil 115 | } 116 | 117 | // GetSpecificSimpleContainer given a URL (ending in /) then get the SIMPLE container that represents it. 118 | // returns the container of the last most part of the url. 119 | // eg. if the url was https://myacct.blob.core.windows.net/realazurecontainer/vdir1/vdir2/ then the simple container 120 | // returned is vdir2. 121 | func (ah *AzureHandler) GetSpecificSimpleContainer(URL string) (*models.SimpleContainer, error) { 122 | 123 | lastChar := URL[len(URL)-1:] 124 | // MUST be a better way to get the last character. 125 | if lastChar != "/" { 126 | return nil, errors.New("Needs to end with a /") 127 | } 128 | 129 | _, containerName, blobPrefix, _, err := ah.validateURL(URL) 130 | if err != nil { 131 | log.Fatal("GetSpecificSimpleContainer err", err) 132 | } 133 | 134 | var simpleContainer *models.SimpleContainer 135 | 136 | simpleContainer, err = ah.getAzureContainerAsSimpleContainer(containerName) 137 | if err != nil { 138 | 139 | log.Debugf("container %s didn't exist, trying to create it: %s", containerName, err) 140 | 141 | _, _ = ah.getOrCreateContainer( containerName) 142 | simpleContainer, err = ah.getAzureContainerAsSimpleContainer(containerName) 143 | if err != nil { 144 | return nil, err 145 | } 146 | } 147 | 148 | subContainer, lastContainer := ah.generateSubContainers(simpleContainer, blobPrefix) 149 | 150 | if subContainer != nil { 151 | simpleContainer.ContainerSlice = append(simpleContainer.ContainerSlice, subContainer) 152 | } 153 | 154 | // return the "deepest" container. 155 | return lastContainer, nil 156 | } 157 | 158 | // GetContainerContentsOverChannel given a URL (ending in /) returns all the contents of the container over a channel 159 | // This returns a COPY of the original source container but has been populated with *some* of the blobs/subcontainers in it. 160 | func (ah *AzureHandler) GetContainerContentsOverChannel(sourceContainer models.SimpleContainer, blobChannel chan models.SimpleContainer) error { 161 | 162 | azureContainer, blobPrefix := containerutils.GetContainerAndBlobPrefix(&sourceContainer) 163 | 164 | containerURL := ah.serviceURL.NewContainerURL(azureContainer.Name) 165 | 166 | ctx, cancel := context.WithTimeout(context.Background(), 600*time.Second) 167 | defer cancel() 168 | 169 | marker := storage.Marker{} 170 | 171 | // now we have the azure container and the prefix, we should be able to get a list of 172 | // SimpleContainers and SimpleBlobs to add this to original container. 173 | // Keep max results to 1000, can loop through and 174 | // params := storage.ListBlobsParameters{Prefix: blobPrefix, MaxResults: 1000} 175 | done := false 176 | for done == false { 177 | // copy of container, dont want to send back ever growing container via the channel. 178 | containerClone := sourceContainer 179 | 180 | //azureContainer := ah.blobStorageClient.GetContainerReference(azureContainer.Name) 181 | blobListResponse, err := containerURL.ListBlobs( ctx, marker, storage.ListBlobsOptions{ Prefix: blobPrefix}) 182 | if err != nil { 183 | log.Fatal("Error") 184 | } 185 | 186 | ah.populateSimpleContainer(blobListResponse, &containerClone, blobPrefix) 187 | 188 | // return entire container via channel. 189 | blobChannel <- containerClone 190 | 191 | // if marker, then keep going. 192 | if blobListResponse.NextMarker.NotDone() { 193 | marker = blobListResponse.NextMarker 194 | } else { 195 | done = true 196 | } 197 | } 198 | 199 | close(blobChannel) 200 | return nil 201 | } 202 | 203 | func (ah *AzureHandler) GeneratePresignedURL(blob *models.SimpleBlob) (string, error) { 204 | return "", nil 205 | } 206 | 207 | // GetSpecificSimpleBlob given a URL (NOT ending in /) then get the SIMPLE blob that represents it. 208 | // The DestName will be the last element of the URL, whether it's a real blobname or not. 209 | // eg. https://...../mycontainer/vdir1/vdir2/blobname will return a DestName of "blobname" even though strictly 210 | // speaking the true blobname is "vdir1/vdir2/blobname". 211 | // Will revisit this if it causes a problem. 212 | func (ah *AzureHandler) GetSpecificSimpleBlob(URL string) (*models.SimpleBlob, error) { 213 | // MUST be a better way to get the last character. 214 | if URL[len(URL)-2:len(URL)-1] == "/" { 215 | return nil, errors.New("Cannot end with a /") 216 | } 217 | _, containerName, blobName, pretendBlobName, err := ah.validateURL(URL) 218 | if err != nil { 219 | return nil, err 220 | } 221 | 222 | simpleContainer, err := ah.getAzureContainerAsSimpleContainer(containerName) 223 | 224 | b := models.SimpleBlob{} 225 | 226 | // if we're talking vdir1/vdir2/blobname then b.Name should be "blobname" 227 | b.Name = pretendBlobName 228 | b.Origin = models.Azure 229 | b.ParentContainer = simpleContainer 230 | b.BlobCloudName = blobName 231 | return &b, nil 232 | } 233 | 234 | // ReadBlob reads a blob of a given name from a particular SimpleContainer and returns the SimpleBlob 235 | // The SimpleContainer is NOT necessarily a direct mapping to an Azure container but may be representing a virtual directory. 236 | // ie we might have RootSimpleContainer -> SimpleContainer(myrealcontainer) -> SimpleContainer(vdir1) -> SimpleContainer(vdir2) 237 | // and if the blobName is "myblob" then the REAL underlying Azure structure would be container == "myrealcontainer" 238 | // and the blob name is vdir/vdir2/myblob 239 | func (ah *AzureHandler) ReadBlob(container models.SimpleContainer, blobName string) models.SimpleBlob { 240 | var blob models.SimpleBlob 241 | 242 | return blob 243 | } 244 | 245 | // PopulateBlob. Used to read a blob IFF we already have a reference to it. 246 | func (ah *AzureHandler) PopulateBlob(blob *models.SimpleBlob) error { 247 | azureContainerName := ah.generateAzureContainerName(*blob) 248 | azureBlobName := blob.BlobCloudName 249 | 250 | containerURL := ah.serviceURL.NewContainerURL(azureContainerName) 251 | blobURL := containerURL.NewBlobURL(azureBlobName) 252 | 253 | ctx, cancel := context.WithTimeout(context.Background(), 600*time.Second) 254 | defer cancel() 255 | 256 | 257 | resp, err := blobURL.GetBlob(ctx, storage.BlobRange{}, storage.BlobAccessConditions{}, false) 258 | if err != nil { 259 | return err 260 | } 261 | defer resp.Body().Close() 262 | 263 | // file stream for cache. 264 | var cacheFile *os.File 265 | 266 | // populate this to disk. 267 | if ah.cacheToDisk { 268 | 269 | cacheName := misc.GenerateCacheName(azureContainerName + blob.BlobCloudName) 270 | blob.DataCachedAtPath = ah.cacheLocation + "/" + cacheName 271 | log.Debugf("azure blob %s cached at location %s", blob.BlobCloudName, blob.DataCachedAtPath) 272 | cacheFile, err = os.OpenFile(blob.DataCachedAtPath, os.O_WRONLY|os.O_CREATE, 0666) 273 | defer cacheFile.Close() 274 | 275 | if err != nil { 276 | log.Fatalf("Populate blob %s", err) 277 | return err 278 | } 279 | } else { 280 | blob.DataInMemory = []byte{} 281 | } 282 | 283 | // 100k buffer... way too small? 284 | buffer := make([]byte, 1024*100) 285 | numBytesRead := 0 286 | 287 | finishedProcessing := false 288 | for finishedProcessing == false { 289 | numBytesRead, err = resp.Body().Read(buffer) 290 | if err != nil { 291 | finishedProcessing = true 292 | } 293 | 294 | if numBytesRead <= 0 { 295 | finishedProcessing = true 296 | continue 297 | } 298 | 299 | // if we're caching, write to a file. 300 | if ah.cacheToDisk { 301 | _, err = cacheFile.Write(buffer[:numBytesRead]) 302 | if err != nil { 303 | log.Fatal(err) 304 | return err 305 | } 306 | } else { 307 | 308 | // needs to go into a byte array. How do we expand a slice again? 309 | blob.DataInMemory = append(blob.DataInMemory, buffer[:numBytesRead]...) 310 | } 311 | } 312 | 313 | return nil 314 | } 315 | 316 | func (ah *AzureHandler) WriteContainer(sourceContainer *models.SimpleContainer, destContainer *models.SimpleContainer) error { 317 | return nil 318 | } 319 | 320 | // WriteBlob writes a blob to an Azure container. 321 | // The SimpleContainer is NOT necessarily a direct mapping to an Azure container but may be representing a virtual directory. 322 | // ie we might have RootSimpleContainer -> SimpleContainer(myrealcontainer) -> SimpleContainer(vdir1) -> SimpleContainer(vdir2) 323 | // and if the blobName is "myblob" then the REAL underlying Azure structure would be container == "myrealcontainer" 324 | // and the blob name is vdir/vdir2/myblob 325 | func (ah *AzureHandler) WriteBlob(destContainer *models.SimpleContainer, sourceBlob *models.SimpleBlob) error { 326 | 327 | log.Debugf("Azure WriteBlob destcont %s blob %s", destContainer.Name, sourceBlob.Name) 328 | 329 | var err error 330 | if ah.cacheToDisk { 331 | err = ah.writeBlobFromCache(destContainer, sourceBlob) 332 | } else { 333 | err = ah.writeBlobFromMemory(destContainer, sourceBlob) 334 | } 335 | 336 | if err != nil { 337 | log.Fatal(err) 338 | return err 339 | } 340 | 341 | return nil 342 | } 343 | 344 | // CreateContainer creates an Azure container. 345 | // ie will only do ROOT level containers (ie REAL Azure container) 346 | func (ah *AzureHandler) CreateContainer(containerName string) (models.SimpleContainer, error) { 347 | var container models.SimpleContainer 348 | 349 | _, err := ah.getOrCreateContainer(containerName) 350 | if err != nil { 351 | log.Fatal(err) 352 | } 353 | 354 | // dont get it... creates an empty simplecontainer... this needs to be relooked at! 355 | // only command uses it and discards it straight away, so might be ok. 356 | return container, nil 357 | } 358 | 359 | // GetContainer gets a container. Populating the subtree? OR NOT? hmmmm 360 | func (ah *AzureHandler) GetContainer(containerName string) models.SimpleContainer { 361 | var container models.SimpleContainer 362 | 363 | return container 364 | } 365 | 366 | // GetContainerContents populates the passed container with the real contents. 367 | // Can determine if the SimpleContainer is a real container or something virtual. 368 | // We need to trace back to the root node and determine what is really a container and 369 | // what is a blob. 370 | // 371 | // For Azure only the children of the root node can be a real azure container. Everything else 372 | // is a blob or a blob pretending to have vdirs. 373 | // 374 | // TODO(kpfaulkner) use marker and get next lot of results when we have > 5000 blobs. 375 | func (ah *AzureHandler) GetContainerContents(container *models.SimpleContainer) error { 376 | 377 | azureContainer, blobPrefix := containerutils.GetContainerAndBlobPrefix(container) 378 | 379 | // now we have the azure container and the prefix, we should be able to get a list of 380 | // SimpleContainers and SimpleBlobs to add this to original container. 381 | containerURL := ah.serviceURL.NewContainerURL(azureContainer.Name) 382 | ctx := context.Background() // This example uses a never-expiring context 383 | 384 | blobListResponse, err := containerURL.ListBlobs(ctx, storage.Marker{}, storage.ListBlobsOptions{Prefix: blobPrefix} ) 385 | if err != nil { 386 | log.Fatal("Error") 387 | } 388 | 389 | ah.populateSimpleContainer(blobListResponse, azureContainer, blobPrefix) 390 | 391 | return nil 392 | } 393 | 394 | // DoCopyBlobUsingAzureCopyBlobFlag copy using Azure CopyBlob flag. 395 | func (ah *AzureHandler) DoCopyBlobUsingAzureCopyBlobFlag(sourceBlob *models.SimpleBlob, destContainer *models.SimpleContainer, destBlobName string) error { 396 | 397 | return nil 398 | } 399 | 400 | // Get container... or create a new one. 401 | func (ah *AzureHandler) getOrCreateContainer(containerName string) (*storage.ContainerURL, error) { 402 | 403 | containerURL := ah.serviceURL.NewContainerURL( containerName) 404 | ctx := context.Background() // This example uses a never-expiring context 405 | _, err := containerURL.Create(ctx, storage.Metadata{}, storage.PublicAccessNone) 406 | 407 | if serr, ok := err.(storage.StorageError); ok { // This error is a Service-specific error 408 | if serr.ServiceCode() != storage.ServiceCodeContainerAlreadyExists { 409 | return nil, err 410 | } 411 | } 412 | 413 | return &containerURL, nil 414 | } 415 | 416 | func (ah *AzureHandler) generateSubContainers(azureContainer *models.SimpleContainer, blobPrefix string) (*models.SimpleContainer, *models.SimpleContainer) { 417 | 418 | var containerToReturn *models.SimpleContainer 419 | var lastContainer *models.SimpleContainer 420 | doneFirst := false 421 | 422 | // strip off last / 423 | if len(blobPrefix) > 0 { 424 | blobPrefix = blobPrefix[:len(blobPrefix)-1] 425 | sp := strings.Split(blobPrefix, "/") 426 | 427 | for _, segment := range sp { 428 | container := models.NewSimpleContainer() 429 | container.Name = segment 430 | if !doneFirst { 431 | container.ParentContainer = azureContainer 432 | containerToReturn = container 433 | doneFirst = true 434 | } else { 435 | container.ParentContainer = lastContainer 436 | lastContainer.ContainerSlice = append(lastContainer.ContainerSlice, container) 437 | } 438 | 439 | lastContainer = container 440 | } 441 | } else { 442 | 443 | // just return existing container (lastContainer) and no subcontainer (containerToReturn) 444 | containerToReturn = nil 445 | lastContainer = azureContainer 446 | } 447 | 448 | return containerToReturn, lastContainer 449 | } 450 | 451 | func (ah *AzureHandler) getAzureContainerAsSimpleContainer(containerName string) (*models.SimpleContainer, error) { 452 | 453 | rootContainer := ah.GetRootContainer() 454 | 455 | for _, container := range rootContainer.ContainerSlice { 456 | if container.Name == containerName { 457 | return container, nil 458 | } 459 | } 460 | 461 | return nil, errors.New("Unable to find container") 462 | 463 | } 464 | 465 | 466 | 467 | // validateURL returns accountName, container Name, blob Name and error 468 | // passes real URL such as https://myacct.blob.core.windows.net/mycontainer/vdir1/vdir2/blobPrefix 469 | func (ah *AzureHandler) validateURL(URL string) (string, string, string, string, error) { 470 | 471 | lowerURL := strings.ToLower(URL) 472 | 473 | // ugly, do this properly!!! TODO(kpfaulkner) 474 | pruneCount := 0 475 | match, _ := regexp.MatchString("http://", lowerURL) 476 | if match { 477 | pruneCount = 7 478 | } 479 | 480 | match, _ = regexp.MatchString("https://", lowerURL) 481 | if match { 482 | pruneCount = 8 483 | } 484 | 485 | // trim protocol 486 | URL = URL[pruneCount:] 487 | sp := strings.Split(URL, "/") 488 | 489 | sp2 := strings.Split(sp[0], ".") 490 | accountName := sp2[0] 491 | 492 | var containerName string 493 | var blobName string 494 | 495 | if !ah.IsEmulator { 496 | containerName = sp[1] 497 | blobName = strings.Join(sp[2:], "/") 498 | } else { 499 | containerName = sp[2] 500 | blobName = strings.Join(sp[3:], "/") 501 | } 502 | 503 | pretendBlobName := sp[ len(sp) -1] 504 | return accountName, containerName, blobName,pretendBlobName, nil 505 | } 506 | 507 | 508 | // PopulateBlob. Used to read a blob IFF we already have a reference to it. 509 | func (ah *AzureHandler) getBlobURL( containerName string, azureBlobName string) (*storage.BlobURL, error) { 510 | containerURL := ah.serviceURL.NewContainerURL(containerName) 511 | blobURL := containerURL.NewBlobURL(azureBlobName) 512 | 513 | return &blobURL, nil 514 | } 515 | 516 | // generateAzureContainerName gets the REAL Azure container name for the simpleBlob 517 | func (ah *AzureHandler) generateAzureContainerName(blob models.SimpleBlob) string { 518 | currentContainer := blob.ParentContainer 519 | 520 | for currentContainer.ParentContainer != nil { 521 | currentContainer = currentContainer.ParentContainer 522 | } 523 | return currentContainer.Name 524 | } 525 | 526 | 527 | 528 | func (ah *AzureHandler) getContainerAndBlobNames(destContainer *models.SimpleContainer, sourceBlobName string) (string, string) { 529 | 530 | azureContainer, blobPrefix := containerutils.GetContainerAndBlobPrefix(destContainer) 531 | azureContainerName := azureContainer.Name 532 | 533 | var azureBlobName string 534 | 535 | if blobPrefix != "" { 536 | if misc.GetLastChar(blobPrefix) == "/" { 537 | azureBlobName = blobPrefix + sourceBlobName 538 | 539 | } else { 540 | azureBlobName = blobPrefix + "/" + sourceBlobName 541 | } 542 | } else { 543 | azureBlobName = sourceBlobName 544 | } 545 | 546 | return azureContainerName, azureBlobName 547 | } 548 | 549 | // writeBlobFromCache.. read the cache file and pass the byte slice onto the real writer. 550 | func (ah *AzureHandler) writeBlobFromCache(destContainer *models.SimpleContainer, sourceBlob *models.SimpleBlob) error { 551 | azureContainerName, azureBlobName := ah.getContainerAndBlobNames(destContainer, sourceBlob.Name) 552 | 553 | _, err := ah.getOrCreateContainer( azureContainerName) 554 | if err != nil { 555 | log.Fatal(err) 556 | return err 557 | } 558 | 559 | log.Debugf("writeBlobFromCache container: %s blob: %s", azureContainerName, azureBlobName) 560 | // file stream for cache. 561 | var cacheFile *os.File 562 | 563 | // need to get cache dir from somewhere! 564 | cacheFile, err = os.OpenFile(sourceBlob.DataCachedAtPath, os.O_RDONLY, 0) 565 | if err != nil { 566 | log.Fatal(err) 567 | return err 568 | } 569 | defer cacheFile.Close() 570 | 571 | buffer := make([]byte, 1024*100) 572 | numBytesRead := 0 573 | blockIDList := []string{} 574 | finishedProcessing := false 575 | for finishedProcessing == false { 576 | numBytesRead, err = cacheFile.Read(buffer) 577 | if err != nil { 578 | finishedProcessing = true 579 | continue 580 | } 581 | 582 | if numBytesRead <= 0 { 583 | finishedProcessing = true 584 | continue 585 | } 586 | blockID, err := ah.writeMemoryToBlob(azureContainerName, azureBlobName, buffer[:numBytesRead]) 587 | if err != nil { 588 | log.Fatal("Unable to write memory to blob ", err) 589 | } 590 | 591 | blockIDList = append(blockIDList, blockID) 592 | } 593 | 594 | // finialize the blob 595 | err = ah.putBlockIDList(azureContainerName, azureBlobName, blockIDList) 596 | if err != nil { 597 | log.Fatal("putBlockIDList failed ", err) 598 | } 599 | 600 | return nil 601 | } 602 | 603 | func (ah *AzureHandler) writeBlobFromMemory(destContainer *models.SimpleContainer, sourceBlob *models.SimpleBlob) error { 604 | 605 | azureContainerName, azureBlobName := ah.getContainerAndBlobNames(destContainer, sourceBlob.Name) 606 | 607 | _, err := ah.getOrCreateContainer(azureContainerName) 608 | if err != nil { 609 | log.Fatal(err) 610 | return err 611 | } 612 | 613 | totalBytes := len(sourceBlob.DataInMemory) 614 | bufferSize := 1024 * 100 615 | buffer := make([]byte, bufferSize) 616 | numBytesRead := 0 617 | bytesWritten := 0 618 | 619 | blockIDList := []string{} 620 | 621 | for bytesWritten < totalBytes { 622 | 623 | checkNumBytesToRead := bufferSize 624 | if totalBytes-numBytesRead < bufferSize { 625 | checkNumBytesToRead = totalBytes - numBytesRead 626 | } 627 | 628 | // write 100k at a time? 629 | // too small? too big? 630 | buffer = sourceBlob.DataInMemory[numBytesRead : numBytesRead+checkNumBytesToRead] 631 | 632 | blockID, err := ah.writeMemoryToBlob(azureContainerName, azureBlobName, buffer) 633 | if err != nil { 634 | log.Fatal("Unable to write memory to blob ", err) 635 | } 636 | 637 | blockIDList = append(blockIDList, blockID) 638 | } 639 | 640 | // finialize the blob 641 | err = ah.putBlockIDList(destContainer.Name, sourceBlob.Name, blockIDList) 642 | if err != nil { 643 | log.Fatal("putBlockIDList failed ", err) 644 | } 645 | 646 | return nil 647 | } 648 | 649 | func (ah *AzureHandler) putBlockIDList(containerName string, blobName string, blockIDList []string) error { 650 | 651 | log.Debugf("putBlockIDList container %s: blobName %s", containerName, blobName) 652 | 653 | containerURL := ah.serviceURL.NewContainerURL(containerName) 654 | blobURL := containerURL.NewBlockBlobURL(blobName) 655 | 656 | ctx, cancel := context.WithTimeout(context.Background(), 600*time.Second) 657 | defer cancel() 658 | _, err := blobURL.PutBlockList(ctx, blockIDList, storage.BlobHTTPHeaders{}, storage.Metadata{}, storage.BlobAccessConditions{}) 659 | return err 660 | 661 | } 662 | 663 | func (ah *AzureHandler) writeMemoryToBlob(containerName string, blobName string, buffer []byte) (string, error) { 664 | 665 | // generate hash of bytearray. 666 | blockID := "" 667 | 668 | //hasher := sha1.New() 669 | //hasher.Write(buffer) 670 | //blockID = hex.EncodeToString(hasher.Sum(nil)) 671 | blockID = fmt.Sprintf("%s", uuid.NewV1()) 672 | 673 | blockID = base64.StdEncoding.EncodeToString([]byte(blockID)) 674 | 675 | containerURL := ah.serviceURL.NewContainerURL(containerName) 676 | blobURL := containerURL.NewBlockBlobURL(blobName) 677 | log.Debugf("blockID %s", blockID) 678 | 679 | ctx, cancel := context.WithTimeout(context.Background(), 600*time.Second) 680 | defer cancel() 681 | 682 | _, err := blobURL.PutBlock(ctx, blockID, bytes.NewReader(buffer), storage.LeaseAccessConditions{}) 683 | 684 | if err != nil { 685 | log.Fatal("Unable to PutBlock ", blockID) 686 | } 687 | 688 | return blockID, nil 689 | } 690 | 691 | 692 | // populateSimpleContainer takes a list of Azure blobs and breaks them into virtual directories (SimpleContainers) and 693 | // SimpleBlob trees. 694 | // 695 | // vdir1/vdir2/blob1 696 | // vdir1/blob2 697 | // vdir1/vdir3/blob3 698 | // blob4 699 | func (ah *AzureHandler) populateSimpleContainer(blobListResponse *storage.ListBlobsResponse, container *models.SimpleContainer, blobPrefix string) { 700 | 701 | for _, blob := range blobListResponse.Blobs.Blob { 702 | 703 | log.Debugf("populateSimpleContainer blob %s", blob.Name) 704 | sp := strings.Split(blob.Name, "/") 705 | 706 | // if no / then no subdirs etc. Just add as is. 707 | if len(sp) == 1 { 708 | b := models.SimpleBlob{} 709 | b.Name = blob.Name 710 | b.Origin = container.Origin 711 | b.ParentContainer = container 712 | b.BlobCloudName = blob.Name 713 | // add to the blob slice within the container 714 | container.BlobSlice = append(container.BlobSlice, &b) 715 | } else { 716 | 717 | currentContainer := container 718 | // if slashes, then split into chunks and create accordingly. 719 | // skip last one since thats the blob name. 720 | spShort := sp[0 : len(sp)-1] 721 | for _, segment := range spShort { 722 | 723 | // check if container already has a subcontainer with appropriate name 724 | subContainer := ah.getSubContainer(currentContainer, segment) 725 | 726 | if subContainer != nil { 727 | // then we have a blob so add it to currentContainer 728 | currentContainer = subContainer 729 | } 730 | } 731 | 732 | b := models.SimpleBlob{} 733 | b.Name = sp[len(sp)-1] 734 | b.Origin = container.Origin 735 | b.ParentContainer = container 736 | b.BlobCloudName = blob.Name // cloud specific name... ie the REAL name. 737 | 738 | containerURL := ah.serviceURL.NewContainerURL(container.Name) 739 | blobURL := containerURL.NewBlobURL(blob.Name) 740 | b.URL = blobURL.String() 741 | currentContainer.BlobSlice = append(currentContainer.BlobSlice, &b) 742 | currentContainer.Populated = true 743 | } 744 | } 745 | container.Populated = true 746 | } 747 | 748 | // getSubContainer gets an existing subcontainer with parent of container and name of segment. 749 | // otherwise it creates it, adds it to the parent container and returns the new one. 750 | func (ah *AzureHandler) getSubContainer(container *models.SimpleContainer, segment string) *models.SimpleContainer { 751 | 752 | // MUST be a shorthand way of doing this. But still crawling in GO. 753 | for _, c := range container.ContainerSlice { 754 | if c.Name == segment { 755 | return c 756 | } 757 | } 758 | 759 | // create a new one. 760 | newContainer := models.SimpleContainer{} 761 | newContainer.Name = segment 762 | newContainer.Origin = container.Origin 763 | newContainer.ParentContainer = container 764 | container.ContainerSlice = append(container.ContainerSlice, &newContainer) 765 | return &newContainer 766 | } 767 | -------------------------------------------------------------------------------- /azurecopy/handlers/CloudHandlerInterface.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "azurecopy/azurecopy/models" 5 | ) 6 | 7 | // CloudHandlerInterface is the interface for all cloud based operations 8 | // each cloud handler will implement these. 9 | // list blobs/containers/read/write etc. 10 | type CloudHandlerInterface interface { 11 | 12 | // gets root container. This will get containers/blobs in this container 13 | // NOT recursive. 14 | GetRootContainer() models.SimpleContainer 15 | 16 | // create container. 17 | CreateContainer(containerName string) (models.SimpleContainer, error) 18 | 19 | // GetSpecificSimpleContainer given a URL (ending in /) then get the SIMPLE container that represents it. 20 | // does not have to have all blobs populated in it. Those can be retrieved later via GetContainerContentsOverChannel 21 | // This is up to specific handlers. Currently (for example) Dropbox handler returns all the contents 22 | // blobs and subcontainers during this call. THIS MAY BE REVISED!! 23 | GetSpecificSimpleContainer(URL string) (*models.SimpleContainer, error) 24 | 25 | // GetContainerContentsOverChannel given a URL (ending in /) returns all the contents of the container over a channel 26 | // GetContainerContentsOverChannel given a URL (ending in /) returns all the contents of the container over a channel 27 | // This returns a COPY of the original source container but has been populated with *some* of the blobs/subcontainers in it. 28 | GetContainerContentsOverChannel(sourceContainer models.SimpleContainer, blobChannel chan models.SimpleContainer) error 29 | 30 | // GetSpecificSimpleBlob given a URL (NOT ending in /) then get the SIMPLE blob that represents it. 31 | // The DestName will be the last element of the URL, whether it's a real blobname or not. 32 | // eg. https://...../mycontainer/vdir1/vdir2/blobname will return a DestName of "blobname" even though strictly 33 | // speaking the true blobname is "vdir1/vdir2/blobname". 34 | // Will revisit this if it causes a problem. 35 | GetSpecificSimpleBlob(URL string) (*models.SimpleBlob, error) 36 | 37 | // Given a container and a blob name, read the blob. 38 | ReadBlob(container models.SimpleContainer, blobName string) models.SimpleBlob 39 | 40 | // Does blob exist 41 | BlobExists(container models.SimpleContainer, blobName string) (bool, error) 42 | 43 | // if we already have a reference to a SimpleBlob, then read it and populate it. 44 | // ie we're populating our in process copy of the blob (ie reading it from the provider). 45 | PopulateBlob(blob *models.SimpleBlob) error 46 | 47 | // given a container and blob, write blob. 48 | WriteBlob(container *models.SimpleContainer, blob *models.SimpleBlob) error 49 | 50 | // write a container (and subcontents) to the appropriate data store 51 | WriteContainer(sourceContainer *models.SimpleContainer, destContainer *models.SimpleContainer) error 52 | 53 | // Gets a container. Populating the subtree? OR NOT? hmmmm 54 | GetContainer(containerName string) models.SimpleContainer 55 | 56 | // populates container with data. 57 | GetContainerContents(container *models.SimpleContainer) error 58 | 59 | // generates presigned URL so Azure can access blob for CopyBlob flag operation. 60 | GeneratePresignedURL(blob *models.SimpleBlob) (string, error) 61 | } 62 | -------------------------------------------------------------------------------- /azurecopy/handlers/DropboxHandler.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "azurecopy/azurecopy/models" 5 | "azurecopy/azurecopy/utils/blobutils" 6 | "azurecopy/azurecopy/utils/containerutils" 7 | "azurecopy/azurecopy/utils/helpers" 8 | "errors" 9 | "fmt" 10 | "io" 11 | "io/ioutil" 12 | "os" 13 | "path" 14 | "regexp" 15 | "strings" 16 | "bytes" 17 | log "github.com/Sirupsen/logrus" 18 | "github.com/dropbox/dropbox-sdk-go-unofficial/dropbox" 19 | "github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/files" 20 | "time" 21 | ) 22 | 23 | type DropboxHandler struct { 24 | 25 | // determine if we're caching the blob to disk during copy operations. 26 | // or if we're keeping it in memory 27 | cacheToDisk bool 28 | cacheLocation string 29 | 30 | // is this handler for the source or dest? 31 | IsSource bool 32 | } 33 | 34 | var config *dropbox.Config 35 | 36 | // NewDropboxHandler factory to create new one. Evil? 37 | func NewDropboxHandler(isSource bool, cacheToDisk bool) (*DropboxHandler, error) { 38 | 39 | dh := new(DropboxHandler) 40 | dh.cacheToDisk = cacheToDisk 41 | dir, err := ioutil.TempDir("", "azurecopy") 42 | if err != nil { 43 | log.Fatalf("Unable to create temp directory %s", err) 44 | } 45 | 46 | dh.cacheLocation = dir 47 | dh.IsSource = isSource 48 | 49 | config, err = helpers.SetupConnection() 50 | if err != nil { 51 | log.Fatalf("Unable to setup dropbox %s", err) 52 | } 53 | 54 | return dh, nil 55 | } 56 | 57 | // GetRootContainer gets root container of S3. Gets the list of buckets and THOSE are the immediate child containers here. 58 | func (dh *DropboxHandler) GetRootContainer() models.SimpleContainer { 59 | container := models.SimpleContainer{} 60 | dbx := files.New(*config) 61 | arg := files.NewListFolderArg("") 62 | 63 | res, err := dbx.ListFolder(arg) 64 | if err != nil { 65 | log.Fatalf("Dropbox::GetRootContainer error %s", err) 66 | } 67 | 68 | log.Debugf("results are %s", res) 69 | return container 70 | } 71 | 72 | // BlobExists checks if blob exists 73 | func (dh *DropboxHandler) BlobExists(container models.SimpleContainer, blobName string) (bool, error) { 74 | return false, nil 75 | } 76 | 77 | // GetContainerContentsOverChannel given a URL (ending in /) returns all the contents of the container over a channel 78 | // This returns a COPY of the original source container but has been populated with *some* of the blobs/subcontainers in it. 79 | // The way the dropbox code is written the sourceContainer should actually be holding ALL the blobs already so there is nothing to do except 80 | // push to channel. 81 | // 82 | // This might be an issue later with massive number of dropbox blobs. FIXME(kpfaulkner) 83 | func (dh *DropboxHandler) GetContainerContentsOverChannel(sourceContainer models.SimpleContainer, blobChannel chan models.SimpleContainer) error { 84 | 85 | log.Debugf("dropbox::GetContainerContentsOverChannel container %s", sourceContainer.Name) 86 | defer close(blobChannel) 87 | 88 | sourceContainer.DisplayContainer("") 89 | 90 | blobChannel <- sourceContainer 91 | 92 | // have deleted a bunch of code here... no longer required, but check history if something 93 | // seems off. 94 | 95 | return nil 96 | } 97 | 98 | // GetSpecificSimpleContainer returns the DEEPEST container. eg. if the url is ...../vdir1/vdir2/vdir3 then the simplecontainer returned 99 | // is vdir3 100 | // GetSpecificSimpleContainer given a URL (ending in /) then get the SIMPLE container that represents it. 101 | // For DROPBOX the simplecontainer will be fully populated with the blobs/subcontainers. 102 | func (dh *DropboxHandler) GetSpecificSimpleContainer(URL string) (*models.SimpleContainer, error) { 103 | 104 | log.Debugf("DB: GetSpecificSimpleContainer url %s", URL) 105 | dbx := files.New(*config) 106 | dirArg := dh.getDirArg(URL) 107 | log.Debugf("DirArg is %s", dirArg) 108 | 109 | arg := files.NewListFolderArg(dirArg) 110 | 111 | // loop manually so we can track parent containers etc!!!!!!!!!!!!! DUH 112 | arg.Recursive = true 113 | 114 | res, err := dbx.ListFolder(arg) 115 | if err != nil { 116 | log.Fatalf("Dropbox::GetRootContainer error %s", err) 117 | } 118 | 119 | container := models.SimpleContainer{} 120 | container.Name = "" // getLastSegmentOfPath(dirArg). This is the root? 121 | processEntries(res, dirArg, &container) 122 | 123 | for res.HasMore { 124 | arg := files.NewListFolderContinueArg(res.Cursor) 125 | 126 | res, err = dbx.ListFolderContinue(arg) 127 | if err != nil { 128 | log.Debugf("ListFolderContinue err: %s", err) 129 | return nil, err 130 | } 131 | 132 | processEntries(res, dirArg, &container) 133 | } 134 | 135 | // get the container we're actually after. 136 | wantedContainer, err := filterContainer(&container, dirArg) 137 | if err != nil { 138 | log.Debugf("filterContainer returned error %s", err) 139 | return nil, err 140 | } 141 | 142 | log.Debugf("Dropbox::GetSpecificSimpleContainer returns container %s", wantedContainer.Name) 143 | return wantedContainer, nil 144 | } 145 | 146 | // filterContainer gets the container we're after by checking dirArg 147 | // 148 | // ie rootContainer is literally the root, but maybe we were after /temp/dir1/dir2/ so we return the 149 | // subcontainer referencing dir2 150 | func filterContainer(rootContainer *models.SimpleContainer, dirArg string) (*models.SimpleContainer, error) { 151 | 152 | log.Debugf("filter container %s", dirArg) 153 | sp := strings.Split(dirArg, "/") 154 | 155 | log.Debugf("rootContainer has %d sub containers", len( rootContainer.ContainerSlice)) 156 | container := rootContainer 157 | for _, dir := range sp { 158 | if dir != "" { 159 | log.Debugf("checking %s", dir) 160 | log.Debugf("container %s has %d subcontainers", container.Name, len(container.ContainerSlice)) 161 | var childContainer *models.SimpleContainer 162 | 163 | foundChild := false 164 | // check children. 165 | for _, childContainer = range container.ContainerSlice { 166 | log.Debugf("comparing against %s", childContainer.Name) 167 | if strings.ToLower(childContainer.Name) == strings.ToLower(dir) { 168 | // found what we want. 169 | foundChild = true 170 | 171 | log.Debugf("FOUND container %s", dir) 172 | break 173 | } 174 | } 175 | 176 | if foundChild { 177 | container = childContainer 178 | } else { 179 | // haven't found what we want. Return error 180 | return nil, errors.New("Unable to find container") 181 | } 182 | } 183 | 184 | } 185 | 186 | return container, nil 187 | 188 | } 189 | 190 | // getSubContainer gets an existing subcontainer with parent of container and name of segment. 191 | // otherwise it creates it, adds it to the parent container and returns the new one. 192 | func (dh *DropboxHandler) getSubContainer(container *models.SimpleContainer, segment string) *models.SimpleContainer { 193 | 194 | // MUST be a shorthand way of doing this. But still crawling in GO. 195 | for _, c := range container.ContainerSlice { 196 | if c.Name == segment { 197 | return c 198 | } 199 | } 200 | 201 | // create a new one. 202 | newContainer := models.SimpleContainer{} 203 | newContainer.Name = segment 204 | newContainer.Origin = container.Origin 205 | newContainer.ParentContainer = container 206 | container.ContainerSlice = append(container.ContainerSlice, &newContainer) 207 | return &newContainer 208 | } 209 | 210 | func getLastSegmentOfPath(path string) string { 211 | 212 | log.Debugf("getLastSegmentOfPath %s", path) 213 | if strings.HasSuffix(path, "/") { 214 | path = strings.TrimSuffix(path, "/") 215 | } 216 | 217 | sp := strings.Split(path, "/") 218 | 219 | log.Debugf("split path %s", sp) 220 | return sp[len(sp)-1] 221 | 222 | } 223 | 224 | func trimContainerName(containerName string) string { 225 | 226 | path := containerName 227 | if strings.HasPrefix(path, "/") { 228 | path = strings.TrimPrefix(path, "/") 229 | } 230 | 231 | if strings.HasSuffix(path, "/") { 232 | path = strings.TrimSuffix(path, "/") 233 | } 234 | 235 | return path 236 | } 237 | 238 | func processEntries(results *files.ListFolderResult, dirArg string, rootContainer *models.SimpleContainer) { 239 | log.Debugf("processEntries sourceContainer %s", rootContainer.Name) 240 | for _, i := range results.Entries { 241 | switch f := i.(type) { 242 | case *files.FileMetadata: 243 | log.Debugf("DB is file %s", f.PathDisplay) 244 | blob := models.SimpleBlob{} 245 | blob.Name = f.Name 246 | //blob.URL = fmt.Sprintf("https://www.dropbox.com%s", f.PathDisplay) // NOT A REAL URL.... do we need it? 247 | blob.URL = f.PathDisplay // NOT A REAL URL.... do we need it? 248 | blob.Origin = models.DropBox 249 | 250 | // adds to appropriate container. Will create intermediate containers if required. 251 | addToContainer(&blob, f.PathDisplay, rootContainer) 252 | 253 | //blob.ParentContainer = container 254 | //container.BlobSlice = append(container.BlobSlice, &blob) 255 | log.Debugf("FILE %s", f.Name) 256 | log.Debugf("path display %s", f.PathDisplay) 257 | break 258 | 259 | // folder (real folder)... create simplecontainer and populate? 260 | // might not be needed... since addToContainer (for files) should create the intermediate 261 | // containers as it goes. (I think) 262 | // Will only be missed for empty directories, which I can live with. 263 | // REALLY DONT CARE ABOUT THIS! 264 | case *files.FolderMetadata: 265 | log.Debugf("FOLDER %s", f.Name) 266 | addSubContainer( f.PathDisplay, rootContainer) 267 | break 268 | } 269 | 270 | } 271 | 272 | } 273 | 274 | // addToContainer adds the blob to the rootContainer but will make appropriate child containers if required. 275 | func addToContainer(blob *models.SimpleBlob, path string, rootContainer *models.SimpleContainer) { 276 | 277 | sp := strings.Split(path, "/") 278 | 279 | // just 1 length so member of root container. 280 | if len(sp) == 1 { 281 | rootContainer.BlobSlice = append(rootContainer.BlobSlice, blob) 282 | return 283 | } 284 | 285 | parentContainer := rootContainer 286 | 287 | for i := 0; i < len(sp)-1; i++ { 288 | segment := sp[i] 289 | log.Debugf("Container segment :%s:", segment) 290 | 291 | // dont want to add root container... already have it! 292 | if segment != "" { 293 | container := containerutils.GetContainerByName(parentContainer, segment) 294 | parentContainer = container 295 | } 296 | 297 | } 298 | 299 | // now add blob to parentContainer 300 | parentContainer.BlobSlice = append(parentContainer.BlobSlice, blob) 301 | } 302 | 303 | // addSubContainer adds the subcontainer(s) to the root container. 304 | func addSubContainer(path string, rootContainer *models.SimpleContainer) { 305 | 306 | sp := strings.Split(path, "/") 307 | 308 | parentContainer := rootContainer 309 | 310 | for i := 0; i < len(sp); i++ { 311 | segment := sp[i] 312 | log.Debugf("Container segment :%s:", segment) 313 | 314 | // dont want to add root container... already have it! 315 | if segment != "" { 316 | container := containerutils.GetContainerByName(parentContainer, segment) 317 | parentContainer = container 318 | } 319 | 320 | } 321 | } 322 | 323 | 324 | // getDirArg gets the directory argument for listing contents. 325 | func (dh *DropboxHandler) getDirArg(URL string) string { 326 | lowerURL := strings.ToLower(URL) 327 | 328 | pruneCount := 0 329 | match, _ := regexp.MatchString("http://", lowerURL) 330 | if match { 331 | pruneCount = 7 332 | } 333 | 334 | match, _ = regexp.MatchString("https://", lowerURL) 335 | if match { 336 | pruneCount = 8 337 | } 338 | 339 | // trim protocol 340 | lowerURL = lowerURL[pruneCount:] 341 | sp := strings.Split(lowerURL, "/") 342 | 343 | dirPrefix := "/" + strings.Join(sp[1:], "/") 344 | 345 | if dirPrefix == "/" { 346 | return "" 347 | } 348 | 349 | return dirPrefix 350 | } 351 | 352 | // GetSpecificSimpleBlob given a URL (NOT ending in /) then get the SIMPLE blob that represents it. 353 | func (dh *DropboxHandler) GetSpecificSimpleBlob(URL string) (*models.SimpleBlob, error) { 354 | 355 | return nil, nil 356 | } 357 | 358 | // ReadBlob reads a blob of a given name from a particular SimpleContainer and returns the SimpleBlob 359 | // The SimpleContainer is NOT necessarily a direct mapping to an Azure container but may be representing a virtual directory. 360 | // ie we might have RootSimpleContainer -> SimpleContainer(myrealcontainer) -> SimpleContainer(vdir1) -> SimpleContainer(vdir2) 361 | // and if the blobName is "myblob" then the REAL underlying Azure structure would be container == "myrealcontainer" 362 | // and the blob name is vdir/vdir2/myblob 363 | func (dh *DropboxHandler) ReadBlob(container models.SimpleContainer, blobName string) models.SimpleBlob { 364 | var blob models.SimpleBlob 365 | 366 | return blob 367 | } 368 | 369 | // PopulateBlob. Used to read a blob IFF we already have a reference to it. 370 | func (dh *DropboxHandler) PopulateBlob(blob *models.SimpleBlob) error { 371 | log.Debugf("populateblob %s", blob.Name) 372 | 373 | dbx := files.New(*config) 374 | arg := files.NewDownloadArg(blob.URL) 375 | log.Debugf("DB URL to download %s", blob.URL) 376 | res, contents, err := dbx.Download(arg) 377 | 378 | //if err != nil { 379 | // log.Errorf("DB Cannot download blob %s, %s", blob.URL, err) 380 | // return err 381 | // } 382 | 383 | log.Debugf("res %s", res) 384 | log.Debugf("contents %s", contents) 385 | 386 | err = blobutils.ReadBlob(contents, blob, dh.cacheToDisk, dh.cacheLocation) 387 | if err != nil { 388 | log.Errorf("Error reading Dropbox blob %s", err) 389 | return err 390 | } 391 | 392 | return nil 393 | } 394 | 395 | func (dh *DropboxHandler) WriteContainer(sourceContainer *models.SimpleContainer, destContainer *models.SimpleContainer) error { 396 | return nil 397 | } 398 | 399 | // generateDestDir returns a directory path for DB... but does NOT include the final blobname 400 | // ie prune it off from sourceBlob 401 | func generateDestDir( destContainer *models.SimpleContainer, sourceBlob *models.SimpleBlob) string { 402 | dirSlice := make([]string, 5) 403 | 404 | dirSlice = append(dirSlice, destContainer.Name ) 405 | 406 | container := destContainer 407 | done := false 408 | for done != true { 409 | if container.ParentContainer != nil && container.ParentContainer.Name != "" { 410 | 411 | // yes, prepending by appending... 412 | // I blame Go not having a reverse function :) 413 | dirSlice = append([]string{ container.ParentContainer.Name}, dirSlice...) 414 | container = container.ParentContainer 415 | } else { 416 | done = true 417 | } 418 | } 419 | 420 | dir := path.Join(dirSlice...) 421 | //sp := strings.Split(sourceBlob.Name, "/") 422 | //dir2 := path.Join(sp[:len(sp)-1]...) 423 | 424 | // get path portion of 425 | //return "/"+dir + "/" + dir2 +"/" 426 | return "/"+dir+"/" 427 | } 428 | 429 | // WriteBlob writes a blob to an Azure container. 430 | // The SimpleContainer is NOT necessarily a direct mapping to an Azure container but may be representing a virtual directory. 431 | // ie we might have RootSimpleContainer -> SimpleContainer(myrealcontainer) -> SimpleContainer(vdir1) -> SimpleContainer(vdir2) 432 | // and if the blobName is "myblob" then the REAL underlying Azure structure would be container == "myrealcontainer" 433 | // and the blob name is vdir/vdir2/myblob 434 | func (dh *DropboxHandler) WriteBlob(destContainer *models.SimpleContainer, sourceBlob *models.SimpleBlob) error { 435 | log.Debugf("DB: should be writing blobs!!") 436 | 437 | log.Debugf("DB: dest container is %s", destContainer.Name) 438 | log.Debugf("DB: destcontainer parent is %s", destContainer.ParentContainer.Name) 439 | 440 | destDir := generateDestDir( destContainer, sourceBlob) 441 | 442 | log.Debugf("DEST DIR is %s", destDir) 443 | dbx := files.New(*config) 444 | dst := destDir + sourceBlob.Name 445 | 446 | log.Debugf("db: full dest path %s", dst) 447 | commitInfo := files.NewCommitInfo(dst) 448 | commitInfo.Mode.Tag = "overwrite" 449 | commitInfo.ClientModified = time.Now().UTC().Round(time.Second) 450 | 451 | 452 | // if cached to disk we should probably upload in chunked matter. 453 | // will figure that out later. TODO(kpfaulkner) 454 | if dh.cacheToDisk { 455 | cacheFile, err := os.OpenFile(sourceBlob.DataCachedAtPath, os.O_RDONLY, 0) 456 | if err != nil { 457 | log.Fatal(err) 458 | return err 459 | } 460 | defer cacheFile.Close() 461 | s, err := cacheFile.Stat() 462 | if err != nil { 463 | log.Fatal(err) 464 | return err 465 | } 466 | dh.uploadChunked(dbx, cacheFile, commitInfo, s.Size()) 467 | } else { 468 | fileBytes := bytes.NewReader(sourceBlob.DataInMemory) // convert to io.ReadSeeker type 469 | dh.uploadChunked(dbx, fileBytes, commitInfo, int64(len(sourceBlob.DataInMemory))) 470 | } 471 | 472 | return nil 473 | } 474 | 475 | // uploadChunked upload to dropbox in a chunked manner (for >150M files). 476 | // Heavily inspired by the Dropbox code in dbxcli demo program. 477 | func (dh *DropboxHandler) uploadChunked(dbx files.Client, r io.Reader, commitInfo *files.CommitInfo, sizeTotal int64) (err error) { 478 | 479 | chunkSize := int64(1024*1024*150) // 150M 480 | 481 | if sizeTotal < chunkSize { 482 | chunkSize = sizeTotal 483 | } 484 | 485 | res, err := dbx.UploadSessionStart(files.NewUploadSessionStartArg(), 486 | &io.LimitedReader{R: r, N: chunkSize}) 487 | if err != nil { 488 | fmt.Printf("error is %s\n", err) 489 | return err 490 | } 491 | 492 | written := chunkSize 493 | 494 | for (sizeTotal - written) > chunkSize { 495 | cursor := files.NewUploadSessionCursor(res.SessionId, uint64(written)) 496 | args := files.NewUploadSessionAppendArg(cursor) 497 | 498 | err = dbx.UploadSessionAppendV2(args, &io.LimitedReader{R: r, N: chunkSize}) 499 | if err != nil { 500 | return 501 | } 502 | written += chunkSize 503 | } 504 | 505 | cursor := files.NewUploadSessionCursor(res.SessionId, uint64(written)) 506 | args := files.NewUploadSessionFinishArg(cursor, commitInfo) 507 | 508 | if _, err = dbx.UploadSessionFinish(args, r); err != nil { 509 | return 510 | } 511 | 512 | return 513 | } 514 | 515 | func (dh *DropboxHandler) CreateContainer(containerName string) (models.SimpleContainer, error) { 516 | var container models.SimpleContainer 517 | 518 | return container, nil 519 | } 520 | 521 | // GetContainer gets a container. Populating the subtree? OR NOT? hmmmm 522 | func (dh *DropboxHandler) GetContainer(containerName string) models.SimpleContainer { 523 | var container models.SimpleContainer 524 | 525 | return container 526 | } 527 | 528 | // GetContainerContents populates the passed container with the real contents. 529 | // Can determine if the SimpleContainer is a real container or something virtual. 530 | // We need to trace back to the root node and determine what is really a container and 531 | // what is a blob. 532 | // need to populate.... for Dropbox GetSpecificSimpleContainer is doing too much already! 533 | func (dh *DropboxHandler) GetContainerContents(container *models.SimpleContainer) error { 534 | 535 | return nil 536 | } 537 | 538 | /* presign URL code..... use it eventually. 539 | */ 540 | 541 | func (dh *DropboxHandler) GeneratePresignedURL(blob *models.SimpleBlob) (string, error) { 542 | 543 | return "", nil 544 | } 545 | -------------------------------------------------------------------------------- /azurecopy/handlers/FTPHandler.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "azurecopy/azurecopy/models" 5 | "azurecopy/azurecopy/utils/containerutils" 6 | "azurecopy/azurecopy/utils/misc" 7 | "io" 8 | "io/ioutil" 9 | "path/filepath" 10 | "strings" 11 | "errors" 12 | "os" 13 | "bytes" 14 | "time" 15 | 16 | log "github.com/Sirupsen/logrus" 17 | "github.com/jlaffaye/ftp" 18 | ) 19 | 20 | // FTPHandler basic data structure for FTP handling. 21 | type FTPHandler struct { 22 | 23 | // root directory we're dealing with. 24 | // This is the prefix for any containers. For example, if we're after c:\temp\data\s3\ then the rootContainerPath is really c:\temp\data\ 25 | // and the container used later will be s3. Need to revisit this structure later if gets too confusing. 26 | rootContainerPath string 27 | 28 | // basePath used for prefix. 29 | basePath string 30 | 31 | // container in URL 32 | container string 33 | 34 | // is this source or dest handler? 35 | IsSource bool 36 | 37 | // client connection 38 | client *ftp.ServerConn 39 | 40 | // determine if we're caching the blob to disk during copy operations. 41 | // or if we're keeping it in memory 42 | cacheToDisk bool 43 | cacheLocation string 44 | 45 | } 46 | 47 | // NewFTPHandler factory to create new one. Evil? 48 | func NewFTPHandler(address string, username string, password string, isSource bool, cacheToDisk bool) (*FTPHandler, error) { 49 | 50 | fh := new(FTPHandler) 51 | fh.IsSource = isSource 52 | 53 | client, err := ftp.Dial(address) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | if err := client.Login(username, password); err != nil { 59 | return nil, err 60 | } 61 | 62 | fh.client = client 63 | fh.cacheToDisk = cacheToDisk 64 | 65 | dir, err := ioutil.TempDir("", "azurecopy") 66 | if err != nil { 67 | log.Fatalf("Unable to create temp directory %s", err) 68 | } 69 | 70 | fh.cacheLocation = dir 71 | 72 | return fh, nil 73 | } 74 | 75 | // gets root container. This will get containers/blobs in this container 76 | // NOT recursive. 77 | func (fh *FTPHandler) GetRootContainer() models.SimpleContainer { 78 | //entries, _ := fh.client.List("") 79 | 80 | return models.SimpleContainer{} 81 | } 82 | 83 | // create container. 84 | func (fh *FTPHandler) CreateContainer(containerName string) (models.SimpleContainer, error) { 85 | return models.SimpleContainer{}, nil 86 | } 87 | 88 | // get base path and container name 89 | // path will be something like myftp.com/dir1/dir2/mydir3 90 | // will return /dir1/dir2 and /mydir3/ 91 | func getFTPContainerNameFromURL(url string) (string, string) { 92 | 93 | log.Debugf("rootContainerPath %s", url) 94 | if url != "" { 95 | var sp = strings.Split(url, "/") 96 | l := len(sp) 97 | 98 | log.Debugf("sp is %s", sp) 99 | genPath := strings.Join(sp[:l-2], "/") + "/" 100 | container := sp[l-2] 101 | 102 | return genPath, container 103 | } 104 | 105 | // wasn't passed, so return nada 106 | return "", "" 107 | } 108 | 109 | // GetSpecificSimpleContainer given a URL (ending in /) then get the SIMPLE container that represents it. 110 | // does not have to have all blobs populated in it. Those can be retrieved later via GetContainerContentsOverChannel 111 | // This is up to specific handlers. Currently (for example). For FTP if the url is myftpsite.com/dir1/dir2/dir3/ then it 112 | // will return a SimpleContainer representing dir3 with all its contents.6 113 | func (fh *FTPHandler) GetSpecificSimpleContainer(URL string) (*models.SimpleContainer, error) { 114 | if URL != "" { 115 | 116 | // check if its a container. 117 | if isContainer(URL) { 118 | 119 | var sp= strings.Split(URL, "/") 120 | parentContainer := models.NewSimpleContainer() 121 | parentContainer.IsRootContainer = true 122 | parentContainer.Origin = models.FTP 123 | currentContainer := parentContainer 124 | for _, segment := range sp[1:] { 125 | container := models.NewSimpleContainer() 126 | container.URL = URL 127 | container.Origin = models.FTP 128 | container.Name = segment 129 | container.IsRootContainer = false 130 | container.ParentContainer = currentContainer 131 | currentContainer.ContainerSlice = append(currentContainer.ContainerSlice, container) 132 | 133 | currentContainer = container 134 | log.Debugf("segment is %s\n", segment) 135 | } 136 | 137 | return currentContainer, nil 138 | } 139 | } 140 | 141 | return nil, errors.New("URL cannot be empty") 142 | } 143 | 144 | // GetContainerContents populates the container (directory) with the next level contents 145 | // currently wont do recursive. 146 | func (fh *FTPHandler) GetContainerContents(sourceContainer *models.SimpleContainer) error { 147 | 148 | /* WIP!!! 149 | fullPath := fh.generateFullPath(sourceContainer) 150 | 151 | entryList, err := fh.client.List( fullPath) 152 | 153 | // loop through until all directories done.... 154 | done := false 155 | for done == false { 156 | // copy of container, dont want to send back ever growing container via the channel. 157 | containerClone := sourceContainer 158 | 159 | //azureContainer := ah.blobStorageClient.GetContainerReference(azureContainer.Name) 160 | blobListResponse, err := containerURL.ListBlobs( ctx, marker, storage.ListBlobsOptions{ Prefix: blobPrefix}) 161 | if err != nil { 162 | log.Fatal("Error") 163 | } 164 | 165 | ah.populateSimpleContainer(blobListResponse, &containerClone, blobPrefix) 166 | 167 | // return entire container via channel. 168 | blobChannel <- containerClone 169 | 170 | // if marker, then keep going. 171 | if blobListResponse.NextMarker.NotDone() { 172 | marker = blobListResponse.NextMarker 173 | } else { 174 | done = true 175 | } 176 | } 177 | 178 | close(blobChannel) 179 | 180 | */ 181 | return nil 182 | 183 | } 184 | 185 | // GetContainerContentsOverChannel given a URL (ending in /) returns all the contents of the container over a channel 186 | // GetContainerContentsOverChannel given a URL (ending in /) returns all the contents of the container over a channel 187 | // This returns a COPY of the original source container but has been populated with *some* of the blobs/subcontainers in it. 188 | // This is going to be inefficient from a memory allocation pov. 189 | // Am still creating various structs that we strictly do not require for copying (all the tree structure etc) but this will 190 | // at least help each cloud provider be consistent from a dev pov. Think it's worth the overhead. TODO(kpfaulkner) confirm :) 191 | func (fh *FTPHandler) GetContainerContentsOverChannel(sourceContainer models.SimpleContainer, blobChannel chan models.SimpleContainer) error { 192 | 193 | defer close(blobChannel) 194 | // just do it in bulk for FS. Figure out later if its an issue. 195 | fh.GetContainerContents(&sourceContainer) 196 | blobChannel <- sourceContainer 197 | 198 | return nil 199 | } 200 | 201 | // GetSpecificSimpleBlob given a URL (NOT ending in /) then get the SIMPLE blob that represents it. 202 | // The DestName will be the last element of the URL, whether it's a real blobname or not. 203 | // eg. https://...../mycontainer/vdir1/vdir2/blobname will return a DestName of "blobname" even though strictly 204 | // speaking the true blobname is "vdir1/vdir2/blobname". 205 | // Will revisit this if it causes a problem. 206 | func (fh *FTPHandler) GetSpecificSimpleBlob(URL string) (*models.SimpleBlob, error) { 207 | 208 | } 209 | 210 | // dupe of filesystem. Need to check if can just use single method instance. 211 | func (fh *FTPHandler) generateFullPath(container *models.SimpleContainer) string { 212 | 213 | path := container.Name 214 | currentContainer := container.ParentContainer 215 | for currentContainer != nil { 216 | if currentContainer.Name != "" { 217 | path = filepath.Join(currentContainer.Name, path) 218 | } 219 | 220 | currentContainer = currentContainer.ParentContainer 221 | } 222 | 223 | fullPath := fh.basePath + path + "/" 224 | // if full path is rootContainerPath then we need to actually generate 225 | return fullPath 226 | } 227 | 228 | func reverseStringSlice( s []string ) []string{ 229 | for i := len(s)/2-1; i >= 0; i-- { 230 | opp := len(s)-1-i 231 | s[i], s[opp] = s[opp], s[i] 232 | } 233 | 234 | return s 235 | } 236 | // generate complete path of blob 237 | func (fh *FTPHandler) generateBlobFullPath(blob *models.SimpleBlob) string { 238 | 239 | nameElements := []string{} 240 | nameElements = append(nameElements, blob.DestName) 241 | 242 | currentContainer := blob.ParentContainer 243 | for currentContainer != nil { 244 | if currentContainer.Name != "" { 245 | nameElements = append(nameElements, currentContainer.Name) 246 | } 247 | currentContainer = currentContainer.ParentContainer 248 | } 249 | 250 | nameElements = reverseStringSlice( nameElements) 251 | fullPath := strings.Join(nameElements, "/") 252 | return fullPath 253 | } 254 | 255 | 256 | // Given a container and a blob name, read the blob. 257 | func (fh *FTPHandler) ReadBlob(container models.SimpleContainer, blobName string) models.SimpleBlob { 258 | var blob models.SimpleBlob 259 | 260 | dirPath := fh.generateFullPath(&container) 261 | fullPath := filepath.Join(dirPath, blobName) 262 | 263 | // populate this to disk. 264 | if fh.cacheToDisk { 265 | 266 | cacheName := misc.GenerateCacheName(blob.BlobCloudName) 267 | blob.DataCachedAtPath = fh.cacheLocation + "/" + cacheName 268 | log.Debugf("azure blob %s cached at location %s", blob.BlobCloudName, blob.DataCachedAtPath) 269 | } 270 | 271 | r, err := fh.client.Retr(fullPath) 272 | if err != nil { 273 | log.Fatal(err) 274 | } else { 275 | 276 | if fh.cacheToDisk { 277 | // read directly into cached file 278 | cacheFile, err := os.OpenFile(blob.DataCachedAtPath, os.O_WRONLY|os.O_CREATE, 0666) 279 | defer cacheFile.Close() 280 | if err != nil { 281 | log.Fatalf("Populate blob %s", err) 282 | } 283 | _, err = io.Copy(cacheFile, r) 284 | blob.BlobInMemory = false 285 | 286 | } else { 287 | buf, err := ioutil.ReadAll(r) 288 | if err != nil { 289 | log.Fatal(err) 290 | } 291 | r.Close() // test we can close two times 292 | 293 | blob.DataInMemory = buf 294 | blob.BlobInMemory = true 295 | } 296 | } 297 | 298 | blob.DataCachedAtPath = fullPath 299 | blob.Name = blobName 300 | blob.ParentContainer = &container 301 | blob.Origin = container.Origin 302 | blob.URL = fullPath 303 | return blob 304 | } 305 | 306 | // Does blob exist 307 | // question if error should be returned? 308 | func (fh *FTPHandler) BlobExists(container models.SimpleContainer, blobName string) (bool, error) { 309 | dirPath := fh.generateFullPath(&container) 310 | fullPath := filepath.Join(dirPath, blobName) 311 | 312 | _, err := fh.client.FileSize( fullPath) 313 | if err != nil { 314 | return false, nil 315 | } 316 | 317 | return true, nil 318 | 319 | } 320 | 321 | // if we already have a reference to a SimpleBlob, then read it and populate it. 322 | // ie we're populating our in process copy of the blob (ie reading it from the provider). 323 | func (fh *FTPHandler) PopulateBlob(blob *models.SimpleBlob) error { 324 | fullPath := fh.generateBlobFullPath( blob) 325 | 326 | // populate this to disk. 327 | if fh.cacheToDisk { 328 | 329 | cacheName := misc.GenerateCacheName(blob.BlobCloudName) 330 | blob.DataCachedAtPath = fh.cacheLocation + "/" + cacheName 331 | log.Debugf("azure blob %s cached at location %s", blob.BlobCloudName, blob.DataCachedAtPath) 332 | } 333 | 334 | r, err := fh.client.Retr(fullPath) 335 | if err != nil { 336 | log.Fatal(err) 337 | } else { 338 | 339 | if fh.cacheToDisk { 340 | // read directly into cached file 341 | cacheFile, err := os.OpenFile(blob.DataCachedAtPath, os.O_WRONLY|os.O_CREATE, 0666) 342 | defer cacheFile.Close() 343 | if err != nil { 344 | log.Fatalf("Populate blob %s", err) 345 | } 346 | _, err = io.Copy(cacheFile, r) 347 | blob.BlobInMemory = false 348 | 349 | } else { 350 | buf, err := ioutil.ReadAll(r) 351 | if err != nil { 352 | log.Fatal(err) 353 | } 354 | r.Close() // test we can close two times 355 | 356 | blob.DataInMemory = buf 357 | blob.BlobInMemory = true 358 | } 359 | } 360 | 361 | blob.DataCachedAtPath = fullPath 362 | blob.URL = fullPath 363 | 364 | return nil 365 | } 366 | 367 | func (fh *FTPHandler) createSubDirectories(fullPath string) error { 368 | err := fh.client.MakeDir(fullPath) 369 | if err != nil { 370 | return err 371 | } 372 | 373 | return nil 374 | } 375 | 376 | // given a container and blob, write blob. 377 | func (fh *FTPHandler) WriteBlob(destContainer *models.SimpleContainer, sourceBlob *models.SimpleBlob) error { 378 | blobName := sourceBlob.Name 379 | if blobName[0] == os.PathSeparator { 380 | blobName = blobName[1:] 381 | } 382 | 383 | fullPath := fh.generateFullPath(destContainer) + blobName 384 | 385 | // make sure subdirs are created. 386 | err := fh.createSubDirectories(fullPath) 387 | if err != nil { 388 | log.Fatal(err) 389 | return err 390 | } 391 | 392 | if !sourceBlob.BlobInMemory { 393 | // cached on disk. 394 | newFile, err := os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE, 0) 395 | if err != nil { 396 | return err 397 | } 398 | err = fh.client.Stor(fullPath, newFile ) 399 | if err != nil { 400 | return err 401 | } 402 | } else { 403 | // in memory. 404 | err = fh.client.Stor( fullPath,bytes.NewReader( sourceBlob.DataInMemory) ) 405 | if err != nil { 406 | log.Fatal("Unable to upload file " + fullPath, err) 407 | } 408 | } 409 | 410 | return nil 411 | } 412 | 413 | // write a container (and subcontents) to the appropriate data store 414 | func (fh *FTPHandler) WriteContainer(sourceContainer *models.SimpleContainer, destContainer *models.SimpleContainer) error { 415 | return nil 416 | } 417 | 418 | // Gets a container. Populating the subtree? OR NOT? hmmmm 419 | func (fh *FTPHandler) GetContainer(containerName string) models.SimpleContainer { 420 | var container models.SimpleContainer 421 | 422 | return container 423 | } 424 | 425 | // generates presigned URL so Azure can access blob for CopyBlob flag operation. 426 | func (fh *FTPHandler) GeneratePresignedURL(blob *models.SimpleBlob) (string, error) { 427 | return "", nil 428 | 429 | } 430 | -------------------------------------------------------------------------------- /azurecopy/handlers/FilesystemHandler.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "azurecopy/azurecopy/models" 5 | "errors" 6 | "os" 7 | "path" 8 | "path/filepath" 9 | "strings" 10 | 11 | log "github.com/Sirupsen/logrus" 12 | ) 13 | 14 | // FilesystemHandler basic data structure for FS handling. 15 | type FilesystemHandler struct { 16 | 17 | // root directory we're dealing with. 18 | // This is the prefix for any containers. For example, if we're after c:\temp\data\s3\ then the rootContainerPath is really c:\temp\data\ 19 | // and the container used later will be s3. Need to revisit this structure later if gets too confusing. 20 | rootContainerPath string 21 | 22 | // basePath used for prefix. 23 | basePath string 24 | 25 | // container in URL 26 | container string 27 | 28 | // is this source or dest handler? 29 | IsSource bool 30 | } 31 | 32 | // get base path and container name 33 | // assumes using WRONG format. Hmmm will this be cross platform? 34 | func generateBasePath(rootContainerPath string) (string, string) { 35 | 36 | log.Debugf("rootContainerPath %s", rootContainerPath) 37 | if rootContainerPath != "" { 38 | var sp = strings.Split(rootContainerPath, string(os.PathSeparator)) 39 | l := len(sp) 40 | 41 | log.Debugf("sp is %s", sp) 42 | genPath := strings.Join(sp[:l-2], string(os.PathSeparator)) + string(os.PathSeparator) 43 | container := sp[l-2] 44 | 45 | return genPath, container 46 | } 47 | 48 | // wasn't passed, so return nada 49 | return "", "" 50 | } 51 | 52 | // NewFilesystemHandler factory to create new one. Evil? 53 | func NewFilesystemHandler(rootContainerPath string, isSource bool) (*FilesystemHandler, error) { 54 | 55 | fh := new(FilesystemHandler) 56 | fh.basePath, fh.container = generateBasePath(rootContainerPath) 57 | 58 | fh.rootContainerPath = rootContainerPath 59 | fh.IsSource = isSource 60 | 61 | return fh, nil 62 | } 63 | 64 | // GetRootContainer gets root container of Azure. In reality there isn't a root container, but this would basically be a SimpleContainer 65 | // that has the containerSlice populated with the real Azure containers. 66 | func (fh *FilesystemHandler) GetRootContainer() models.SimpleContainer { 67 | 68 | dir, err := os.OpenFile(fh.rootContainerPath, os.O_RDONLY, 0) 69 | if err != nil { 70 | log.Fatal("ERR OpenFile ", err) 71 | } 72 | 73 | fileInfos, err := dir.Readdir(0) 74 | if err != nil { 75 | log.Fatal("ERR ReadDir", err) 76 | } 77 | 78 | rootContainer := models.NewSimpleContainer() 79 | rootContainer.URL = fh.rootContainerPath 80 | rootContainer.Origin = models.Filesystem 81 | rootContainer.Name = fh.container 82 | rootContainer.IsRootContainer = true 83 | 84 | for _, f := range fileInfos { 85 | 86 | // determine if file or directory. 87 | // do we go recursive? 88 | if f.IsDir() { 89 | sc := models.NewSimpleContainer() 90 | sc.Name = f.Name() 91 | sc.Origin = models.Filesystem 92 | sc.ParentContainer = rootContainer 93 | sc.Populated = false 94 | rootContainer.ContainerSlice = append(rootContainer.ContainerSlice, sc) 95 | } else { 96 | b := models.SimpleBlob{} 97 | b.Name = f.Name() 98 | b.ParentContainer = rootContainer 99 | b.Origin = models.Filesystem 100 | rootContainer.BlobSlice = append(rootContainer.BlobSlice, &b) 101 | 102 | } 103 | } 104 | rootContainer.Populated = true 105 | 106 | return *rootContainer 107 | } 108 | 109 | // ReadBlob in theory reads the blob. Given we're already dealing with a local filesystem DO we need to read it at all? 110 | // No point keeping it in memory, local disk is good enough. Also any point making a copy to the cache directory? 111 | // for now, just mark the blob as cached and point to original file dir. 112 | func (fh *FilesystemHandler) ReadBlob(container models.SimpleContainer, blobName string) models.SimpleBlob { 113 | var blob models.SimpleBlob 114 | 115 | dirPath := fh.generateFullPath(&container) 116 | fullPath := filepath.Join(dirPath, blobName) 117 | 118 | blob.DataCachedAtPath = fullPath 119 | blob.BlobInMemory = false 120 | blob.Name = blobName 121 | blob.ParentContainer = &container 122 | blob.Origin = container.Origin 123 | blob.URL = fullPath 124 | return blob 125 | } 126 | 127 | // PopulateBlob. Used to read a blob IFF we already have a reference to it. 128 | func (fh *FilesystemHandler) PopulateBlob(blob *models.SimpleBlob) error { 129 | 130 | blob.DataCachedAtPath = blob.URL 131 | blob.BlobInMemory = false 132 | 133 | return nil 134 | } 135 | 136 | // generateAzureContainerName gets the REAL Azure container name for the simpleBlob 137 | func (fh *FilesystemHandler) generateAzureContainerName(blob *models.SimpleBlob) string { 138 | currentContainer := blob.ParentContainer 139 | return currentContainer.Name 140 | } 141 | 142 | // WriteBlob writes a blob to an Azure container. 143 | // The SimpleContainer is NOT necessarily a direct mapping to an Azure container but may be representiing a virtual directory. 144 | // ie we might have RootSimpleContainer -> SimpleContainer(myrealcontainer) -> SimpleContainer(vdir1) -> SimpleContainer(vdir2) 145 | // and if the blobName is "myblob" then the REAL underlying Azure structure would be container == "myrealcontainer" 146 | // and the blob name is vdir/vdir2/myblob 147 | func (fh *FilesystemHandler) WriteBlob(destContainer *models.SimpleContainer, sourceBlob *models.SimpleBlob) error { 148 | 149 | blobName := sourceBlob.Name 150 | if blobName[0] == os.PathSeparator { 151 | blobName = blobName[1:] 152 | } 153 | 154 | fullPath := fh.generateFullPath(destContainer) + blobName 155 | 156 | // make sure subdirs are created. 157 | err := fh.createSubDirectories(fullPath) 158 | if err != nil { 159 | log.Fatal(err) 160 | return err 161 | } 162 | 163 | if !sourceBlob.BlobInMemory { 164 | err := fh.copyFile(sourceBlob.DataCachedAtPath, fullPath) 165 | 166 | if err != nil { 167 | log.Fatal("FilesystemHandler::WriteBlob err ", err) 168 | } 169 | } else { 170 | // from memory. 171 | newFile, err := os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE, 0) 172 | if err != nil { 173 | log.Fatal("FilesystemHandler::WriteBlob unable to open destination file", err) 174 | } 175 | 176 | var totalBytesWritten int = 0 177 | fileSize := len(sourceBlob.DataInMemory) 178 | 179 | for totalBytesWritten < fileSize { 180 | bytesWritten, err := newFile.Write(sourceBlob.DataInMemory[totalBytesWritten:]) 181 | 182 | if err != nil { 183 | log.Fatal("FilesystemHandler::WriteBlob unable to open destination file", err) 184 | } 185 | totalBytesWritten += bytesWritten 186 | } 187 | } 188 | 189 | return nil 190 | } 191 | 192 | func (fh *FilesystemHandler) createSubDirectories(fullPath string) error { 193 | var dirPath = path.Dir(fullPath) 194 | os.MkdirAll(dirPath, 0777) 195 | 196 | return nil 197 | } 198 | 199 | func (fh *FilesystemHandler) copyFile(sourceFile string, destFile string) error { 200 | cacheFile, err := os.OpenFile(sourceFile, os.O_RDONLY, 0) 201 | if err != nil { 202 | log.Fatal("FilesystemHandler::WriteBlob err ", err) 203 | } 204 | 205 | // location of destination blob. 206 | fullPath := destFile 207 | newFile, err := os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE, 0) 208 | if err != nil { 209 | log.Fatal("FilesystemHandler::WriteBlob unable to open destination file", err) 210 | } 211 | 212 | buffer := make([]byte, 1024*100) 213 | numBytesRead := 0 214 | 215 | finishedProcessing := false 216 | for finishedProcessing == false { 217 | numBytesRead, err = cacheFile.Read(buffer) 218 | if err != nil { 219 | finishedProcessing = true 220 | } 221 | 222 | if numBytesRead <= 0 { 223 | finishedProcessing = true 224 | continue 225 | } 226 | 227 | _, err = newFile.Write(buffer[:numBytesRead]) 228 | if err != nil { 229 | log.Fatal(err) 230 | return err 231 | } 232 | } 233 | 234 | return nil 235 | } 236 | 237 | func (fh *FilesystemHandler) WriteContainer(sourceContainer *models.SimpleContainer, destContainer *models.SimpleContainer) error { 238 | 239 | return nil 240 | } 241 | 242 | func (fh *FilesystemHandler) CreateContainer(containerName string) (models.SimpleContainer, error) { 243 | var container models.SimpleContainer 244 | 245 | return container, nil 246 | } 247 | 248 | // GetContainer gets a container. Populating the subtree? OR NOT? hmmmm 249 | func (fh *FilesystemHandler) GetContainer(containerName string) models.SimpleContainer { 250 | var container models.SimpleContainer 251 | 252 | return container 253 | } 254 | 255 | func (fh *FilesystemHandler) generateFullPath(container *models.SimpleContainer) string { 256 | 257 | path := container.Name 258 | currentContainer := container.ParentContainer 259 | for currentContainer != nil { 260 | if currentContainer.Name != "" { 261 | path = filepath.Join(currentContainer.Name, path) 262 | } 263 | 264 | currentContainer = currentContainer.ParentContainer 265 | } 266 | 267 | fullPath := fh.basePath + path + string(os.PathSeparator) 268 | // if full path is rootContainerPath then we need to actually generate 269 | return fullPath 270 | } 271 | 272 | // GetContainerContents populates the container (directory) with the next level contents 273 | // currently wont do recursive. 274 | func (fh *FilesystemHandler) GetContainerContents(container *models.SimpleContainer) error { 275 | 276 | fullPath := fh.generateFullPath(container) 277 | dir, err := os.OpenFile(fullPath, os.O_RDONLY, 0) 278 | if err != nil { 279 | log.Fatal("ERR OpenFile ", err) 280 | } 281 | 282 | fileInfos, err := dir.Readdir(0) 283 | if err != nil { 284 | log.Fatal("ERR ReadDir", err) 285 | } 286 | 287 | for _, f := range fileInfos { 288 | 289 | // determine if file or directory. 290 | // do we go recursive? 291 | if f.IsDir() { 292 | sc := models.NewSimpleContainer() 293 | sc.Name = f.Name() 294 | sc.Origin = models.Filesystem 295 | sc.ParentContainer = container 296 | sc.Populated = false 297 | sc.IsRootContainer = false 298 | fh.GetContainerContents(sc) 299 | container.ContainerSlice = append(container.ContainerSlice, sc) 300 | 301 | } else { 302 | b := models.SimpleBlob{} 303 | b.Name = f.Name() 304 | b.ParentContainer = container 305 | b.Origin = models.Filesystem 306 | b.URL = filepath.Join(fh.generateFullPath(container), b.Name) 307 | container.BlobSlice = append(container.BlobSlice, &b) 308 | 309 | } 310 | } 311 | container.Populated = true 312 | 313 | return nil 314 | } 315 | 316 | // BlobExists checks if blob already exists 317 | func (fh *FilesystemHandler) BlobExists(container models.SimpleContainer, blobName string) (bool, error) { 318 | 319 | if blobName[0] == os.PathSeparator { 320 | blobName = blobName[1:] 321 | } 322 | 323 | fullPath := fh.generateFullPath(&container) + blobName 324 | 325 | log.Printf("FH %s", fullPath) 326 | 327 | return false, nil 328 | } 329 | 330 | // getSubContainer gets an existing subcontainer with parent of container and name of segment. 331 | // otherwise it creates it, adds it to the parent container and returns the new one. 332 | func (fh *FilesystemHandler) getSubContainer(container *models.SimpleContainer, segment string) *models.SimpleContainer { 333 | // create a new one. 334 | newContainer := models.SimpleContainer{} 335 | return &newContainer 336 | } 337 | 338 | // GetContainerContentsOverChannel given a URL (ending in /) returns all the contents of the container over a channel 339 | // This is going to be inefficient from a memory allocation pov. 340 | // Am still creating various structs that we strictly do not require for copying (all the tree structure etc) but this will 341 | // at least help each cloud provider be consistent from a dev pov. Think it's worth the overhead. TODO(kpfaulkner) confirm :) 342 | func (fh *FilesystemHandler) GetContainerContentsOverChannel(sourceContainer models.SimpleContainer, blobChannel chan models.SimpleContainer) error { 343 | 344 | defer close(blobChannel) 345 | // just do it in bulk for FS. Figure out later if its an issue. 346 | fh.GetContainerContents(&sourceContainer) 347 | blobChannel <- sourceContainer 348 | 349 | return nil 350 | } 351 | 352 | func isContainer(url string) bool { 353 | fi, err := os.Stat(url) 354 | if err != nil { 355 | log.Fatalf("Unable to handle path %s\n", url) 356 | } 357 | return fi.Mode().IsDir() 358 | } 359 | 360 | // GetSpecificSimpleContainer given a URL (ending in /) then get the SIMPLE container that represents it. 361 | // eg. c:\temp\mydir1\mydir2\ 362 | func (fh *FilesystemHandler) GetSpecificSimpleContainer(URL string) (*models.SimpleContainer, error) { 363 | 364 | if URL != "" { 365 | 366 | // check if its a container. 367 | if isContainer(URL) { 368 | 369 | var sp= strings.Split(URL, "/") 370 | parentContainer := models.NewSimpleContainer() 371 | parentContainer.IsRootContainer = true 372 | parentContainer.Origin = models.Filesystem 373 | currentContainer := parentContainer 374 | for _, segment := range sp[1:] { 375 | container := models.NewSimpleContainer() 376 | container.URL = URL 377 | container.Origin = models.Filesystem 378 | container.Name = segment 379 | container.IsRootContainer = false 380 | container.ParentContainer = currentContainer 381 | currentContainer.ContainerSlice = append(currentContainer.ContainerSlice, container) 382 | 383 | currentContainer = container 384 | log.Debugf("segment is %s\n", segment) 385 | } 386 | 387 | return currentContainer, nil 388 | } 389 | } 390 | 391 | return nil, errors.New("URL cannot be empty") 392 | 393 | } 394 | 395 | func (fh *FilesystemHandler) GeneratePresignedURL(blob *models.SimpleBlob) (string, error) { 396 | return "", nil 397 | } 398 | 399 | // GetSpecificSimpleBlob given a URL (NOT ending in /) then get the SIMPLE blob that represents it. 400 | func (fh *FilesystemHandler) GetSpecificSimpleBlob(URL string) (*models.SimpleBlob, error) { 401 | return nil, nil 402 | 403 | } 404 | -------------------------------------------------------------------------------- /azurecopy/handlers/S3Handler.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "azurecopy/azurecopy/models" 5 | "azurecopy/azurecopy/utils/containerutils" 6 | "azurecopy/azurecopy/utils/misc" 7 | "bytes" 8 | "errors" 9 | "fmt" 10 | "io/ioutil" 11 | "os" 12 | "regexp" 13 | "strings" 14 | "time" 15 | 16 | log "github.com/Sirupsen/logrus" 17 | "github.com/aws/aws-sdk-go/aws" 18 | "github.com/aws/aws-sdk-go/aws/credentials" 19 | "github.com/aws/aws-sdk-go/aws/session" 20 | "github.com/aws/aws-sdk-go/service/s3" 21 | ) 22 | 23 | type S3Handler struct { 24 | s3Client *s3.S3 25 | 26 | // determine if we're caching the blob to disk during copy operations. 27 | // or if we're keeping it in memory 28 | cacheToDisk bool 29 | cacheLocation string 30 | 31 | // is this handler for the source or dest? 32 | IsSource bool 33 | } 34 | 35 | // NewS3Handler factory to create new one. Evil? 36 | func NewS3Handler(accessID string, accessSecret string, region string, isSource bool, cacheToDisk bool) (*S3Handler, error) { 37 | 38 | sh := new(S3Handler) 39 | 40 | sh.cacheToDisk = cacheToDisk 41 | dir, err := ioutil.TempDir("", "azurecopy") 42 | if err != nil { 43 | log.Fatalf("Unable to create temp directory %s", err) 44 | } 45 | 46 | sh.cacheLocation = dir 47 | sh.IsSource = isSource 48 | 49 | creds := credentials.NewStaticCredentials(accessID, accessSecret, "") 50 | _, err = creds.Get() 51 | if err != nil { 52 | log.Fatalf("Bad S3 credentials: %s", err) 53 | } 54 | 55 | cfg := aws.NewConfig().WithRegion(region).WithCredentials(creds) 56 | 57 | log.Print(cfg) 58 | sh.s3Client = s3.New(session.New(), cfg) 59 | 60 | return sh, nil 61 | } 62 | 63 | // GetRootContainer gets root container of S3. Gets the list of buckets and THOSE are the immediate child containers here. 64 | func (sh *S3Handler) GetRootContainer() models.SimpleContainer { 65 | result, err := sh.s3Client.ListBuckets(&s3.ListBucketsInput{}) 66 | if err != nil { 67 | log.Fatalf("Unable to get S3 buckets", err) 68 | } 69 | 70 | rootContainer := models.NewSimpleContainer() 71 | 72 | for _, bucket := range result.Buckets { 73 | sc := models.NewSimpleContainer() 74 | sc.Name = *bucket.Name 75 | sc.Origin = models.S3 76 | rootContainer.ContainerSlice = append(rootContainer.ContainerSlice, sc) 77 | } 78 | 79 | return *rootContainer 80 | } 81 | 82 | // BlobExists checks if blob exists 83 | func (sh *S3Handler) BlobExists(container models.SimpleContainer, blobName string) (bool, error) { 84 | // TODO(kpfaulkner) implement me!!! 85 | return false, nil 86 | } 87 | 88 | // convertURL converts from https://bucketname.s3.amazonaws.com/myblob to https://s3.amazonaws.com/bucketname/myblob format 89 | func (sh *S3Handler) convertURL(URL string) string { 90 | // TODO(kpfaulkner) implement me!!! 91 | return URL 92 | } 93 | 94 | // need a more solid way to generate this. 95 | func generateS3URL(key string, containerName string) string { 96 | return fmt.Sprintf("https://%s.s3.amazonaws.com/%s", containerName, key) 97 | } 98 | 99 | 100 | // GetContainerContentsOverChannel given a simpleContainer returns all the contents of the container over a channel 101 | // This returns a COPY of the original source container but has been populated with *some* of the blobs/subcontainers in it. 102 | func (sh *S3Handler) GetContainerContentsOverChannel(sourceContainer models.SimpleContainer, blobChannel chan models.SimpleContainer) error { 103 | 104 | log.Debugf("GetContainerContentsOverChannel source container %s", sourceContainer.Name) 105 | s3Container, blobPrefix := containerutils.GetContainerAndBlobPrefix(&sourceContainer) 106 | 107 | log.Debugf("s3 container %s BlobPrefix %s", s3Container, blobPrefix) 108 | defer close(blobChannel) 109 | 110 | params := s3.ListObjectsV2Input{ 111 | Bucket: &s3Container.Name, 112 | Prefix: &blobPrefix, 113 | } 114 | 115 | err := sh.s3Client.ListObjectsV2Pages(¶ms, 116 | func(page *s3.ListObjectsV2Output, lastPage bool) bool { 117 | // copy of container, dont want to send back ever growing container via the channel. 118 | containerClone := sourceContainer 119 | sh.populateSimpleContainer(page.Contents, &containerClone, blobPrefix) 120 | blobChannel <- containerClone 121 | 122 | return !lastPage 123 | }) 124 | 125 | if err != nil { 126 | return err 127 | } 128 | 129 | return nil 130 | } 131 | 132 | func (sh *S3Handler) getS3Bucket(containerName string) (*models.SimpleContainer, error) { 133 | 134 | rootContainer := sh.GetRootContainer() 135 | for _, container := range rootContainer.ContainerSlice { 136 | if container.Name == containerName { 137 | return container, nil 138 | } 139 | } 140 | 141 | return nil, errors.New("Unable to find container") 142 | } 143 | 144 | func (sh *S3Handler) generateSubContainers(s3Container *models.SimpleContainer, blobPrefix string) (*models.SimpleContainer, *models.SimpleContainer) { 145 | 146 | var containerToReturn *models.SimpleContainer 147 | var lastContainer *models.SimpleContainer 148 | doneFirst := false 149 | 150 | // strip off last / 151 | if len(blobPrefix) > 0 { 152 | blobPrefix = blobPrefix[:len(blobPrefix)-1] 153 | sp := strings.Split(blobPrefix, "/") 154 | 155 | for _, segment := range sp { 156 | container := models.NewSimpleContainer() 157 | container.Name = segment 158 | if !doneFirst { 159 | container.ParentContainer = s3Container 160 | containerToReturn = container 161 | doneFirst = true 162 | } else { 163 | container.ParentContainer = lastContainer 164 | lastContainer.ContainerSlice = append(lastContainer.ContainerSlice, container) 165 | } 166 | 167 | lastContainer = container 168 | } 169 | } else { 170 | 171 | // just return existing container (lastContainer) and no subcontainer (containerToReturn) 172 | containerToReturn = nil 173 | lastContainer = s3Container 174 | } 175 | 176 | return containerToReturn, lastContainer 177 | } 178 | 179 | // GetSpecificSimpleContainer for S3 will be the bucket. 180 | // Conversion from https://bucketname.s3.amazonaws.com/myblob to https://s3.amazonaws.com/bucketname/myblob is done first. 181 | func (sh *S3Handler) GetSpecificSimpleContainer(URL string) (*models.SimpleContainer, error) { 182 | 183 | URL = sh.convertURL(URL) 184 | 185 | lastChar := URL[len(URL)-1:] 186 | // MUST be a better way to get the last character. 187 | if lastChar != "/" { 188 | return nil, errors.New("Needs to end with a /") 189 | } 190 | 191 | containerName, blobPrefix, err := sh.validateURL(URL) 192 | if err != nil { 193 | log.Fatal("GetSpecificSimpleContainer err", err) 194 | } 195 | 196 | log.Debugf("S3 blobprefix %s", blobPrefix) 197 | container, err := sh.getS3Bucket(containerName) 198 | if err != nil { 199 | log.Fatal(err) 200 | } 201 | 202 | subContainer, lastContainer := sh.generateSubContainers(container, blobPrefix) 203 | 204 | if subContainer != nil { 205 | container.ContainerSlice = append(container.ContainerSlice, subContainer) 206 | } 207 | 208 | fmt.Printf("S3 specific container %v\n", lastContainer) 209 | fmt.Printf("S3 specific container name %s\n", lastContainer.Name) 210 | return lastContainer, nil 211 | } 212 | 213 | // validateURL returns container (bucket) Name, blob Name and error 214 | // passes real URL such as https://s3.amazonaws.com/mybucket/myfileprefix/ 215 | func (sh *S3Handler) validateURL(URL string) (string, string, error) { 216 | 217 | lowerURL := strings.ToLower(URL) 218 | 219 | // ugly, do this properly!!! TODO(kpfaulkner) 220 | pruneCount := 0 221 | match, _ := regexp.MatchString("http://", lowerURL) 222 | if match { 223 | pruneCount = 7 224 | } 225 | 226 | match, _ = regexp.MatchString("https://", lowerURL) 227 | if match { 228 | pruneCount = 8 229 | } 230 | 231 | // trim protocol 232 | lowerURL = lowerURL[pruneCount:] 233 | sp := strings.Split(lowerURL, "/") 234 | 235 | containerName := sp[1] 236 | blobName := strings.Join(sp[2:], "/") 237 | 238 | return containerName, blobName, nil 239 | } 240 | 241 | // GetSpecificSimpleBlob given a URL (NOT ending in /) then get the SIMPLE blob that represents it. 242 | // The DestName will be the last element of the URL, whether it's a real blobname or not. 243 | // eg. https://...../mycontainer/vdir1/vdir2/blobname will return a DestName of "blobname" even though strictly 244 | // speaking the true blobname is "vdir1/vdir2/blobname". 245 | // Will revisit this if it causes a problem. 246 | func (sh *S3Handler) GetSpecificSimpleBlob(URL string) (*models.SimpleBlob, error) { 247 | // MUST be a better way to get the last character. 248 | if URL[len(URL)-2:len(URL)-1] == "/" { 249 | return nil, errors.New("Cannot end with a /") 250 | } 251 | 252 | containerName, blobName, err := sh.validateURL(URL) 253 | if err != nil { 254 | log.Fatal("GetSpecificSimpleContainer err", err) 255 | } 256 | 257 | // get parent container (ie this will be the real S3 bucket) 258 | parentContainer, err := sh.getS3Bucket(containerName) 259 | if err != nil { 260 | return nil, err 261 | } 262 | 263 | b := models.SimpleBlob{} 264 | b.Name = blobName 265 | b.Origin = models.S3 266 | b.ParentContainer = parentContainer 267 | b.BlobCloudName = blobName 268 | return &b, nil 269 | } 270 | 271 | // ReadBlob reads a blob of a given name from a particular SimpleContainer and returns the SimpleBlob 272 | // The SimpleContainer is NOT necessarily a direct mapping to an Azure container but may be representing a virtual directory. 273 | // ie we might have RootSimpleContainer -> SimpleContainer(myrealcontainer) -> SimpleContainer(vdir1) -> SimpleContainer(vdir2) 274 | // and if the blobName is "myblob" then the REAL underlying Azure structure would be container == "myrealcontainer" 275 | // and the blob name is vdir/vdir2/myblob 276 | func (sh *S3Handler) ReadBlob(container models.SimpleContainer, blobName string) models.SimpleBlob { 277 | var blob models.SimpleBlob 278 | 279 | return blob 280 | } 281 | 282 | // generateS3ContainerName gets the REAL Azure container name for the simpleBlob 283 | func (sh *S3Handler) generateS3ContainerName(blob models.SimpleBlob) string { 284 | currentContainer := blob.ParentContainer 285 | 286 | for currentContainer.ParentContainer != nil { 287 | currentContainer = currentContainer.ParentContainer 288 | } 289 | return currentContainer.Name 290 | } 291 | 292 | // PopulateBlob. Used to read a blob IFF we already have a reference to it. 293 | func (sh *S3Handler) PopulateBlob(blob *models.SimpleBlob) error { 294 | 295 | containerName := sh.generateS3ContainerName(*blob) 296 | 297 | req := &s3.GetObjectInput{ 298 | Bucket: &containerName, 299 | Key: aws.String(blob.BlobCloudName), 300 | } 301 | 302 | objectData, err := sh.s3Client.GetObject(req) 303 | if err != nil { 304 | log.Error(err) 305 | return err 306 | } 307 | 308 | defer objectData.Body.Close() 309 | 310 | // file stream for cache. 311 | var cacheFile *os.File 312 | 313 | // populate this to disk. 314 | if sh.cacheToDisk { 315 | 316 | cacheName := misc.GenerateCacheName(containerName + blob.BlobCloudName) 317 | blob.DataCachedAtPath = sh.cacheLocation + "/" + cacheName 318 | 319 | cacheFile, err = os.OpenFile(blob.DataCachedAtPath, os.O_WRONLY|os.O_CREATE, 0666) 320 | defer cacheFile.Close() 321 | 322 | if err != nil { 323 | log.Fatal(err) 324 | return err 325 | } 326 | } else { 327 | blob.DataInMemory = []byte{} 328 | } 329 | 330 | // 100k buffer... way too small? 331 | buffer := make([]byte, 1024*100) 332 | numBytesRead := 0 333 | 334 | finishedProcessing := false 335 | for finishedProcessing == false { 336 | numBytesRead, err = objectData.Body.Read(buffer) 337 | if err != nil { 338 | finishedProcessing = true 339 | } 340 | 341 | if numBytesRead <= 0 { 342 | finishedProcessing = true 343 | continue 344 | } 345 | 346 | // if we're caching, write to a file. 347 | if sh.cacheToDisk { 348 | _, err = cacheFile.Write(buffer[:numBytesRead]) 349 | if err != nil { 350 | log.Fatal(err) 351 | return err 352 | } 353 | } else { 354 | 355 | // needs to go into a byte array. How do we expand a slice again? 356 | blob.DataInMemory = append(blob.DataInMemory, buffer[:numBytesRead]...) 357 | } 358 | } 359 | 360 | if cacheFile != nil { 361 | log.Debugf("manually closing file %s", blob.DataCachedAtPath) 362 | cacheFile.Close() 363 | } else { 364 | log.Debug("no manual closing needed") 365 | } 366 | 367 | return nil 368 | } 369 | 370 | func (sh *S3Handler) WriteContainer(sourceContainer *models.SimpleContainer, destContainer *models.SimpleContainer) error { 371 | return nil 372 | } 373 | 374 | // WriteBlob writes a blob to an Azure container. 375 | // The SimpleContainer is NOT necessarily a direct mapping to an Azure container but may be representing a virtual directory. 376 | // ie we might have RootSimpleContainer -> SimpleContainer(myrealcontainer) -> SimpleContainer(vdir1) -> SimpleContainer(vdir2) 377 | // and if the blobName is "myblob" then the REAL underlying Azure structure would be container == "myrealcontainer" 378 | // and the blob name is vdir/vdir2/myblob 379 | func (sh *S3Handler) WriteBlob(destContainer *models.SimpleContainer, sourceBlob *models.SimpleBlob) error { 380 | 381 | var err error 382 | if sh.cacheToDisk { 383 | err = sh.writeBlobFromCache(destContainer, sourceBlob) 384 | } else { 385 | err = sh.writeBlobFromMemory(destContainer, sourceBlob) 386 | } 387 | 388 | if err != nil { 389 | log.Fatal(err) 390 | return err 391 | } 392 | 393 | return nil 394 | } 395 | 396 | func (sh *S3Handler) getContainerAndBlobNames(destContainer *models.SimpleContainer, sourceBlobName string) (string, string) { 397 | 398 | container, blobPrefix := containerutils.GetContainerAndBlobPrefix(destContainer) 399 | containerName := container.Name 400 | 401 | var blobName string 402 | 403 | if blobPrefix != "" { 404 | blobName = blobPrefix + "/" + sourceBlobName 405 | } else { 406 | blobName = sourceBlobName 407 | } 408 | 409 | return containerName, blobName 410 | } 411 | 412 | // writeBlobFromCache.. read the cache file and pass the byte slice onto the real writer. 413 | func (sh *S3Handler) writeBlobFromCache(destContainer *models.SimpleContainer, sourceBlob *models.SimpleBlob) error { 414 | containerName, blobName := sh.getContainerAndBlobNames(destContainer, sourceBlob.Name) 415 | 416 | // file stream for cache. 417 | var cacheFile *os.File 418 | 419 | // need to get cache dir from somewhere! 420 | cacheFile, err := os.OpenFile(sourceBlob.DataCachedAtPath, os.O_RDONLY, 0) 421 | if err != nil { 422 | log.Errorf("Unable to open cache file %s", sourceBlob.DataCachedAtPath) 423 | return err 424 | } 425 | defer cacheFile.Close() 426 | 427 | params := &s3.PutObjectInput{ 428 | Bucket: aws.String(containerName), 429 | Key: aws.String(blobName), 430 | Body: cacheFile, 431 | } 432 | _, err = sh.s3Client.PutObject(params) 433 | if err != nil { 434 | log.Errorf("Unable to upload %s", blobName) 435 | return err 436 | } 437 | 438 | return nil 439 | } 440 | 441 | func (sh *S3Handler) writeBlobFromMemory(destContainer *models.SimpleContainer, sourceBlob *models.SimpleBlob) error { 442 | containerName, blobName := sh.getContainerAndBlobNames(destContainer, sourceBlob.Name) 443 | 444 | fileBytes := bytes.NewReader(sourceBlob.DataInMemory) // convert to io.ReadSeeker type 445 | 446 | params := &s3.PutObjectInput{ 447 | Bucket: aws.String(containerName), 448 | Key: aws.String(blobName), 449 | Body: fileBytes, 450 | } 451 | _, err := sh.s3Client.PutObject(params) 452 | if err != nil { 453 | log.Errorf("Unable to upload %s", blobName) 454 | return err 455 | } 456 | 457 | return nil 458 | } 459 | 460 | func (sh *S3Handler) CreateContainer(containerName string) (models.SimpleContainer, error) { 461 | var container models.SimpleContainer 462 | 463 | return container, nil 464 | } 465 | 466 | // GetContainer gets a container. Populating the subtree? OR NOT? hmmmm 467 | func (ah *S3Handler) GetContainer(containerName string) models.SimpleContainer { 468 | var container models.SimpleContainer 469 | 470 | return container 471 | } 472 | 473 | // GetContainerContents populates the passed container with the real contents. 474 | // Can determine if the SimpleContainer is a real container or something virtual. 475 | // We need to trace back to the root node and determine what is really a container and 476 | // what is a blob. 477 | // 478 | // For S3 only the children of the root node can be a real azure container. Everything else 479 | // is a blob or a blob pretending to have vdirs. 480 | func (sh *S3Handler) GetContainerContents(container *models.SimpleContainer) error { 481 | s3Container, blobPrefix := containerutils.GetContainerAndBlobPrefix(container) 482 | 483 | params := s3.ListObjectsV2Input{ 484 | Bucket: &s3Container.Name, 485 | Prefix: &blobPrefix, 486 | } 487 | 488 | // slice of every object. This might get a tad large. 489 | // do we need to pass in pieces over channels? 490 | blobSlice := []*s3.Object{} 491 | 492 | err := sh.s3Client.ListObjectsV2Pages(¶ms, 493 | func(page *s3.ListObjectsV2Output, lastPage bool) bool { 494 | // copy of container, dont want to send back ever growing container via the channel. 495 | 496 | // variadic functions... look it up. :) 497 | blobSlice = append(blobSlice, page.Contents...) 498 | 499 | // return entire container via channel. 500 | return !lastPage 501 | }) 502 | 503 | if err != nil { 504 | return err 505 | } 506 | 507 | sh.populateSimpleContainer(blobSlice, container, blobPrefix) 508 | 509 | return nil 510 | } 511 | 512 | /* presign URL code..... use it eventually. 513 | */ 514 | 515 | func (sh *S3Handler) GeneratePresignedURL(blob *models.SimpleBlob) (string, error) { 516 | 517 | log.Debugf("S3:GeneratePresignedURL") 518 | s3Container, _ := containerutils.GetContainerAndBlobPrefix(blob.ParentContainer) 519 | 520 | r, _ := sh.s3Client.GetObjectRequest(&s3.GetObjectInput{ 521 | Bucket: aws.String(s3Container.Name), 522 | Key: aws.String(blob.BlobCloudName), 523 | }) 524 | 525 | //r.HTTPRequest.Header.Set("content-type", "application/octet-stream") 526 | // r.HTTPRequest.Header.Set("Content-MD5", checksum) 527 | url, err := r.Presign(15 * time.Minute) 528 | if err != nil { 529 | log.Error("error presigning request", err) 530 | return "", err 531 | } 532 | 533 | log.Debugf("presigning with %s and %s", s3Container.Name, blob.BlobCloudName) 534 | log.Debugf("presigned URL is %s", url) 535 | return url, nil 536 | } 537 | 538 | 539 | // populateSimpleContainer takes a list of S3 blobs and breaks them into virtual directories (SimpleContainers) and 540 | // SimpleBlob trees. 541 | // 542 | // vdir1/vdir2/blob1 543 | // vdir1/blob2 544 | // vdir1/vdir3/blob3 545 | // blob3 546 | func (sh *S3Handler) populateSimpleContainer(s3Objects []*s3.Object, container *models.SimpleContainer, blobPrefix string) { 547 | 548 | log.Debugf("populateSimpleContainer original container %s", container.Name) 549 | for _, blob := range s3Objects { 550 | log.Debugf("populateSimpleContainer %s", *blob.Key) 551 | 552 | // if key ends in / then its just a fake directory. 553 | // do we even want to store that? 554 | // for now, skip it. 555 | 556 | if strings.HasSuffix(*blob.Key, "/") { 557 | // skip it. 558 | continue 559 | } 560 | 561 | prunedBlobName := *blob.Key 562 | if blobPrefix != "" { 563 | prunedBlobName = prunedBlobName[len(blobPrefix):] 564 | } 565 | 566 | log.Debugf("pruned blob name %s", prunedBlobName) 567 | 568 | // need to shorten name to remove the container name itself. 569 | // ie if the name of a blob if foo/bar.txt but we are currently in the "foo" container (fake vdir) 570 | // then we need to prune the container name from the DestName for the blob. 571 | sp := strings.Split(prunedBlobName, "/") 572 | 573 | // if no / then no subdirs etc. Just add as is. 574 | if len(sp) == 1 { 575 | b := models.SimpleBlob{} 576 | b.Name = prunedBlobName 577 | b.Origin = container.Origin 578 | b.ParentContainer = container 579 | b.BlobCloudName = *blob.Key 580 | b.URL = generateS3URL(*blob.Key, container.Name) 581 | // add to the blob slice within the container 582 | container.BlobSlice = append(container.BlobSlice, &b) 583 | log.Debugf("1 S3 blob %v", b) 584 | } else { 585 | 586 | currentContainer := container 587 | // if slashes, then split into chunks and create accordingly. 588 | // skip last one since thats the blob name. 589 | spShort := sp[0 : len(sp)-1] 590 | for _, segment := range spShort { 591 | 592 | // check if container already has a subcontainer with appropriate name 593 | subContainer := sh.getSubContainer(currentContainer, segment) 594 | if subContainer != nil { 595 | // then we have a blob so add it to currentContainer 596 | currentContainer = subContainer 597 | } 598 | } 599 | 600 | b := models.SimpleBlob{} 601 | b.Name = sp[len(sp)-1] 602 | b.Origin = container.Origin 603 | b.ParentContainer = container 604 | b.BlobCloudName = *blob.Key // cloud specific name... ie the REAL name. 605 | b.URL = generateS3URL(*blob.Key, container.Name) 606 | currentContainer.BlobSlice = append(currentContainer.BlobSlice, &b) 607 | currentContainer.Populated = true 608 | 609 | log.Debugf("2 S3 blob name %s", b.Name) 610 | 611 | } 612 | } 613 | container.Populated = true 614 | } 615 | 616 | // getSubContainer gets an existing subcontainer with parent of container and name of segment. 617 | // otherwise it creates it, adds it to the parent container and returns the new one. 618 | func (sh *S3Handler) getSubContainer(container *models.SimpleContainer, segment string) *models.SimpleContainer { 619 | 620 | // MUST be a shorthand way of doing this. But still crawling in GO. 621 | for _, c := range container.ContainerSlice { 622 | if c.Name == segment { 623 | return c 624 | } 625 | } 626 | 627 | // create a new one. 628 | newContainer := models.SimpleContainer{} 629 | newContainer.Name = segment 630 | newContainer.Origin = container.Origin 631 | newContainer.ParentContainer = container 632 | container.ContainerSlice = append(container.ContainerSlice, &newContainer) 633 | return &newContainer 634 | } 635 | -------------------------------------------------------------------------------- /azurecopy/models/constants.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type CloudType int 4 | 5 | // fugly. 6 | const ( 7 | Azure CloudType = 1 + iota 8 | S3 9 | DropBox 10 | OneDrive 11 | Filesystem 12 | FTP 13 | ) 14 | -------------------------------------------------------------------------------- /azurecopy/models/simpleblob.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | // SimpleBlob is AzureCopy's cloud agnostic version of a blob 4 | // Although real clouds (Azure/S3 etc) allow blob names to simulate virtual directories 5 | // ie blob name can be "vdir1/vdir2/myblob" we will only store the "file" part of the URL. 6 | // so in this case it would be "myblob" and the containing container would be "vdir2" which 7 | // would also have a parent container "vdir1" etc. 8 | type SimpleBlob struct { 9 | 10 | // data, if cached in memory 11 | DataInMemory []byte 12 | 13 | // path to cached version of blob on disk. 14 | DataCachedAtPath string 15 | 16 | // if true then DataInMemory contains blob data 17 | // else DataCachedAtPath contains path to on disk cache of blob. 18 | BlobInMemory bool 19 | 20 | // name of blob. Generic nice stuff. (ie no fake vdirs) 21 | Name string 22 | URL string 23 | 24 | // destination blob name 25 | DestName string 26 | 27 | // REAL platform (Azure, S3 etc) name of blob. 28 | // ie including the nasty vdirs etc. 29 | BlobCloudName string 30 | 31 | Origin CloudType 32 | 33 | // indicates if this container was read from the source or destination. 34 | IsSource bool 35 | 36 | // parent. 37 | ParentContainer *SimpleContainer 38 | } 39 | -------------------------------------------------------------------------------- /azurecopy/models/simplecontainer.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | ) 7 | 8 | // SimpleContainer is AzureCopy's cloud agnostic version of a container 9 | // SimpleContainers will NOT necessarily match real cloud provider definitions of 10 | // containers. 11 | // 12 | // eg. Azure/S3 only have top level containers, but blobs inside of those can 13 | // contain blobs with "virtual directory" like names. 14 | // ie. container name is "foo" but blob name is "vdir1/vdir2/myblob" 15 | // In this case we will end up with 3 SimpleContainers and 1 SimpleBlob. 16 | // 17 | // UNLESS I CHANGE MY MIND RANDOMLY, WHICH IS VERY POSSIBLE. 18 | type SimpleContainer struct { 19 | Name string 20 | URL string 21 | Origin CloudType 22 | 23 | // indicates if this container was read from the source or destination. 24 | IsSource bool 25 | 26 | // parent. 27 | // if nil parent then its the root. 28 | ParentContainer *SimpleContainer 29 | 30 | // slice of all blobs in this container 31 | BlobSlice []*SimpleBlob 32 | 33 | // slice of all containers in this container 34 | ContainerSlice []*SimpleContainer 35 | 36 | // have we attempted to populate this container? 37 | Populated bool 38 | 39 | // is Root container. 40 | IsRootContainer bool 41 | } 42 | 43 | // NewSimpleContainer factory time! 44 | func NewSimpleContainer() *SimpleContainer { 45 | c := SimpleContainer{} 46 | c.BlobSlice = []*SimpleBlob{} 47 | c.ContainerSlice = []*SimpleContainer{} 48 | c.ParentContainer = nil 49 | c.Populated = false 50 | return &c 51 | } 52 | 53 | // GetBlob gets a reference to a blob in the container. Does NOT go recursive, recursive, recursive, recursive...... 54 | func (sc *SimpleContainer) GetBlob(blobName string) (*SimpleBlob, error) { 55 | 56 | for _, b := range sc.BlobSlice { 57 | if b.Name == blobName { 58 | return b, nil 59 | } 60 | } 61 | err := errors.New("Blob " + blobName + " not found") 62 | return nil, err 63 | } 64 | 65 | // GetBlob gets a reference to a blob in the container. Does NOT go recursive, recursive, recursive, recursive...... 66 | func (sc *SimpleContainer) GetContainer(containerName string) (*SimpleContainer, error) { 67 | 68 | for _, c := range sc.ContainerSlice { 69 | if c.Name == containerName { 70 | return c, nil 71 | } 72 | } 73 | err := errors.New("Container " + containerName + " not found") 74 | return nil, err 75 | } 76 | 77 | 78 | func (sc *SimpleContainer) DisplayContainerURLsOnly() { 79 | 80 | 81 | for _, b := range sc.BlobSlice { 82 | fmt.Println(b.URL) 83 | } 84 | 85 | for _, c := range sc.ContainerSlice { 86 | c.DisplayContainerURLsOnly() 87 | } 88 | } 89 | 90 | func (sc *SimpleContainer) DisplayContainer(padding string) { 91 | 92 | fmt.Println("+" + padding + sc.Name) 93 | 94 | padding = padding + " " 95 | 96 | for _, b := range sc.BlobSlice { 97 | fmt.Println(padding + b.Name + "(" + b.URL + ")") 98 | } 99 | 100 | for _, c := range sc.ContainerSlice { 101 | c.DisplayContainer(padding) 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /azurecopy/utils/blobutils/blobutils.go: -------------------------------------------------------------------------------- 1 | package blobutils 2 | 3 | import ( 4 | "azurecopy/azurecopy/models" 5 | "azurecopy/azurecopy/utils/misc" 6 | "io" 7 | "os" 8 | 9 | log "github.com/Sirupsen/logrus" 10 | ) 11 | 12 | func ReadBlob(reader io.ReadCloser, blob *models.SimpleBlob, cacheToDisk bool, cacheLocation string) error { 13 | // file stream for cache. 14 | var cacheFile *os.File 15 | var err error 16 | 17 | // populate this to disk. 18 | if cacheToDisk { 19 | 20 | cacheName := misc.GenerateCacheName(blob.BlobCloudName) 21 | blob.DataCachedAtPath = cacheLocation + "/" + cacheName 22 | log.Debugf("cache location is %s", blob.DataCachedAtPath) 23 | cacheFile, err = os.OpenFile(blob.DataCachedAtPath, os.O_WRONLY|os.O_CREATE, 0666) 24 | //defer cacheFile.Close() 25 | 26 | if err != nil { 27 | log.Fatalf("Populate blob %s", err) 28 | return err 29 | } 30 | } else { 31 | blob.DataInMemory = []byte{} 32 | } 33 | 34 | log.Debugf("cachefile early is %s", cacheFile) 35 | // 100k buffer... way too small? 36 | buffer := make([]byte, 1024*100) 37 | finishedProcessing := false 38 | for finishedProcessing == false { 39 | numBytesRead, err := reader.Read(buffer) 40 | if err != nil { 41 | finishedProcessing = true 42 | } 43 | 44 | if numBytesRead <= 0 { 45 | finishedProcessing = true 46 | continue 47 | } 48 | 49 | //log.Debugf("bytes %s", buffer) 50 | log.Debugf("number of bytes read %d", numBytesRead) 51 | // if we're caching, write to a file. 52 | if cacheToDisk { 53 | _, err := cacheFile.Write(buffer[:numBytesRead]) 54 | if err != nil { 55 | log.Debugf("cachefile %s", cacheFile) 56 | 57 | log.Fatalf("cache to disk fatal %s", err) 58 | return err 59 | } 60 | } else { 61 | 62 | // needs to go into a byte array. How do we expand a slice again? 63 | blob.DataInMemory = append(blob.DataInMemory, buffer[:numBytesRead]...) 64 | } 65 | } 66 | 67 | return nil 68 | } 69 | -------------------------------------------------------------------------------- /azurecopy/utils/containerutils/containerutils.go: -------------------------------------------------------------------------------- 1 | package containerutils 2 | 3 | import ( 4 | "azurecopy/azurecopy/models" 5 | 6 | log "github.com/Sirupsen/logrus" 7 | ) 8 | 9 | // GetRootContainer Get root container of a simple container. 10 | func GetRootContainer(container *models.SimpleContainer) *models.SimpleContainer { 11 | var p *models.SimpleContainer 12 | 13 | for p = container.ParentContainer; p.ParentContainer != nil; { 14 | p = p.ParentContainer 15 | } 16 | 17 | return p 18 | } 19 | 20 | // GetContainerAndBlobPrefix Gets the REAL Azure container and the blob prefix for a given SimpleContainer 21 | // that has been passed in. 22 | func GetContainerAndBlobPrefix(container *models.SimpleContainer) (*models.SimpleContainer, string) { 23 | var p *models.SimpleContainer 24 | blobPrefix := "" 25 | var realContainer *models.SimpleContainer 26 | 27 | for p = container; p != nil; { 28 | 29 | // if parent container is not nil, then we're NOT a real azure container. 30 | if p.ParentContainer != nil { 31 | blobPrefix = p.Name + "/" + blobPrefix 32 | } else { 33 | // parent IS nil, therefore we're in the real azure container. 34 | realContainer = p 35 | } 36 | 37 | p = p.ParentContainer 38 | 39 | } 40 | 41 | log.Debugf("Got container: %s , blobprefix: %s", realContainer.Name, blobPrefix) 42 | return realContainer, blobPrefix 43 | } 44 | 45 | // GetContainerByName returns an existing container by name OR creates a container, adds it to the parent and returns the new one. 46 | func GetContainerByName(parentContainer *models.SimpleContainer, containerName string) *models.SimpleContainer { 47 | 48 | for _, container := range parentContainer.ContainerSlice { 49 | if container.Name == containerName { 50 | return container 51 | } 52 | } 53 | 54 | container := models.SimpleContainer{} 55 | container.Name = containerName 56 | container.ParentContainer = parentContainer 57 | parentContainer.ContainerSlice = append(parentContainer.ContainerSlice, &container) 58 | return &container 59 | } 60 | -------------------------------------------------------------------------------- /azurecopy/utils/handlerutils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "azurecopy/azurecopy/handlers" 5 | "azurecopy/azurecopy/models" 6 | "azurecopy/azurecopy/utils/misc" 7 | 8 | log "github.com/Sirupsen/logrus" 9 | ) 10 | 11 | // GetHandler gets the appropriate handler for the cloudtype. 12 | // Should I be doing this another way? 13 | func GetHandler(cloudType models.CloudType, isSource bool, config misc.CloudConfig, cacheToDisk bool, isEmulator bool) handlers.CloudHandlerInterface { 14 | switch cloudType { 15 | case models.Azure: 16 | 17 | accountName, accountKey := GetAzureCredentials(isSource, config) 18 | 19 | log.Debug("Got Azure Handler") 20 | ah, _ := handlers.NewAzureHandler(accountName, accountKey, isSource, cacheToDisk, isEmulator) 21 | return ah 22 | 23 | case models.Filesystem: 24 | log.Debug("Got Filesystem Handler") 25 | var URL string 26 | if isSource { 27 | URL = config.Configuration[misc.Source] 28 | } else { 29 | URL = config.Configuration[misc.Dest] 30 | } 31 | fh, _ := handlers.NewFilesystemHandler(URL, isSource) // default path? 32 | return fh 33 | 34 | case models.S3: 35 | log.Debug("Got S3 Handler") 36 | accessID, accessSecret, region := getS3Credentials(isSource, config) 37 | 38 | sh, _ := handlers.NewS3Handler(accessID, accessSecret, region, isSource, true) 39 | return sh 40 | 41 | case models.DropBox: 42 | log.Debug("Got Dropbox Handler") 43 | dh, _ := handlers.NewDropboxHandler(isSource, true) 44 | return dh 45 | } 46 | 47 | return nil 48 | } 49 | 50 | func GetAzureCredentials(isSource bool, config misc.CloudConfig) (accountName string, accountKey string) { 51 | if isSource { 52 | accountName = config.Configuration[misc.AzureSourceAccountName] 53 | accountKey = config.Configuration[misc.AzureSourceAccountKey] 54 | } else { 55 | accountName = config.Configuration[misc.AzureDestAccountName] 56 | accountKey = config.Configuration[misc.AzureDestAccountKey] 57 | } 58 | 59 | if accountName == "" || accountKey == "" { 60 | accountName = config.Configuration[misc.AzureDefaultAccountName] 61 | accountKey = config.Configuration[misc.AzureDefaultAccountKey] 62 | } 63 | 64 | return accountName, accountKey 65 | } 66 | 67 | func getS3Credentials(isSource bool, config misc.CloudConfig) (accessID string, accessSecret string, region string) { 68 | if isSource { 69 | accessID = config.Configuration[misc.S3SourceAccessID] 70 | accessSecret = config.Configuration[misc.S3SourceAccessSecret] 71 | region = config.Configuration[misc.S3SourceRegion] 72 | } else { 73 | accessID = config.Configuration[misc.S3DestAccessID] 74 | accessSecret = config.Configuration[misc.S3DestAccessSecret] 75 | region = config.Configuration[misc.S3DestRegion] 76 | } 77 | 78 | if accessID == "" || accessSecret == "" { 79 | accessID = config.Configuration[misc.S3DefaultAccessID] 80 | accessSecret = config.Configuration[misc.S3DefaultAccessSecret] 81 | region = config.Configuration[misc.S3DefaultRegion] 82 | } 83 | 84 | return accessID, accessSecret, region 85 | } 86 | -------------------------------------------------------------------------------- /azurecopy/utils/helpers/azurehelper.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | type AzureHelper struct { 4 | } 5 | 6 | /* 7 | func NewAzureHelper(accountName string, accountKey string) *AzureHelper { 8 | 9 | ah := new(AzureHelper) 10 | 11 | client, err := storage.NewBasicClient(accountName, accountKey) 12 | if err != nil { 13 | log.Fatalf("NewAzureHelper cannot generate Azure Storage client", err) 14 | } 15 | 16 | ah.client = client.GetBlobService() 17 | return ah 18 | } 19 | 20 | // DoCopyBlobUsingAzureCopyBlobFlag copy using Azure CopyBlob flag. 21 | // Have to create a new instance of the storage client. I can't get it out of AzureHandler since 22 | // we get that via an interface, and obviously not all handlers will have Azure clients. 23 | // TODO(kpfaulkner) revisit and find a better way, but new client for now isn't completely terrible. 24 | func (ah *AzureHelper) DoCopyBlobUsingAzureCopyBlobFlag(url string, destContainer *models.SimpleContainer, destBlobName string) error { 25 | 26 | // need to get real azure container but I *think* destBlobName has already been correctly converted. 27 | // need to check that! TODO(kpfaulkner) 28 | 29 | container, prefix := containerutils.GetContainerAndBlobPrefix(destContainer) 30 | 31 | destBlobNameWithPrefix := prefix + destBlobName 32 | log.Debugf("CopyBlob: source %s : dest container %s : blobname %s : prefix %s : fullDestname %s", url, container.Name, destBlobName, prefix, destBlobNameWithPrefix) 33 | 34 | err := ah.client.CopyBlob(container.Name, destBlobNameWithPrefix, url) 35 | if err != nil { 36 | log.Errorf("Unable to copy %s %s", url, err) 37 | } 38 | return nil 39 | } 40 | 41 | // TODO(kpfaulkner) revisit and find a better way, but new client for now isn't completely terrible. 42 | func (ah *AzureHelper) GetOrCreateContainer(containerName string) (storage.ContainerURL, error) { 43 | 44 | } 45 | 46 | */ -------------------------------------------------------------------------------- /azurecopy/utils/helpers/dropboxhelper.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path" 9 | "path/filepath" 10 | 11 | log "github.com/Sirupsen/logrus" 12 | "github.com/dropbox/dropbox-sdk-go-unofficial/dropbox" 13 | homedir "github.com/mitchellh/go-homedir" 14 | "golang.org/x/oauth2" 15 | ) 16 | 17 | const ( 18 | configFileName = "azurecopyauth.json" 19 | appKey = "iq11ew8vur0c5x7" 20 | appSecret = "p3rm8w02t1ru048" 21 | dropboxScheme = "dropbox" 22 | 23 | tokenPersonal = "personal" 24 | tokenTeamAccess = "teamAccess" 25 | tokenTeamManage = "teamManage" 26 | ) 27 | 28 | // Map of map of strings 29 | // For each domain, we want to save different tokens depending on the 30 | // command type: personal, team access and team manage 31 | type TokenMap map[string]map[string]string 32 | 33 | func WriteTokens(filePath string, tokens TokenMap) { 34 | // Check if file exists 35 | if _, err := os.Stat(filePath); os.IsNotExist(err) { 36 | // Doesn't exist; lets create it 37 | err = os.MkdirAll(filepath.Dir(filePath), 0700) 38 | if err != nil { 39 | return 40 | } 41 | } 42 | 43 | // At this point, file must exist. Lets (over)write it. 44 | b, err := json.Marshal(tokens) 45 | if err != nil { 46 | return 47 | } 48 | if err = ioutil.WriteFile(filePath, b, 0600); err != nil { 49 | return 50 | } 51 | } 52 | 53 | func ReadTokens(filePath string) (TokenMap, error) { 54 | b, err := ioutil.ReadFile(filePath) 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | var tokens TokenMap 60 | if json.Unmarshal(b, &tokens) != nil { 61 | return nil, err 62 | } 63 | 64 | return tokens, nil 65 | } 66 | 67 | func SetupConnection() (*dropbox.Config, error) { 68 | conf := oauth2.Config{ 69 | ClientID: appKey, 70 | ClientSecret: appSecret, 71 | Endpoint: dropbox.OAuthEndpoint(""), 72 | } 73 | 74 | dir, err := homedir.Dir() 75 | if err != nil { 76 | return nil, err 77 | } 78 | filePath := path.Join(dir, ".config", "azurecopy", configFileName) 79 | tokType := tokenPersonal 80 | 81 | tokenMap, err := ReadTokens(filePath) 82 | if tokenMap == nil { 83 | tokenMap = make(TokenMap) 84 | } 85 | domain := "" 86 | 87 | if tokenMap[domain] == nil { 88 | tokenMap[domain] = make(map[string]string) 89 | } 90 | tokens := tokenMap[domain] 91 | 92 | if err != nil || tokens[tokType] == "" { 93 | fmt.Printf("1. Go to %v\n", conf.AuthCodeURL("state")) 94 | fmt.Printf("2. Click \"Allow\" (you might have to log in first).\n") 95 | fmt.Printf("3. Copy the authorization code.\n") 96 | fmt.Printf("Enter the authorization code here: ") 97 | 98 | var code string 99 | if _, err = fmt.Scan(&code); err != nil { 100 | return nil, err 101 | } 102 | var token *oauth2.Token 103 | token, err = conf.Exchange(oauth2.NoContext, code) 104 | if err != nil { 105 | return nil, err 106 | } 107 | tokens[tokType] = token.AccessToken 108 | WriteTokens(filePath, tokenMap) 109 | } else { 110 | log.Debugf("Already have Dropbox token") 111 | } 112 | 113 | config := dropbox.Config{tokens[tokType], dropbox.LogOff, nil, "", domain, nil, nil, nil} 114 | 115 | //config := dropbox.Config{tokens[tokType], true, "", domain} 116 | 117 | return &config, nil 118 | } 119 | -------------------------------------------------------------------------------- /azurecopy/utils/helpers/ftphelper.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | type FTPHelper struct { 4 | } 5 | 6 | -------------------------------------------------------------------------------- /azurecopy/utils/misc/config.go: -------------------------------------------------------------------------------- 1 | package misc 2 | 3 | // misc consts for credentials. 4 | // need a more dynamic way to add for new cloud types. 5 | // but for now, it will do. 6 | const ( 7 | // Azure. 8 | AzureDefaultAccountName = "AzureDefaultAccountName" 9 | AzureDefaultAccountKey = "AzureDefaultAccountKey" 10 | AzureSourceAccountName = "AzureSourceAccountName" 11 | AzureSourceAccountKey = "AzureSourceAccountKey" 12 | AzureDestAccountName = "AzureDestAccountName" 13 | AzureDestAccountKey = "AzureDestAccountKey" 14 | 15 | // S3 16 | S3DefaultAccessID = "S3DefaultAccessID" 17 | S3DefaultAccessSecret = "S3DefaultAccessSecret" 18 | S3DefaultRegion = "S3DefaultRegion" 19 | 20 | S3SourceAccessID = "S3SourceAccessID" 21 | S3SourceAccessSecret = "S3SourceAccessSecret" 22 | S3SourceRegion = "S3SourceRegion" 23 | 24 | S3DestAccessID = "S3DestAccessID" 25 | S3DestAccessSecret = "S3DestAccessSecret" 26 | S3DestRegion = "S3DestRegion" 27 | 28 | // debug 29 | Debug = "Debug" 30 | Source = "Source" 31 | Dest = "Dest" 32 | Replace = "Replace" 33 | 34 | // container name to create. 35 | CreateContainerName = "CreateContainer" 36 | ) 37 | 38 | // Commands to execute 39 | const ( 40 | CommandCopy = iota 41 | CommandList 42 | CommandCreateContainer 43 | CommandUnknown 44 | CommandListContainer 45 | CommandCopyBlob 46 | ) 47 | 48 | // CloudConfig UGLY UGLY UGLY way to store the configuration. 49 | // globally accessible, otherwise I'm passing it everywhere. 50 | type CloudConfig struct { 51 | Configuration map[string]string 52 | 53 | Debug bool // are we in debug mode. 54 | 55 | Command int // command we're executing 56 | 57 | SimpleOutput bool // want simple output (URLs) or tree displays. 58 | 59 | Replace bool // will replace at destination 60 | 61 | Version bool // display version 62 | 63 | ConcurrentCount uint // how many goroutines do we have in the pool? 64 | } 65 | 66 | // NewCloudConfig Make new (and only really) configuration map 67 | func NewCloudConfig() *CloudConfig { 68 | cc := CloudConfig{} 69 | cc.Configuration = make(map[string]string) 70 | return &cc 71 | } 72 | -------------------------------------------------------------------------------- /azurecopy/utils/misc/pathutils.go: -------------------------------------------------------------------------------- 1 | package misc 2 | 3 | import ( 4 | "crypto/md5" 5 | "encoding/hex" 6 | ) 7 | 8 | // GetLastChar gets last char of string. 9 | // SURELY there is a built in function for this??!? 10 | func GetLastChar(str string) string { 11 | return str[len(str)-1:] 12 | } 13 | 14 | func GenerateCacheName(path string) string { 15 | hasher := md5.New() 16 | hasher.Write([]byte(path)) 17 | return hex.EncodeToString(hasher.Sum(nil)) 18 | } 19 | -------------------------------------------------------------------------------- /azurecopycommand/.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | *.dll 6 | *.cmd 7 | 8 | # Folders 9 | _obj 10 | _test 11 | 12 | # Architecture specific extensions/prefixes 13 | *.[568vq] 14 | [568vq].out 15 | 16 | *.cgo1.go 17 | *.cgo2.c 18 | _cgo_defun.c 19 | _cgo_gotypes.go 20 | _cgo_export.* 21 | 22 | _testmain.go 23 | 24 | *.exe 25 | *.test 26 | *.prof 27 | 28 | # Output of the go coverage tool, specifically when used with LiteIDE 29 | *.out 30 | -------------------------------------------------------------------------------- /azurecopycommand/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "name": "Launch", 6 | "type": "go", 7 | "request": "launch", 8 | "mode": "debug", 9 | "remotePath": "", 10 | "port": 2345, 11 | "host": "127.0.0.1", 12 | "program": "${workspaceRoot}", 13 | "env": {}, 14 | "args": [], 15 | "showLog": true 16 | } 17 | ] 18 | } -------------------------------------------------------------------------------- /azurecopycommand/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "azurecopy/azurecopy" 5 | "azurecopy/azurecopy/models" 6 | "azurecopy/azurecopy/utils/misc" 7 | "flag" 8 | "fmt" 9 | 10 | "os" 11 | 12 | log "github.com/Sirupsen/logrus" 13 | ) 14 | 15 | var Version string 16 | 17 | func generateSpace(c int) string { 18 | s := "" 19 | for i := 0; i < c; i++ { 20 | s = s + " " 21 | } 22 | 23 | return s 24 | } 25 | 26 | func printContainer(container *models.SimpleContainer, depth int) { 27 | s := generateSpace(depth) 28 | 29 | log.Printf("%scontainer: %s", s, container.Name) 30 | 31 | depth = depth + 2 32 | s = generateSpace(depth) 33 | 34 | for _, b := range container.BlobSlice { 35 | log.Printf("%sblob: %s", s, b.Name) 36 | } 37 | 38 | for _, c := range container.ContainerSlice { 39 | printContainer(c, depth) 40 | } 41 | 42 | } 43 | 44 | // getCommand. Naive way to determine what the actual user wants to do. Copy, list etc etc. 45 | // rework when it gets more complex. 46 | func getCommand(copyCommand bool, listCommand bool, createContainerCommand string, copyBlobCommand bool) int { 47 | 48 | if !copyCommand && !listCommand && createContainerCommand == "" && !copyBlobCommand { 49 | fmt.Println("No command given") 50 | os.Exit(1) 51 | } 52 | 53 | if copyCommand { 54 | return misc.CommandCopy 55 | } 56 | 57 | if copyBlobCommand { 58 | return misc.CommandCopyBlob 59 | } 60 | 61 | if listCommand { 62 | return misc.CommandList 63 | } 64 | 65 | if createContainerCommand != "" { 66 | log.Debug("createcommand issued") 67 | return misc.CommandCreateContainer 68 | } 69 | 70 | log.Fatal("unsure of command to use") 71 | return misc.CommandUnknown 72 | } 73 | 74 | func setupConfiguration() *misc.CloudConfig { 75 | config := misc.NewCloudConfig() 76 | 77 | var concurrentCount = flag.Uint("cc", 5, "Concurrent Count. How many blobs are copied concurrently") 78 | 79 | var version = flag.Bool("version", false, "Display Version") 80 | var source = flag.String("source", "", "Source URL") 81 | var dest = flag.String("dest", "", "Destination URL") 82 | var debug = flag.Bool("debug", false, "Debug output") 83 | var copyCommand = flag.Bool("copy", false, "Copy from source to destination") 84 | var copyBlobCommand = flag.Bool("copyblob", false, "Copy from source to destination using Azure CopyBlob flag. Can only be used if Azure is destination") 85 | 86 | //var copyBlobCommand = false 87 | 88 | var listCommand = flag.Bool("list", false, "List contents from source") 89 | var createContainerCommand = flag.String("createcontainer", "", "Create container for destination") 90 | 91 | var simpleOutput = flag.Bool("simpleoutput", false, "Simple output, URLs over trees") 92 | 93 | var replace = flag.Bool("replace", true, "Replace blob if already exists") 94 | 95 | var azureDefaultAccountName = flag.String("AzureDefaultAccountName", "", "Default Azure Account Name") 96 | var azureDefaultAccountKey = flag.String("AzureDefaultAccountKey", "", "Default Azure Account Key") 97 | var azureSourceAccountName = flag.String("AzureSourceAccountName", "", "Source Azure Account Name") 98 | var azureSourceAccountKey = flag.String("AzureSourceAccountKey", "", "Source Azure Account Key") 99 | var azureDestAccountName = flag.String("AzureDestAccountName", "", "Destination Azure Account Name") 100 | var azureDestAccountKey = flag.String("AzureDestAccountKey", "", "Destination Azure Account Key") 101 | 102 | var s3DefaultAccessID = flag.String("S3DefaultAccessID", "", "Default S3 Access ID") 103 | var s3DefaultAccessSecret = flag.String("S3DefaultAccessSecret", "", "Default S3 Access Secret") 104 | var s3DefaultRegion = flag.String("S3DefaultRegion", "", "Default S3 Region") 105 | var s3SourceAccessID = flag.String("S3SourceAccessID", "", "Source S3 Access ID") 106 | var s3SourceAccessSecret = flag.String("S3SourceAccessSecret", "", "Source S3 Access Secret") 107 | var s3SourceRegion = flag.String("S3SourceRegion", "", "Source S3 Region") 108 | var s3DestAccessID = flag.String("S3DestAccessID", "", "Destination S3 Access ID") 109 | var s3DestAccessSecret = flag.String("S3DestAccessSecret", "", "Destination S3 Access Secret") 110 | var s3DestRegion = flag.String("S3DestRegion", "", "Destination S3 Region") 111 | 112 | flag.Parse() 113 | 114 | config.Version = *version 115 | config.Debug = *debug 116 | if !*version { 117 | 118 | // seems toooooo manual. Figure out something nicer later. 119 | if *concurrentCount > 1000 { 120 | fmt.Printf("Maximum number for concurrent count is 1000") 121 | os.Exit(1) 122 | } 123 | 124 | config.Command = getCommand(*copyCommand, *listCommand, *createContainerCommand, *copyBlobCommand) 125 | config.Configuration[misc.Source] = *source 126 | config.Configuration[misc.Dest] = *dest 127 | config.Replace = *replace 128 | config.SimpleOutput = *simpleOutput 129 | config.ConcurrentCount = *concurrentCount 130 | config.Configuration[misc.CreateContainerName] = *createContainerCommand 131 | 132 | config.Configuration[misc.AzureDefaultAccountName] = *azureDefaultAccountName 133 | config.Configuration[misc.AzureDefaultAccountKey] = *azureDefaultAccountKey 134 | config.Configuration[misc.AzureSourceAccountName] = *azureSourceAccountName 135 | config.Configuration[misc.AzureSourceAccountKey] = *azureSourceAccountKey 136 | config.Configuration[misc.AzureDestAccountName] = *azureDestAccountName 137 | config.Configuration[misc.AzureDestAccountKey] = *azureDestAccountKey 138 | 139 | config.Configuration[misc.S3DefaultAccessID] = *s3DefaultAccessID 140 | config.Configuration[misc.S3DefaultAccessSecret] = *s3DefaultAccessSecret 141 | config.Configuration[misc.S3DefaultRegion] = *s3DefaultRegion 142 | config.Configuration[misc.S3SourceAccessID] = *s3SourceAccessID 143 | config.Configuration[misc.S3SourceAccessSecret] = *s3SourceAccessSecret 144 | config.Configuration[misc.S3SourceRegion] = *s3SourceRegion 145 | config.Configuration[misc.S3DestAccessID] = *s3DestAccessID 146 | config.Configuration[misc.S3DestAccessSecret] = *s3DestAccessSecret 147 | config.Configuration[misc.S3DestRegion] = *s3DestRegion 148 | } 149 | 150 | return config 151 | } 152 | 153 | // "so it begins" 154 | func main() { 155 | 156 | config := setupConfiguration() 157 | 158 | if !config.Debug { 159 | log.SetLevel(log.InfoLevel) 160 | } else { 161 | log.SetLevel(log.DebugLevel) 162 | } 163 | log.Debug("after config setup") 164 | 165 | // if display version, then display then exit 166 | if config.Version { 167 | fmt.Println("Version: " + Version) 168 | return 169 | } 170 | 171 | ac := azurecopy.NewAzureCopy(*config) 172 | 173 | switch config.Command { 174 | case misc.CommandCopy: 175 | err := ac.CopyBlobByURL(config.Replace, false) 176 | if err != nil { 177 | log.Fatal(err) 178 | } 179 | break 180 | 181 | case misc.CommandCopyBlob: 182 | err := ac.CopyBlobByURL(config.Replace, true) 183 | if err != nil { 184 | log.Fatal(err) 185 | } 186 | break 187 | 188 | case misc.CommandList: 189 | container, err := ac.ListContainer( ) 190 | if err != nil { 191 | log.Fatal(err) 192 | } 193 | 194 | log.Debug("List results") 195 | if config.SimpleOutput { 196 | container.DisplayContainerURLsOnly() 197 | } else { 198 | container.DisplayContainer("") 199 | } 200 | break 201 | 202 | case misc.CommandCreateContainer: 203 | err := ac.CreateContainer(config.Configuration[misc.CreateContainerName]) 204 | if err != nil { 205 | log.Fatal(err) 206 | } 207 | 208 | case misc.CommandUnknown: 209 | log.Fatal("Unsure of command to execute") 210 | } 211 | 212 | } 213 | -------------------------------------------------------------------------------- /azurecopycommand/res: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kpfaulkner/azurecopy-go/d103139e817955efff869f8d98bf15ded0b42d4a/azurecopycommand/res --------------------------------------------------------------------------------