├── .gitignore ├── LICENSE ├── README.md ├── build └── build.sh ├── cmd ├── abort.go ├── abort_test.go ├── bucket_tagging.go ├── bucket_versioning.go ├── buket_tagging_test.go ├── buket_versioning_test.go ├── cat.go ├── cat_test.go ├── config.go ├── config_add.go ├── config_add_test.go ├── config_delete.go ├── config_delete_test.go ├── config_init.go ├── config_init_test.go ├── config_set.go ├── config_set_test.go ├── config_show.go ├── config_show_test.go ├── config_test.go ├── cp.go ├── cp_test.go ├── du.go ├── du_test.go ├── hash.go ├── hash_test.go ├── ls.go ├── ls_test.go ├── lsdu.go ├── lsdu_test.go ├── lsparts.go ├── lsparts_test.go ├── mb.go ├── mb_test.go ├── rb.go ├── rb_test.go ├── restore.go ├── restore_test.go ├── rm.go ├── rm_test.go ├── root.go ├── root_test.go ├── signurl.go ├── signurl_test.go ├── symlink.go ├── symlink_test.go ├── sync.go ├── sync_test.go └── testconfig_test.go ├── go.mod ├── go.sum ├── logger └── logger.go ├── main.go └── util ├── bucket.go ├── cam_auth.go ├── cat_object.go ├── client.go ├── const.go ├── copy.go ├── delete.go ├── download.go ├── error_output.go ├── filter.go ├── get_cos_object.go ├── get_files.go ├── get_ofs_object.go ├── hash.go ├── list.go ├── list_iterator.go ├── listener.go ├── meta.go ├── path.go ├── print.go ├── process_monitor.go ├── processbar.go ├── rename.go ├── restore_objects.go ├── secret.go ├── size.go ├── statistic.go ├── storage_url.go ├── symlink.go ├── synchronize.go ├── types.go ├── upload.go ├── upload_parts.go ├── url.go └── versioning.go /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .vscode 3 | .DS_Store 4 | release 5 | vendor 6 | 7 | coscli 8 | coscli.log 9 | test_clear.sh 10 | dist/ 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # coscli 2 | 3 | ## 介绍 4 | 5 | 腾讯云对象存储（cos）命令行工具，使用 Go 编写，部署方便，且支持跨桶操作。 6 | 7 | ## 下载链接 8 | 9 | 当前版本：v1.0.6 10 | 11 | [Linux-386](https://github.com/tencentyun/coscli/releases/download/v1.0.6/coscli-v1.0.6-linux-386) 12 | 13 | [Linux-amd64](https://github.com/tencentyun/coscli/releases/download/v1.0.6/coscli-v1.0.6-linux-amd64) 14 | 15 | [Linux-arm](https://github.com/tencentyun/coscli/releases/download/v1.0.6/coscli-v1.0.6-linux-arm) 16 | 17 | [Linux-arm64](https://github.com/tencentyun/coscli/releases/download/v1.0.6/coscli-v1.0.6-linux-arm64) 18 | 19 | [Mac-amd64](https://github.com/tencentyun/coscli/releases/download/v1.0.6/coscli-v1.0.6-darwin-amd64) 20 | 21 | [Mac-arm64](https://github.com/tencentyun/coscli/releases/download/v1.0.6/coscli-v1.0.6-darwin-arm64) 22 | 23 | [Windows-386](https://github.com/tencentyun/coscli/releases/download/v1.0.6/coscli-v1.0.6-windows-386.exe) 24 | 25 | [Windows-amd64](https://github.com/tencentyun/coscli/releases/download/v1.0.6/coscli-v1.0.6-windows-amd64.exe) 26 | 27 | ## 使用方法 28 | 29 | 详见[腾讯云官方文档](https://cloud.tencent.com/document/product/436/63143) 30 | 31 | -------------------------------------------------------------------------------- /build/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | rootdir=$(cd `dirname $0`; cd ..; pwd) 4 | output=$rootdir/dist 5 | version=`cat ./util/const.go | grep -E "Version.+ string =" | cut -d"=" -f2 | xargs` 6 | 7 | # 创建输出目录 8 | mkdir -p $output 9 | 10 | # 定义构建函数 11 | build() { 12 | local os=$1 13 | local arch=$2 14 | local file_suffix=$3 15 | 16 | echo "start build coscli for $os on $arch" 17 | cd $rootdir 18 | env GOOS=$os GOARCH=$arch go build -o $output/coscli-$version-$os-$arch$file_suffix 19 | echo "coscli for $os on $arch built successfully" 20 | } 21 | 22 | # 定义计算哈希值的函数 23 | calc_hash() { 24 | cd $output 25 | rm -f sha256sum.log # 删除现有的 sha256sum.log 文件（如果存在） 26 | for file in $(ls *); do 27 | if [ "$file" != "sha256sum.log" ]; then 28 | sha256sum $file >> sha256sum.log 29 | fi 30 | done 31 | } 32 | 33 | # 构建不同平台的二进制文件 34 | build darwin amd64 "" 35 | build darwin arm64 "" 36 | build windows 386 ".exe" 37 | build windows amd64 ".exe" 38 | build linux 386 "" 39 | build linux amd64 "" 40 | build linux arm "" 41 | build linux arm64 "" 42 | 43 | # 计算哈希值 44 | calc_hash -------------------------------------------------------------------------------- /cmd/abort.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | var abortCmd = &cobra.Command{ 10 | Use: "abort", 11 | Short: "Abort parts", 12 | Long: `Abort parts 13 | 14 | Format: 15 | ./coscli abort cos://[/] [flags] 16 | 17 | Example: 18 | ./coscli abort cos://examplebucket/test/`, 19 | Args: cobra.ExactArgs(1), 20 | RunE: func(cmd *cobra.Command, args []string) error { 21 | include, _ := cmd.Flags().GetString("include") 22 | exclude, _ := cmd.Flags().GetString("exclude") 23 | failOutput, _ := cmd.Flags().GetBool("fail-output") 24 | failOutputPath, _ := cmd.Flags().GetString("fail-output-path") 25 | 26 | _, filters := util.GetFilter(include, exclude) 27 | 28 | fo := &util.FileOperations{ 29 | Operation: util.Operation{ 30 | FailOutput: failOutput, 31 | FailOutputPath: failOutputPath, 32 | Filters: filters, 33 | }, 34 | Config: &config, 35 | Param: ¶m, 36 | ErrOutput: &util.ErrOutput{}, 37 | } 38 | 39 | err := util.AbortUploads(args, fo) 40 | return err 41 | }, 42 | } 43 | 44 | func init() { 45 | rootCmd.AddCommand(abortCmd) 46 | 47 | abortCmd.Flags().String("include", "", "List files that meet the specified criteria") 48 | abortCmd.Flags().String("exclude", "", "Exclude files that meet the specified criteria") 49 | abortCmd.Flags().Bool("fail-output", true, "This option determines whether the error output for failed file uploads or downloads is enabled. If enabled, the error messages for any failed file transfers will be recorded in a file within the specified directory (if not specified, the default is coscli_output). If disabled, only the number of error files will be output to the console.") 50 | abortCmd.Flags().String("fail-output-path", "coscli_output", "This option specifies the designated error output folder where the error messages for failed file uploads or downloads will be recorded. By providing a custom folder path, you can control the location and name of the error output folder. If this option is not set, the default error log folder (coscli_output) will be used.") 51 | } 52 | -------------------------------------------------------------------------------- /cmd/abort_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "coscli/util" 6 | "fmt" 7 | . "github.com/agiledragon/gomonkey/v2" 8 | . "github.com/smartystreets/goconvey/convey" 9 | "github.com/tencentyun/cos-go-sdk-v5" 10 | "reflect" 11 | "testing" 12 | ) 13 | 14 | func TestAbortCmd(t *testing.T) { 15 | fmt.Println("TestAbortCmd") 16 | testBucket = randStr(8) 17 | testAlias = testBucket + "-alias" 18 | setUp(testBucket, testAlias, testEndpoint, false, false) 19 | defer tearDown(testBucket, testAlias, testEndpoint, false) 20 | clearCmd() 21 | cmd := rootCmd 22 | cmd.SilenceErrors = true 23 | cmd.SilenceUsage = true 24 | Convey("Test coscli abort", t, func() { 25 | Convey("success", func() { 26 | Convey("0 success 0 fail", func() { 27 | clearCmd() 28 | cmd := rootCmd 29 | args := []string{"abort", 30 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint} 31 | cmd.SetArgs(args) 32 | e := cmd.Execute() 33 | So(e, ShouldBeNil) 34 | }) 35 | Convey("1 success", func() { 36 | clearCmd() 37 | cmd := rootCmd 38 | patches := ApplyFunc(util.GetUploadsListForLs, func(c *cos.Client, cosUrl util.StorageUrl, uploadIDMarker, keyMarker string, limit int, recursive bool) (err error, uploads []struct { 39 | Key string 40 | UploadID string `xml:"UploadId"` 41 | StorageClass string 42 | Initiator *cos.Initiator 43 | Owner *cos.Owner 44 | Initiated string 45 | }, isTruncated bool, nextUploadIDMarker, nextKeyMarker string) { 46 | tmp := []struct { 47 | Key string 48 | UploadID string `xml:"UploadId"` 49 | StorageClass string 50 | Initiator *cos.Initiator 51 | Owner *cos.Owner 52 | Initiated string 53 | }{ 54 | { 55 | Key: "666", 56 | UploadID: "888", 57 | }, 58 | } 59 | 60 | return nil, tmp, false, "", "" 61 | }) 62 | defer patches.Reset() 63 | var c *cos.ObjectService 64 | patches.ApplyMethodFunc(reflect.TypeOf(c), "AbortMultipartUpload", func(ctx context.Context, name string, uploadID string, opt ...*cos.AbortMultipartUploadOptions) (*cos.Response, error) { 65 | return nil, nil 66 | }) 67 | args := []string{"abort", 68 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint} 69 | cmd.SetArgs(args) 70 | e := cmd.Execute() 71 | So(e, ShouldBeNil) 72 | }) 73 | Convey("1 fail", func() { 74 | clearCmd() 75 | cmd := rootCmd 76 | patches := ApplyFunc(util.GetUploadsListForLs, func(c *cos.Client, cosUrl util.StorageUrl, uploadIDMarker, keyMarker string, limit int, recursive bool) (err error, uploads []struct { 77 | Key string 78 | UploadID string `xml:"UploadId"` 79 | StorageClass string 80 | Initiator *cos.Initiator 81 | Owner *cos.Owner 82 | Initiated string 83 | }, isTruncated bool, nextUploadIDMarker, nextKeyMarker string) { 84 | tmp := []struct { 85 | Key string 86 | UploadID string `xml:"UploadId"` 87 | StorageClass string 88 | Initiator *cos.Initiator 89 | Owner *cos.Owner 90 | Initiated string 91 | }{ 92 | { 93 | Key: "666", 94 | UploadID: "888", 95 | }, 96 | } 97 | 98 | return nil, tmp, false, "", "" 99 | }) 100 | defer patches.Reset() 101 | var c *cos.ObjectService 102 | patches.ApplyMethodFunc(reflect.TypeOf(c), "AbortMultipartUpload", func(ctx context.Context, name string, uploadID string, opt ...*cos.AbortMultipartUploadOptions) (*cos.Response, error) { 103 | return nil, fmt.Errorf("test abort fail") 104 | }) 105 | 106 | args := []string{"abort", 107 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint} 108 | cmd.SetArgs(args) 109 | e := cmd.Execute() 110 | So(e, ShouldBeNil) 111 | }) 112 | }) 113 | Convey("failed", func() { 114 | Convey("not enough argument", func() { 115 | clearCmd() 116 | cmd := rootCmd 117 | args := []string{"abort"} 118 | cmd.SetArgs(args) 119 | e := cmd.Execute() 120 | fmt.Printf(" : %v", e) 121 | So(e, ShouldBeError) 122 | }) 123 | Convey("client fail", func() { 124 | clearCmd() 125 | cmd := rootCmd 126 | patches := ApplyFunc(util.NewClient, func(config *util.Config, param *util.Param, bucketName string) (client *cos.Client, err error) { 127 | return nil, fmt.Errorf("test abort client error") 128 | }) 129 | defer patches.Reset() 130 | args := []string{"abort", 131 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint} 132 | cmd.SetArgs(args) 133 | e := cmd.Execute() 134 | fmt.Printf(" : %v", e) 135 | So(e, ShouldBeError) 136 | }) 137 | Convey("GetUpload fail", func() { 138 | clearCmd() 139 | cmd := rootCmd 140 | patches := ApplyFunc(util.GetUploadsListForLs, func(c *cos.Client, cosUrl util.StorageUrl, uploadIDMarker, keyMarker string, limit int, recursive bool) (err error, uploads []struct { 141 | Key string 142 | UploadID string `xml:"UploadId"` 143 | StorageClass string 144 | Initiator *cos.Initiator 145 | Owner *cos.Owner 146 | Initiated string 147 | }, isTruncated bool, nextUploadIDMarker, nextKeyMarker string) { 148 | return fmt.Errorf("test GetUpload client error"), nil, false, "", "" 149 | }) 150 | defer patches.Reset() 151 | args := []string{"abort", 152 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint} 153 | cmd.SetArgs(args) 154 | e := cmd.Execute() 155 | fmt.Printf(" : %v", e) 156 | So(e, ShouldBeError) 157 | }) 158 | }) 159 | }) 160 | } 161 | -------------------------------------------------------------------------------- /cmd/bucket_tagging.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "coscli/util" 6 | "fmt" 7 | "os" 8 | "strings" 9 | 10 | "github.com/olekukonko/tablewriter" 11 | "github.com/spf13/cobra" 12 | "github.com/tencentyun/cos-go-sdk-v5" 13 | ) 14 | 15 | var bucketTaggingCmd = &cobra.Command{ 16 | Use: "bucket-tagging", 17 | Short: "Modify bucket tagging", 18 | Long: `Modify bucket tagging 19 | 20 | Format: 21 | ./coscli bucket-tagging --method [method] cos:// [tag_key]#[tag_value] 22 | 23 | Example: 24 | ./coscli bucket-tagging --method put cos://examplebucket tag1#test1 tag2#test2 25 | ./coscli bucket-tagging --method get cos://examplebucket 26 | ./coscli bucket-tagging --method delete cos://examplebucket 27 | ./coscli bucket-tagging --method delete cos://examplebucket tag1#test1 tag2#test2`, 28 | RunE: func(cmd *cobra.Command, args []string) error { 29 | method, _ := cmd.Flags().GetString("method") 30 | 31 | var err error 32 | if method == "put" { 33 | if len(args) < 2 { 34 | return fmt.Errorf("not enough arguments in call to put bucket tagging") 35 | } 36 | err = putBucketTagging(args[0], args[1:]) 37 | } 38 | 39 | if method == "get" { 40 | if len(args) < 1 { 41 | return fmt.Errorf("not enough arguments in call to get bucket tagging") 42 | } 43 | err = getBucketTagging(args[0]) 44 | } 45 | 46 | if method == "delete" { 47 | if len(args) < 1 { 48 | return fmt.Errorf("not enough arguments in call to delete bucket tagging") 49 | } else if len(args) == 1 { 50 | err = deleteBucketTagging(args[0]) 51 | } else { 52 | err = deleteDesBucketTagging(args[0], args[1:]) 53 | } 54 | } 55 | 56 | return err 57 | }, 58 | } 59 | 60 | func init() { 61 | rootCmd.AddCommand(bucketTaggingCmd) 62 | bucketTaggingCmd.Flags().String("method", "", "put/get/delete") 63 | } 64 | 65 | func putBucketTagging(cosPath string, tags []string) error { 66 | bucketName, _ := util.ParsePath(cosPath) 67 | c, err := util.NewClient(&config, ¶m, bucketName) 68 | if err != nil { 69 | return err 70 | } 71 | tg := &cos.BucketPutTaggingOptions{} 72 | for i := 0; i < len(tags); i += 1 { 73 | tmp := strings.Split(tags[i], "#") 74 | if len(tmp) >= 2 { 75 | tg.TagSet = append(tg.TagSet, cos.BucketTaggingTag{Key: tmp[0], Value: tmp[1]}) 76 | } else { 77 | return fmt.Errorf("invalid tag") 78 | } 79 | } 80 | 81 | _, err = c.Bucket.PutTagging(context.Background(), tg) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | return nil 87 | } 88 | 89 | func getBucketTagging(cosPath string) error { 90 | bucketName, _ := util.ParsePath(cosPath) 91 | c, err := util.NewClient(&config, ¶m, bucketName) 92 | if err != nil { 93 | return err 94 | } 95 | 96 | v, _, err := c.Bucket.GetTagging(context.Background()) 97 | if err != nil { 98 | return err 99 | } 100 | table := tablewriter.NewWriter(os.Stdout) 101 | table.SetHeader([]string{"Key", "Value"}) 102 | for _, t := range v.TagSet { 103 | table.Append([]string{t.Key, t.Value}) 104 | } 105 | table.SetBorder(false) 106 | table.SetAlignment(tablewriter.ALIGN_RIGHT) 107 | table.Render() 108 | 109 | return nil 110 | } 111 | 112 | func deleteBucketTagging(cosPath string) error { 113 | bucketName, _ := util.ParsePath(cosPath) 114 | c, err := util.NewClient(&config, ¶m, bucketName) 115 | if err != nil { 116 | return err 117 | } 118 | 119 | _, err = c.Bucket.DeleteTagging(context.Background()) 120 | if err != nil { 121 | return err 122 | } 123 | return nil 124 | } 125 | 126 | func deleteDesBucketTagging(cosPath string, tags []string) error { 127 | bucketName, _ := util.ParsePath(cosPath) 128 | c, err := util.NewClient(&config, ¶m, bucketName) 129 | if err != nil { 130 | return err 131 | } 132 | d, _, err := c.Bucket.GetTagging(context.Background()) 133 | if err != nil { 134 | return err 135 | } 136 | table := make(map[string]string) 137 | for _, t := range d.TagSet { 138 | table[t.Key] = t.Value 139 | } 140 | var del []string 141 | for i := 0; i < len(tags); i += 1 { 142 | tmp := strings.Split(tags[i], "#") 143 | if len(tmp) >= 2 { 144 | _, ok := table[tmp[0]] 145 | del = append(del, tmp[0]) 146 | if ok { 147 | delete(table, tmp[0]) 148 | } else { 149 | return fmt.Errorf("the BucketTagging %s is not exist", tmp[0]) 150 | } 151 | } else { 152 | return fmt.Errorf("invalid tag") 153 | } 154 | } 155 | tg := &cos.BucketPutTaggingOptions{} 156 | for a, b := range table { 157 | tg.TagSet = append(tg.TagSet, cos.BucketTaggingTag{Key: a, Value: b}) 158 | } 159 | 160 | _, err = c.Bucket.PutTagging(context.Background(), tg) 161 | if err != nil { 162 | return err 163 | } 164 | return nil 165 | } 166 | -------------------------------------------------------------------------------- /cmd/bucket_versioning.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | logger "github.com/sirupsen/logrus" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | var bucketVersioningCmd = &cobra.Command{ 11 | Use: "bucket-versioning", 12 | Short: "Modify bucket versioning", 13 | Long: `Modify bucket versioning 14 | 15 | Format: 16 | ./coscli bucket-versioning --method [method] cos:// versioning 17 | 18 | Example: 19 | ./coscli bucket-versioning --method put cos://examplebucket versioning 20 | ./coscli bucket-versioning --method get cos://examplebucket`, 21 | RunE: func(cmd *cobra.Command, args []string) error { 22 | method, _ := cmd.Flags().GetString("method") 23 | 24 | cosUrl, err := util.FormatUrl(args[0]) 25 | if err != nil { 26 | return fmt.Errorf("cos url format error:%v", err) 27 | } 28 | 29 | if !cosUrl.IsCosUrl() { 30 | return fmt.Errorf("cospath needs to contain cos://") 31 | } 32 | 33 | bucketName := cosUrl.(*util.CosUrl).Bucket 34 | 35 | c, err := util.NewClient(&config, ¶m, bucketName) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | if method == "put" { 41 | if len(args) < 2 { 42 | return fmt.Errorf("not enough arguments in call to put bucket versioning") 43 | } 44 | status := args[1] 45 | if status != util.VersionStatusEnabled && status != util.VersionStatusSuspended { 46 | return fmt.Errorf("the bucket versioning status can only be either Suspended or Enabled") 47 | } 48 | 49 | _, err := util.PutBucketVersioning(c, status) 50 | if err != nil { 51 | return err 52 | } 53 | logger.Infof("the bucket versioning status has been changed to %s", status) 54 | } 55 | 56 | if method == "get" { 57 | res, _, err := util.GetBucketVersioning(c) 58 | if err != nil { 59 | return err 60 | } 61 | switch res.Status { 62 | case util.VersionStatusEnabled, util.VersionStatusSuspended: 63 | logger.Infof("bucket versioning status is %s", res.Status) 64 | default: 65 | logger.Infof("bucket versioning status is Closed") 66 | } 67 | } 68 | 69 | return err 70 | }, 71 | } 72 | 73 | func init() { 74 | rootCmd.AddCommand(bucketVersioningCmd) 75 | bucketVersioningCmd.Flags().String("method", "", "put/get") 76 | } 77 | -------------------------------------------------------------------------------- /cmd/buket_versioning_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | "testing" 7 | 8 | . "github.com/agiledragon/gomonkey/v2" 9 | . "github.com/smartystreets/goconvey/convey" 10 | "github.com/tencentyun/cos-go-sdk-v5" 11 | ) 12 | 13 | func TestBucketVersionCmd(t *testing.T) { 14 | fmt.Println("TestBucketVersionCmd") 15 | testBucket = randStr(8) 16 | testAlias = testBucket + "-alias" 17 | setUp(testBucket, testAlias, testEndpoint, false, false) 18 | defer tearDown(testBucket, testAlias, testEndpoint, false) 19 | clearCmd() 20 | cmd := rootCmd 21 | cmd.SilenceErrors = true 22 | cmd.SilenceUsage = true 23 | Convey("test coscli bucket_versioning", t, func() { 24 | Convey("success", func() { 25 | Convey("put", func() { 26 | clearCmd() 27 | cmd := rootCmd 28 | args := []string{"bucket-versioning", "--method", "put", 29 | fmt.Sprintf("cos://%s", testAlias), "Enabled"} 30 | cmd.SetArgs(args) 31 | e := cmd.Execute() 32 | So(e, ShouldBeNil) 33 | }) 34 | Convey("get", func() { 35 | clearCmd() 36 | cmd := rootCmd 37 | args := []string{"bucket-versioning", "--method", "get", 38 | fmt.Sprintf("cos://%s", testAlias)} 39 | cmd.SetArgs(args) 40 | e := cmd.Execute() 41 | So(e, ShouldBeNil) 42 | }) 43 | Convey("get status closed", func() { 44 | clearCmd() 45 | cmd := rootCmd 46 | patches := ApplyFunc(util.GetBucketVersioning, func(c *cos.Client) (res *cos.BucketGetVersionResult, resp *cos.Response, err error) { 47 | return &cos.BucketGetVersionResult{}, nil, nil 48 | }) 49 | defer patches.Reset() 50 | args := []string{"bucket-versioning", "--method", "get", 51 | fmt.Sprintf("cos://%s", testAlias)} 52 | cmd.SetArgs(args) 53 | e := cmd.Execute() 54 | So(e, ShouldBeNil) 55 | }) 56 | }) 57 | Convey("fail", func() { 58 | Convey("cos url error", func() { 59 | clearCmd() 60 | cmd := rootCmd 61 | args := []string{"bucket-versioning", "--method", "put", 62 | fmt.Sprintf("co:/%s", testAlias), "Enabled"} 63 | cmd.SetArgs(args) 64 | e := cmd.Execute() 65 | fmt.Printf(" : %v", e) 66 | So(e, ShouldBeError) 67 | }) 68 | Convey("cos url format error", func() { 69 | clearCmd() 70 | cmd := rootCmd 71 | patches := ApplyFunc(util.FormatUrl, func(urlStr string) (util.StorageUrl, error) { 72 | return nil, fmt.Errorf("cos url format error") 73 | }) 74 | defer patches.Reset() 75 | args := []string{"bucket-versioning", "--method", "put", 76 | fmt.Sprintf("cos://%s", testAlias), "Enabled"} 77 | cmd.SetArgs(args) 78 | e := cmd.Execute() 79 | fmt.Printf(" : %v", e) 80 | So(e, ShouldBeError) 81 | }) 82 | Convey("put", func() { 83 | Convey("not enough arguments", func() { 84 | clearCmd() 85 | cmd := rootCmd 86 | args := []string{"bucket-versioning", "--method", "put", 87 | fmt.Sprintf("cos://%s", testAlias)} 88 | cmd.SetArgs(args) 89 | e := cmd.Execute() 90 | fmt.Printf(" : %v", e) 91 | So(e, ShouldBeError) 92 | }) 93 | Convey("clinet err", func() { 94 | clearCmd() 95 | cmd := rootCmd 96 | patches := ApplyFunc(util.NewClient, func(config *util.Config, param *util.Param, bucketName string) (client *cos.Client, err error) { 97 | return nil, fmt.Errorf("test put client error") 98 | }) 99 | defer patches.Reset() 100 | args := []string{"bucket-versioning", "--method", "put", 101 | fmt.Sprintf("cos://%s", testAlias), "Enabled"} 102 | cmd.SetArgs(args) 103 | e := cmd.Execute() 104 | fmt.Printf(" : %v", e) 105 | So(e, ShouldBeError) 106 | }) 107 | Convey("invalid status", func() { 108 | clearCmd() 109 | cmd := rootCmd 110 | args := []string{"bucket-versioning", "--method", "put", 111 | fmt.Sprintf("cos://%s", testAlias), "testStatus"} 112 | cmd.SetArgs(args) 113 | e := cmd.Execute() 114 | fmt.Printf(" : %v", e) 115 | So(e, ShouldBeError) 116 | }) 117 | Convey("put bucket versioning error", func() { 118 | clearCmd() 119 | cmd := rootCmd 120 | patches := ApplyFunc(util.PutBucketVersioning, func(c *cos.Client, status string) (res *cos.Response, err error) { 121 | return nil, fmt.Errorf("put bucket versioning error") 122 | }) 123 | defer patches.Reset() 124 | args := []string{"bucket-versioning", "--method", "put", 125 | fmt.Sprintf("cos://%s", testAlias), "Enabled"} 126 | cmd.SetArgs(args) 127 | e := cmd.Execute() 128 | fmt.Printf(" : %v", e) 129 | So(e, ShouldBeError) 130 | }) 131 | }) 132 | Convey("get", func() { 133 | Convey("clinet err", func() { 134 | clearCmd() 135 | cmd := rootCmd 136 | patches := ApplyFunc(util.NewClient, func(config *util.Config, param *util.Param, bucketName string) (client *cos.Client, err error) { 137 | return nil, fmt.Errorf("test get client error") 138 | }) 139 | defer patches.Reset() 140 | args := []string{"bucket-versioning", "--method", "get", 141 | fmt.Sprintf("cos://%s", testAlias)} 142 | cmd.SetArgs(args) 143 | e := cmd.Execute() 144 | fmt.Printf(" : %v", e) 145 | So(e, ShouldBeError) 146 | }) 147 | Convey("get bucket versioning error", func() { 148 | clearCmd() 149 | cmd := rootCmd 150 | patches := ApplyFunc(util.GetBucketVersioning, func(c *cos.Client) (res *cos.BucketGetVersionResult, resp *cos.Response, err error) { 151 | return nil, nil, fmt.Errorf("get bucket versioning error") 152 | }) 153 | defer patches.Reset() 154 | args := []string{"bucket-versioning", "--method", "get", 155 | fmt.Sprintf("cos://%s", testAlias)} 156 | cmd.SetArgs(args) 157 | e := cmd.Execute() 158 | fmt.Printf(" : %v", e) 159 | So(e, ShouldBeError) 160 | }) 161 | }) 162 | }) 163 | }) 164 | } 165 | -------------------------------------------------------------------------------- /cmd/cat.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | var catCmd = &cobra.Command{ 11 | Use: "cat", 12 | Short: "Cat object info", 13 | Long: `Cat object info 14 | 15 | Format: 16 | ./coscli cat cos://-/ 17 | 18 | Example: 19 | ./coscli cat cos://examplebucket-1234567890/test.txt`, 20 | Args: func(cmd *cobra.Command, args []string) error { 21 | if err := cobra.ExactArgs(1)(cmd, args); err != nil { 22 | return err 23 | } 24 | return nil 25 | }, 26 | RunE: func(cmd *cobra.Command, args []string) error { 27 | cosUrl, err := util.FormatUrl(args[0]) 28 | if err != nil { 29 | return err 30 | } 31 | if !cosUrl.IsCosUrl() { 32 | return fmt.Errorf("cospath needs to contain cos://") 33 | } 34 | 35 | // 实例化cos client 36 | bucketName := cosUrl.(*util.CosUrl).Bucket 37 | c, err := util.NewClient(&config, ¶m, bucketName) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | err = util.CatObject(c, cosUrl) 43 | return err 44 | }, 45 | } 46 | 47 | func init() { 48 | rootCmd.AddCommand(catCmd) 49 | } 50 | -------------------------------------------------------------------------------- /cmd/cat_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | "testing" 7 | 8 | . "github.com/agiledragon/gomonkey/v2" 9 | . "github.com/smartystreets/goconvey/convey" 10 | "github.com/tencentyun/cos-go-sdk-v5" 11 | ) 12 | 13 | func TestCatCmd(t *testing.T) { 14 | fmt.Println("TestCatCmd") 15 | testBucket = randStr(8) 16 | testAlias = testBucket + "-alias" 17 | setUp(testBucket, testAlias, testEndpoint, false, false) 18 | defer tearDown(testBucket, testAlias, testEndpoint, false) 19 | clearCmd() 20 | cmd := rootCmd 21 | cmd.SilenceErrors = true 22 | cmd.SilenceUsage = true 23 | 24 | genDir(testDir, 3) 25 | defer delDir(testDir) 26 | 27 | Convey("test coscli cat", t, func() { 28 | Convey("上传测试单文件", func() { 29 | clearCmd() 30 | cmd := rootCmd 31 | localFileName := fmt.Sprintf("%s/small-file/0", testDir) 32 | cosFileName := fmt.Sprintf("cos://%s/%s", testAlias, "single-small") 33 | args := []string{"cp", localFileName, cosFileName} 34 | cmd.SetArgs(args) 35 | e := cmd.Execute() 36 | So(e, ShouldBeNil) 37 | }) 38 | Convey("fail", func() { 39 | Convey("Not enough argument", func() { 40 | clearCmd() 41 | cmd := rootCmd 42 | args := []string{"cat"} 43 | cmd.SetArgs(args) 44 | e := cmd.Execute() 45 | fmt.Printf(" : %v", e) 46 | So(e, ShouldBeError) 47 | }) 48 | Convey("FormatUrl", func() { 49 | patches := ApplyFunc(util.FormatUrl, func(urlStr string) (util.StorageUrl, error) { 50 | return nil, fmt.Errorf("test FormatUrl error") 51 | }) 52 | defer patches.Reset() 53 | clearCmd() 54 | cmd := rootCmd 55 | args := []string{"cat", fmt.Sprintf("cos://%s", testAlias)} 56 | cmd.SetArgs(args) 57 | e := cmd.Execute() 58 | fmt.Printf(" : %v", e) 59 | So(e, ShouldBeError) 60 | }) 61 | Convey("Not CosUrl", func() { 62 | clearCmd() 63 | cmd := rootCmd 64 | args := []string{"cat", testAlias} 65 | cmd.SetArgs(args) 66 | e := cmd.Execute() 67 | fmt.Printf(" : %v", e) 68 | So(e, ShouldBeError) 69 | }) 70 | Convey("NewClient", func() { 71 | patches := ApplyFunc(util.NewClient, func(config *util.Config, param *util.Param, bucketName string) (client *cos.Client, err error) { 72 | return nil, fmt.Errorf("test NewClient error") 73 | }) 74 | defer patches.Reset() 75 | clearCmd() 76 | cmd := rootCmd 77 | args := []string{"cat", fmt.Sprintf("cos://%s", testAlias)} 78 | cmd.SetArgs(args) 79 | e := cmd.Execute() 80 | fmt.Printf(" : %v", e) 81 | So(e, ShouldBeError) 82 | }) 83 | Convey("CatObject", func() { 84 | patches := ApplyFunc(util.CatObject, func(c *cos.Client, cosUrl util.StorageUrl) error { 85 | return fmt.Errorf("test CatObject error") 86 | }) 87 | defer patches.Reset() 88 | clearCmd() 89 | cmd := rootCmd 90 | args := []string{"cat", fmt.Sprintf("cos://%s", testAlias)} 91 | cmd.SetArgs(args) 92 | e := cmd.Execute() 93 | fmt.Printf(" : %v", e) 94 | So(e, ShouldBeError) 95 | }) 96 | }) 97 | Convey("success", func() { 98 | clearCmd() 99 | cmd := rootCmd 100 | args := []string{"cat", fmt.Sprintf("cos://%s/%s", testAlias, "single-small")} 101 | cmd.SetArgs(args) 102 | e := cmd.Execute() 103 | So(e, ShouldBeNil) 104 | }) 105 | }) 106 | } 107 | -------------------------------------------------------------------------------- /cmd/config.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | ) 6 | 7 | var configCmd = &cobra.Command{ 8 | Use: "config", 9 | Short: "Init or edit config file", 10 | Long: "Init or edit config file", 11 | Run: func(cmd *cobra.Command, args []string) { 12 | _ = cmd.Help() 13 | }, 14 | } 15 | 16 | func init() { 17 | rootCmd.AddCommand(configCmd) 18 | } 19 | -------------------------------------------------------------------------------- /cmd/config_add.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "github.com/mitchellh/go-homedir" 6 | "os" 7 | 8 | "coscli/util" 9 | 10 | logger "github.com/sirupsen/logrus" 11 | "github.com/spf13/cobra" 12 | "github.com/spf13/viper" 13 | ) 14 | 15 | var configAddCmd = &cobra.Command{ 16 | Use: "add", 17 | Short: "Used to add a new bucket configuration", 18 | Long: `Used to add a new bucket configuration 19 | 20 | Format: 21 | ./coscli config add -b -e -a [-c ] 22 | 23 | Example: 24 | ./coscli config add -b example-1234567890 -r ap-shanghai -a example`, 25 | RunE: func(cmd *cobra.Command, args []string) error { 26 | err := addBucketConfig(cmd) 27 | return err 28 | }, 29 | } 30 | 31 | func init() { 32 | configCmd.AddCommand(configAddCmd) 33 | 34 | configAddCmd.Flags().StringP("bucket", "b", "", "Bucket name") 35 | configAddCmd.Flags().StringP("endpoint", "e", "", "Bucket endpoint") 36 | configAddCmd.Flags().StringP("region", "r", "", "Bucket region") 37 | configAddCmd.Flags().StringP("alias", "a", "", "Bucket alias") 38 | configAddCmd.Flags().BoolP("ofs", "o", false, "Bucket ofs") 39 | 40 | _ = configAddCmd.MarkFlagRequired("bucket") 41 | // _ = configAddCmd.MarkFlagRequired("endpoint") 42 | } 43 | 44 | func addBucketConfig(cmd *cobra.Command) error { 45 | name, _ := cmd.Flags().GetString("bucket") 46 | endpoint, _ := cmd.Flags().GetString("endpoint") 47 | region, _ := cmd.Flags().GetString("region") 48 | alias, _ := cmd.Flags().GetString("alias") 49 | ofs, _ := cmd.Flags().GetBool("ofs") 50 | 51 | if alias == "" { 52 | alias = name 53 | } 54 | bucket := util.Bucket{ 55 | Name: name, 56 | Endpoint: endpoint, 57 | Region: region, 58 | Alias: alias, 59 | Ofs: ofs, 60 | } 61 | 62 | for _, b := range config.Buckets { 63 | if name == b.Name { 64 | return fmt.Errorf("The bucket already exists, fail to add!") 65 | } else if alias == b.Name { 66 | return fmt.Errorf("The alias cannot be the same as other bucket's name") 67 | } else if alias == b.Alias { 68 | return fmt.Errorf("The alias already exists, fail to add!") 69 | } 70 | } 71 | 72 | config.Buckets = append(config.Buckets, bucket) 73 | viper.Set("cos.buckets", config.Buckets) 74 | 75 | // 判断config文件是否存在。不存在则创建 76 | home, err := homedir.Dir() 77 | configFile := "" 78 | if cfgFile != "" { 79 | if cfgFile[0] == '~' { 80 | configFile = home + cfgFile[1:] 81 | } else { 82 | configFile = cfgFile 83 | } 84 | } else { 85 | configFile = home + "/.cos.yaml" 86 | } 87 | _, err = os.Stat(configFile) 88 | if os.IsNotExist(err) || cfgFile != "" { 89 | if err := viper.WriteConfigAs(configFile); err != nil { 90 | return err 91 | } 92 | } else { 93 | if err := viper.WriteConfigAs(viper.ConfigFileUsed()); err != nil { 94 | return err 95 | } 96 | } 97 | logger.Infof("Add successfully! name: %s, endpoint: %s, alias: %s\n, ofs: %t\n", name, endpoint, alias, ofs) 98 | return nil 99 | } 100 | -------------------------------------------------------------------------------- /cmd/config_add_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | . "github.com/agiledragon/gomonkey/v2" 8 | . "github.com/smartystreets/goconvey/convey" 9 | "github.com/spf13/viper" 10 | ) 11 | 12 | func TestConfigAddCmd(t *testing.T) { 13 | fmt.Println("TestConfigAddCmd") 14 | testBucket = randStr(8) 15 | testAlias = testBucket + "-alias" 16 | setUp(testBucket, testAlias, testEndpoint, false, false) 17 | defer tearDown(testBucket, testAlias, testEndpoint, false) 18 | copyYaml() 19 | defer delYaml() 20 | clearCmd() 21 | cmd := rootCmd 22 | cmd.SilenceErrors = true 23 | cmd.SilenceUsage = true 24 | Convey("Test coscil config add", t, func() { 25 | // 成功不需要测，setup里就用过了 26 | // Convey("success", func() { 27 | // Convey("All have", func() { 28 | // cmd := rootCmd 29 | // args := []string{"config", "add", "-b", 30 | // fmt.Sprintf("%s-%s", testBucket, appID), "-e", testEndpoint, "-a", testAlias} 31 | // cmd.SetArgs(args) 32 | // e := cmd.Execute() 33 | // So(e, ShouldBeNil) 34 | // }) 35 | // }) 36 | Convey("fail", func() { 37 | Convey("Bucket already exist: name", func() { 38 | clearCmd() 39 | cmd := rootCmd 40 | args := []string{"config", "add", "-b", 41 | fmt.Sprintf("%s-%s", testBucket, appID), "-e", testEndpoint, "-a", "testAlias"} 42 | cmd.SetArgs(args) 43 | e := cmd.Execute() 44 | fmt.Printf(" : %v", e) 45 | So(e, ShouldBeError) 46 | }) 47 | Convey("Bucket already exist: alias-name", func() { 48 | clearCmd() 49 | cmd := rootCmd 50 | args := []string{"config", "add", "-b", 51 | fmt.Sprintf("%s-%s", "testBucket", appID), "-e", testEndpoint, "-a", fmt.Sprintf("%s-%s", testBucket, appID)} 52 | cmd.SetArgs(args) 53 | e := cmd.Execute() 54 | fmt.Printf(" : %v", e) 55 | So(e, ShouldBeError) 56 | }) 57 | Convey("Bucket already exist: alias", func() { 58 | clearCmd() 59 | cmd := rootCmd 60 | args := []string{"config", "add", "-b", 61 | fmt.Sprintf("%s-%s", "testBucket", appID), "-e", testEndpoint, "-a", testAlias} 62 | cmd.SetArgs(args) 63 | e := cmd.Execute() 64 | fmt.Printf(" : %v", e) 65 | So(e, ShouldBeError) 66 | }) 67 | Convey("WriteConfigAs", func() { 68 | patches := ApplyFunc(viper.WriteConfigAs, func(string) error { 69 | config.Buckets = config.Buckets[:len(config.Buckets)-1] 70 | viper.Set("cos.buckets", config.Buckets) 71 | return fmt.Errorf("test write configas error") 72 | }) 73 | defer patches.Reset() 74 | Convey("cfgFile[0]!=~", func() { 75 | clearCmd() 76 | cmd := rootCmd 77 | args := []string{"config", "add", "-b", 78 | fmt.Sprintf("%s-%s", "testBucket", appID), "-e", testEndpoint, "-a", "testAlias", "-c", "./test.yaml"} 79 | cmd.SetArgs(args) 80 | e := cmd.Execute() 81 | fmt.Printf(" : %v", e) 82 | So(e, ShouldBeError) 83 | }) 84 | Convey("no cfgFile", func() { 85 | clearCmd() 86 | cmd := rootCmd 87 | args := []string{"config", "add", "-b", 88 | fmt.Sprintf("%s-%s", "testBucket", appID), "-e", testEndpoint, "-a", ""} 89 | cmd.SetArgs(args) 90 | e := cmd.Execute() 91 | fmt.Printf(" : %v", e) 92 | So(e, ShouldBeError) 93 | }) 94 | }) 95 | }) 96 | }) 97 | } 98 | -------------------------------------------------------------------------------- /cmd/config_delete.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | logger "github.com/sirupsen/logrus" 7 | "github.com/spf13/cobra" 8 | "github.com/spf13/viper" 9 | ) 10 | 11 | var configDeleteCmd = &cobra.Command{ 12 | Use: "delete", 13 | Short: "Used to delete an existing bucket", 14 | Long: `Used to delete an existing bucket 15 | 16 | Format: 17 | ./coscli config delete -a [-c ] 18 | 19 | Example: 20 | ./coscli config delete -a example`, 21 | RunE: func(cmd *cobra.Command, args []string) error { 22 | err := deleteBucketConfig(cmd) 23 | return err 24 | }, 25 | } 26 | 27 | func init() { 28 | configCmd.AddCommand(configDeleteCmd) 29 | 30 | configDeleteCmd.Flags().StringP("alias", "a", "", "Bucket alias") 31 | 32 | _ = configDeleteCmd.MarkFlagRequired("alias") 33 | } 34 | 35 | func deleteBucketConfig(cmd *cobra.Command) error { 36 | alias, _ := cmd.Flags().GetString("alias") 37 | b, i, err := util.FindBucket(&config, alias) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | if i < 0 { 43 | return fmt.Errorf("Bucket not exist in config file!") 44 | } 45 | config.Buckets = append(config.Buckets[:i], config.Buckets[i+1:]...) 46 | 47 | viper.Set("cos.buckets", config.Buckets) 48 | if err := viper.WriteConfigAs(viper.ConfigFileUsed()); err != nil { 49 | return err 50 | } 51 | logger.Infof("Delete successfully! name: %s, endpoint: %s, alias: %s", b.Name, b.Endpoint, b.Alias) 52 | return nil 53 | } 54 | -------------------------------------------------------------------------------- /cmd/config_delete_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | "testing" 7 | 8 | . "github.com/agiledragon/gomonkey/v2" 9 | . "github.com/smartystreets/goconvey/convey" 10 | "github.com/spf13/viper" 11 | ) 12 | 13 | func TestConfigDeleteCmd(t *testing.T) { 14 | fmt.Println("TestConfigDeleteCmd") 15 | // 恢复原来的 Buckets 16 | clearCmd() 17 | cmd := rootCmd 18 | buckets := config.Buckets 19 | defer func() { 20 | viper.Set("cos.buckets", buckets) 21 | viper.WriteConfigAs(viper.ConfigFileUsed()) 22 | }() 23 | cmd.SilenceErrors = true 24 | cmd.SilenceUsage = true 25 | Convey("Test coscil config delete", t, func() { 26 | Convey("fail", func() { 27 | Convey("FindBucket", func() { 28 | clearCmd() 29 | cmd := rootCmd 30 | patches := ApplyFunc(util.FindBucket, func(config *util.Config, bucketName string) (util.Bucket, int, error) { 31 | return util.Bucket{}, 0, fmt.Errorf("test findbucket fail") 32 | }) 33 | defer patches.Reset() 34 | args := []string{"config", "delete", "-a", "testAlias"} 35 | cmd.SetArgs(args) 36 | e := cmd.Execute() 37 | fmt.Printf(" : %v", e) 38 | So(e, ShouldBeError) 39 | }) 40 | Convey("FindBucket i<0", func() { 41 | clearCmd() 42 | cmd := rootCmd 43 | patches := ApplyFunc(util.FindBucket, func(config *util.Config, bucketName string) (util.Bucket, int, error) { 44 | return util.Bucket{}, -1, nil 45 | }) 46 | defer patches.Reset() 47 | args := []string{"config", "delete", "-a", "testAlias"} 48 | cmd.SetArgs(args) 49 | e := cmd.Execute() 50 | fmt.Printf(" : %v", e) 51 | So(e, ShouldBeError) 52 | }) 53 | Convey("viper.WriteConfigAs", func() { 54 | clearCmd() 55 | cmd := rootCmd 56 | patches := ApplyFunc(viper.WriteConfigAs, func(string) error { 57 | return fmt.Errorf("test WriteConfigAs fail") 58 | }) 59 | defer patches.Reset() 60 | patches.ApplyFunc(util.FindBucket, func(config *util.Config, bucketName string) (util.Bucket, int, error) { 61 | return util.Bucket{}, 0, nil 62 | }) 63 | args := []string{"config", "delete", "-a", "testAlias"} 64 | cmd.SetArgs(args) 65 | e := cmd.Execute() 66 | fmt.Printf(" : %v", e) 67 | So(e, ShouldBeError) 68 | }) 69 | }) 70 | }) 71 | } 72 | -------------------------------------------------------------------------------- /cmd/config_init.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | "github.com/spf13/viper" 7 | 8 | homedir "github.com/mitchellh/go-homedir" 9 | "github.com/spf13/cobra" 10 | ) 11 | 12 | var configInitCmd = &cobra.Command{ 13 | Use: "init", 14 | Short: "Used to interactively generate the configuration file", 15 | Long: `Used to interactively generate the configuration file 16 | 17 | Format: 18 | ./coscli config init [-c ] 19 | 20 | Example: 21 | ./coscli config init`, 22 | RunE: func(cmd *cobra.Command, args []string) error { 23 | if cmdCnt >= 1 { 24 | return nil 25 | } 26 | err := initConfigFile(true) 27 | return err 28 | }, 29 | } 30 | 31 | func init() { 32 | configCmd.AddCommand(configInitCmd) 33 | } 34 | 35 | // cfgFlag: 是否允许用户自定义配置文件的输出路径 36 | func initConfigFile(cfgFlag bool) error { 37 | var ( 38 | configFile string 39 | config util.Config 40 | bucket util.Bucket 41 | ) 42 | home, _ := homedir.Dir() 43 | 44 | if cfgFlag { 45 | fmt.Println("Specify the path of the configuration file: (default:" + home + "/.cos.yaml)") 46 | _, _ = fmt.Scanf("%s\n", &configFile) 47 | } 48 | if configFile == "" { 49 | configFile = home + "/.cos.yaml" 50 | } 51 | if configFile[0] == '~' { 52 | configFile = home + configFile[1:] 53 | } 54 | fmt.Println("The path of the configuration file: " + configFile) 55 | 56 | fmt.Println("Input Your Mode:") 57 | _, _ = fmt.Scanf("%s\n", &config.Base.Mode) 58 | 59 | if config.Base.Mode == "CvmRole" { 60 | fmt.Println("Input Your Cvm Role Name:") 61 | _, _ = fmt.Scanf("%s\n", &config.Base.CvmRoleName) 62 | } else { 63 | fmt.Println("Input Your Secret ID:") 64 | _, _ = fmt.Scanf("%s\n", &config.Base.SecretID) 65 | fmt.Println("Input Your Secret Key:") 66 | _, _ = fmt.Scanf("%s\n", &config.Base.SecretKey) 67 | fmt.Println("Input Your Session Token:") 68 | _, _ = fmt.Scanf("%s\n", &config.Base.SessionToken) 69 | fmt.Println("Input Disable Encryption:") 70 | _, _ = fmt.Scanf("%s\n", &config.Base.DisableEncryption) 71 | } 72 | 73 | fmt.Println("Input Auto Switch Host:") 74 | _, _ = fmt.Scanf("%s\n", &config.Base.CloseAutoSwitchHost) 75 | if len(config.Base.SessionToken) < 3 { 76 | config.Base.SessionToken = "" 77 | } 78 | config.Base.Protocol = "https" 79 | 80 | fmt.Println("Input Your Bucket's Name:") 81 | fmt.Println("Format: -，Example: example-1234567890") 82 | _, _ = fmt.Scanf("%s\n", &bucket.Name) 83 | fmt.Println("Input Bucket's Endpoint:") 84 | fmt.Println("Format: cos..myqcloud.com，Example: cos.ap-beijing.myqcloud.com") 85 | _, _ = fmt.Scanf("%s\n", &bucket.Endpoint) 86 | fmt.Println("Input Bucket's Alias: (Input nothing will use the original name)") 87 | _, _ = fmt.Scanf("%s\n", &bucket.Alias) 88 | if bucket.Alias == "" { 89 | bucket.Alias = bucket.Name 90 | } 91 | 92 | config.Buckets = append(config.Buckets, bucket) 93 | fmt.Println("You have configured the bucket:") 94 | for _, b := range config.Buckets { 95 | fmt.Printf("- Name: %s\tEndpoint: %s\tAlias: %s\n", b.Name, b.Endpoint, b.Alias) 96 | } 97 | fmt.Printf("\nIf you want to configure more buckets, you can use the \"config add\" command later.\n") 98 | // 默认加密存储 99 | if config.Base.DisableEncryption != "true" { 100 | config.Base.SecretKey, _ = util.EncryptSecret(config.Base.SecretKey) 101 | config.Base.SecretID, _ = util.EncryptSecret(config.Base.SecretID) 102 | config.Base.SessionToken, _ = util.EncryptSecret(config.Base.SessionToken) 103 | } 104 | 105 | viper.Set("cos", config) 106 | 107 | if err := viper.WriteConfigAs(configFile); err != nil { 108 | return err 109 | } 110 | fmt.Printf("\nThe configuration file is initialized successfully! \nYou can use \"./coscli config show [-c ]\" show the contents of the specified configuration file\n") 111 | return nil 112 | } 113 | -------------------------------------------------------------------------------- /cmd/config_init_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | . "github.com/agiledragon/gomonkey/v2" 8 | . "github.com/smartystreets/goconvey/convey" 9 | ) 10 | 11 | func TestInit(t *testing.T) { 12 | fmt.Println("TestConfigInit") 13 | //Convey("success", t, func() { 14 | // cmd := rootCmd 15 | // cmd.SilenceErrors = true 16 | // cmd.SilenceUsage = true 17 | // args := []string{"config", "init"} 18 | // cmd.SetArgs(args) 19 | // e := cmd.Execute() 20 | // fmt.Printf(" : %v", e) 21 | // So(e, ShouldBeNil) 22 | //}) 23 | Convey("fail", t, func() { 24 | patches := ApplyFunc(initConfigFile, func(cfgFlag bool) error { 25 | return fmt.Errorf("test initConfigFile error") 26 | }) 27 | defer patches.Reset() 28 | clearCmd() 29 | cmd := rootCmd 30 | cmd.SilenceErrors = true 31 | cmd.SilenceUsage = true 32 | args := []string{"config", "init"} 33 | cmd.SetArgs(args) 34 | e := cmd.Execute() 35 | fmt.Printf(" : %v", e) 36 | So(e, ShouldBeError) 37 | }) 38 | } 39 | -------------------------------------------------------------------------------- /cmd/config_set.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | "github.com/mitchellh/go-homedir" 7 | "os" 8 | 9 | logger "github.com/sirupsen/logrus" 10 | "github.com/spf13/cobra" 11 | "github.com/spf13/viper" 12 | ) 13 | 14 | var configSetCmd = &cobra.Command{ 15 | Use: "set", 16 | Short: "Used to modify configuration items in the [base] group of the configuration file", 17 | Long: `Used to modify configuration items in the [base] group of the configuration file 18 | 19 | Format: 20 | ./coscli config set [flags] 21 | 22 | Example: 23 | ./coscli config set -t example-token`, 24 | Args: cobra.ExactArgs(0), 25 | RunE: func(cmd *cobra.Command, args []string) error { 26 | err := setConfigItem(cmd) 27 | return err 28 | }, 29 | } 30 | 31 | func init() { 32 | configCmd.AddCommand(configSetCmd) 33 | 34 | configSetCmd.Flags().StringP("secret_id", "", "", "Set secret id") 35 | configSetCmd.Flags().StringP("secret_key", "", "", "Set secret key") 36 | configSetCmd.Flags().StringP("session_token", "t", "", "Set session token") 37 | configSetCmd.Flags().StringP("mode", "", "", "Set mode") 38 | configSetCmd.Flags().StringP("cvm_role_name", "", "", "Set cvm role name") 39 | configSetCmd.Flags().StringP("close_auto_switch_host", "", "", "Close Auto Switch Host") 40 | configSetCmd.Flags().StringP("disable_encryption", "", "", "Disable Encryption") 41 | } 42 | 43 | func setConfigItem(cmd *cobra.Command) error { 44 | flag := false 45 | secretID, _ := cmd.Flags().GetString("secret_id") 46 | secretKey, _ := cmd.Flags().GetString("secret_key") 47 | sessionToken, _ := cmd.Flags().GetString("session_token") 48 | mode, _ := cmd.Flags().GetString("mode") 49 | cvmRoleName, _ := cmd.Flags().GetString("cvm_role_name") 50 | closeAutoSwitchHost, _ := cmd.Flags().GetString("close_auto_switch_host") 51 | disableEncryption, _ := cmd.Flags().GetString("disable_encryption") 52 | if secretID != "" { 53 | flag = true 54 | if secretID == "@" { 55 | config.Base.SecretID = "" 56 | } else { 57 | config.Base.SecretID = secretID 58 | } 59 | } 60 | if secretKey != "" { 61 | flag = true 62 | if secretKey == "@" { 63 | config.Base.SecretKey = "" 64 | } else { 65 | config.Base.SecretKey = secretKey 66 | } 67 | } 68 | if sessionToken != "" { 69 | flag = true 70 | if sessionToken == "@" { 71 | config.Base.SessionToken = "" 72 | } else { 73 | config.Base.SessionToken = sessionToken 74 | } 75 | } 76 | if mode != "" { 77 | flag = true 78 | if mode != "SecretKey" && mode != "CvmRole" { 79 | return fmt.Errorf("Please Enter Mode As SecretKey Or CvmRole!") 80 | } else { 81 | config.Base.Mode = mode 82 | } 83 | } 84 | if cvmRoleName != "" { 85 | flag = true 86 | if cvmRoleName == "@" { 87 | config.Base.CvmRoleName = "" 88 | } else { 89 | config.Base.CvmRoleName = cvmRoleName 90 | } 91 | } 92 | 93 | if closeAutoSwitchHost != "" { 94 | flag = true 95 | if closeAutoSwitchHost == "@" { 96 | config.Base.CloseAutoSwitchHost = "" 97 | } else { 98 | config.Base.CloseAutoSwitchHost = closeAutoSwitchHost 99 | } 100 | } 101 | 102 | if disableEncryption != "" { 103 | flag = true 104 | if disableEncryption == "@" { 105 | config.Base.DisableEncryption = "" 106 | } else { 107 | config.Base.DisableEncryption = disableEncryption 108 | } 109 | } 110 | 111 | if !flag { 112 | return fmt.Errorf("Enter at least one configuration item to be modified!") 113 | } 114 | // 若未关闭秘钥加密，则先加密秘钥 115 | if config.Base.DisableEncryption != "true" { 116 | config.Base.SecretKey, _ = util.EncryptSecret(config.Base.SecretKey) 117 | config.Base.SecretID, _ = util.EncryptSecret(config.Base.SecretID) 118 | config.Base.SessionToken, _ = util.EncryptSecret(config.Base.SessionToken) 119 | } 120 | 121 | // 判断config文件是否存在。不存在则创建 122 | home, err := homedir.Dir() 123 | configFile := "" 124 | if cfgFile != "" { 125 | if cfgFile[0] == '~' { 126 | configFile = home + cfgFile[1:] 127 | } else { 128 | configFile = cfgFile 129 | } 130 | } else { 131 | configFile = home + "/.cos.yaml" 132 | } 133 | _, err = os.Stat(configFile) 134 | if os.IsNotExist(err) || cfgFile != "" { 135 | viper.Set("cos.base", config.Base) 136 | if err := viper.WriteConfigAs(configFile); err != nil { 137 | return err 138 | } 139 | } else { 140 | viper.Set("cos.base", config.Base) 141 | if err := viper.WriteConfigAs(viper.ConfigFileUsed()); err != nil { 142 | return err 143 | } 144 | } 145 | 146 | logger.Infoln("Modify successfully!") 147 | return nil 148 | } 149 | -------------------------------------------------------------------------------- /cmd/config_set_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | "testing" 7 | 8 | . "github.com/agiledragon/gomonkey/v2" 9 | . "github.com/smartystreets/goconvey/convey" 10 | "github.com/spf13/viper" 11 | ) 12 | 13 | func TestConfigSetCmd(t *testing.T) { 14 | fmt.Println("TestConfigSetCmd") 15 | getConfig() 16 | var oldconfig util.Config = config 17 | secretKey, err := util.DecryptSecret(config.Base.SecretKey) 18 | if err == nil { 19 | oldconfig.Base.SecretKey = secretKey 20 | } 21 | secretId, err := util.DecryptSecret(config.Base.SecretID) 22 | if err == nil { 23 | oldconfig.Base.SecretID = secretId 24 | } 25 | sessionToken, err := util.DecryptSecret(config.Base.SessionToken) 26 | if err == nil { 27 | oldconfig.Base.SessionToken = sessionToken 28 | } 29 | copyYaml() 30 | defer delYaml() 31 | clearCmd() 32 | cmd := rootCmd 33 | cmd.SilenceUsage = true 34 | cmd.SilenceErrors = true 35 | Convey("Test coscil config set", t, func() { 36 | Convey("fail", func() { 37 | Convey("no arguments", func() { 38 | clearCmd() 39 | cmd := rootCmd 40 | args := []string{"config", "set"} 41 | cmd.SetArgs(args) 42 | e := cmd.Execute() 43 | fmt.Printf(" : %v", e) 44 | So(e, ShouldBeError) 45 | }) 46 | Convey("no mode", func() { 47 | clearCmd() 48 | cmd := rootCmd 49 | args := []string{"config", "set", "--mode", "@"} 50 | cmd.SetArgs(args) 51 | e := cmd.Execute() 52 | fmt.Printf(" : %v", e) 53 | So(e, ShouldBeError) 54 | }) 55 | Convey("@", func() { 56 | clearCmd() 57 | cmd := rootCmd 58 | patches := ApplyFunc(viper.WriteConfigAs, func(string) error { 59 | return fmt.Errorf("test WriteConfigAs fail") 60 | }) 61 | defer patches.Reset() 62 | args := []string{"config", "set", "--secret_id", "@", 63 | "--secret_key", "@", "--session_token", "@", "--mode", "", 64 | "--cvm_role_name", "@", "--close_auto_switch_host", "@", "--disable_encryption", "@"} 65 | cmd.SetArgs(args) 66 | e := cmd.Execute() 67 | fmt.Printf(" : %v", e) 68 | So(e, ShouldBeError) 69 | }) 70 | Convey("token,mode,cvm", func() { 71 | clearCmd() 72 | cmd := rootCmd 73 | patches := ApplyFunc(viper.WriteConfigAs, func(string) error { 74 | return fmt.Errorf("test WriteConfigAs fail") 75 | }) 76 | defer patches.Reset() 77 | args := []string{"config", "set", "--session_token", "666", "--mode", "CvmRole", 78 | "--cvm_role_name", "name"} 79 | cmd.SetArgs(args) 80 | e := cmd.Execute() 81 | fmt.Printf(" : %v", e) 82 | So(e, ShouldBeError) 83 | }) 84 | Convey("cfgFile", func() { 85 | patches := ApplyFunc(viper.WriteConfigAs, func(string) error { 86 | return fmt.Errorf("test write configas error") 87 | }) 88 | defer patches.Reset() 89 | clearCmd() 90 | cmd := rootCmd 91 | args := []string{"config", "set", "--secret_id", "@", "-c", "./test.yaml"} 92 | cmd.SetArgs(args) 93 | e := cmd.Execute() 94 | fmt.Printf(" : %v", e) 95 | So(e, ShouldBeError) 96 | }) 97 | }) 98 | Convey("success", func() { 99 | Convey("give arguments", func() { 100 | clearCmd() 101 | cmd := rootCmd 102 | args := []string{"config", "set", "--secret_id", oldconfig.Base.SecretID, 103 | "--secret_key", oldconfig.Base.SecretKey, "--session_token", "@", "--mode", oldconfig.Base.Mode, 104 | "--cvm_role_name", "@", "--close_auto_switch_host", oldconfig.Base.CloseAutoSwitchHost, "--disable_encryption", oldconfig.Base.DisableEncryption} 105 | cmd.SetArgs(args) 106 | e := cmd.Execute() 107 | So(e, ShouldBeNil) 108 | }) 109 | }) 110 | 111 | }) 112 | } 113 | -------------------------------------------------------------------------------- /cmd/config_show.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "github.com/spf13/cobra" 6 | "github.com/spf13/viper" 7 | ) 8 | 9 | var configShowCmd = &cobra.Command{ 10 | Use: "show", 11 | Short: "Prints information from a specified configuration file", 12 | Long: `Prints information from a specified configuration file 13 | 14 | Format: 15 | ./coscli config show [-c ] 16 | 17 | Example: 18 | ./coscli config show`, 19 | Run: func(cmd *cobra.Command, args []string) { 20 | showConfig() 21 | }, 22 | } 23 | 24 | func init() { 25 | configCmd.AddCommand(configShowCmd) 26 | } 27 | 28 | func showConfig() { 29 | fmt.Println("Configuration file path:") 30 | fmt.Printf(" %s\n", viper.ConfigFileUsed()) 31 | fmt.Println("====================") 32 | fmt.Println("Basic Configuration Information:") 33 | fmt.Printf(" Secret ID: %s\n", config.Base.SecretID) 34 | fmt.Printf(" Secret Key: %s\n", config.Base.SecretKey) 35 | fmt.Printf(" Session Token: %s\n", config.Base.SessionToken) 36 | fmt.Printf(" Mode: %s\n", config.Base.Mode) 37 | fmt.Printf(" CvmRoleName: %s\n", config.Base.CvmRoleName) 38 | fmt.Printf(" CloseAutoSwitchHost: %s\n", config.Base.CloseAutoSwitchHost) 39 | fmt.Printf(" DisableEncryption: %s\n", config.Base.DisableEncryption) 40 | fmt.Println("====================") 41 | fmt.Println("Bucket Configuration Information:") 42 | 43 | for i, b := range config.Buckets { 44 | fmt.Printf("- Bucket %d :\n", i+1) 45 | fmt.Printf(" Name: \t%s\n", b.Name) 46 | fmt.Printf(" Endpoint:\t%s\n", b.Endpoint) 47 | fmt.Printf(" Alias: \t%s\n", b.Alias) 48 | fmt.Printf(" Ofs: \t%v\n", b.Ofs) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /cmd/config_show_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | . "github.com/smartystreets/goconvey/convey" 8 | ) 9 | 10 | func TestConfigShowCmd(t *testing.T) { 11 | fmt.Println("TestConfigShowCmd") 12 | Convey("Test coscil config show", t, func() { 13 | Convey("success", func() { 14 | Convey("give arguments", func() { 15 | clearCmd() 16 | cmd := rootCmd 17 | args := []string{"config", "show"} 18 | cmd.SetArgs(args) 19 | e := cmd.Execute() 20 | So(e, ShouldBeNil) 21 | }) 22 | }) 23 | }) 24 | } 25 | -------------------------------------------------------------------------------- /cmd/config_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | . "github.com/smartystreets/goconvey/convey" 8 | ) 9 | 10 | func TestConfigCmd(t *testing.T) { 11 | fmt.Println("TestConfigCmd") 12 | Convey("Test coscli config", t, func() { 13 | clearCmd() 14 | cmd := rootCmd 15 | args := []string{"config"} 16 | cmd.SetArgs(args) 17 | e := cmd.Execute() 18 | So(e, ShouldBeNil) 19 | }) 20 | } 21 | -------------------------------------------------------------------------------- /cmd/du.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | var duCmd = &cobra.Command{ 10 | Use: "du", 11 | Short: "Displays the size of a bucket or objects", 12 | Long: `Displays the size of a bucket or objects 13 | 14 | Format: 15 | ./coscli du cos://[/prefix/] [flags] 16 | 17 | Example: 18 | ./coscli du cos://examplebucket/test/`, 19 | Args: cobra.ExactArgs(1), 20 | RunE: func(cmd *cobra.Command, args []string) error { 21 | include, _ := cmd.Flags().GetString("include") 22 | exclude, _ := cmd.Flags().GetString("exclude") 23 | allVersions, _ := cmd.Flags().GetBool("all-versions") 24 | _, filters := util.GetFilter(include, exclude) 25 | 26 | cosPath := args[0] 27 | cosUrl, err := util.FormatUrl(cosPath) 28 | if err != nil { 29 | return fmt.Errorf("cos url format error:%v", err) 30 | } 31 | if !cosUrl.IsCosUrl() { 32 | return fmt.Errorf("cospath needs to contain %s", util.SchemePrefix) 33 | } 34 | 35 | bucketName := cosUrl.(*util.CosUrl).Bucket 36 | c, err := util.NewClient(&config, ¶m, bucketName) 37 | if err != nil { 38 | return err 39 | } 40 | 41 | // 判断存储桶是否开启版本控制 42 | if allVersions { 43 | res, _, err := util.GetBucketVersioning(c) 44 | if err != nil { 45 | return err 46 | } 47 | if res.Status != util.VersionStatusEnabled { 48 | return fmt.Errorf("versioning is not enabled on the current bucket") 49 | } 50 | } 51 | 52 | err = util.DuObjects(c, cosUrl, filters, util.DU_TYPE_CATEGORIZATION, allVersions) 53 | return err 54 | }, 55 | } 56 | 57 | func init() { 58 | rootCmd.AddCommand(duCmd) 59 | duCmd.Flags().String("include", "", "List files that meet the specified criteria") 60 | duCmd.Flags().String("exclude", "", "Exclude files that meet the specified criteria") 61 | duCmd.Flags().BoolP("all-versions", "", false, "List all versions of objects, only available if bucket versioning is enabled.") 62 | } 63 | -------------------------------------------------------------------------------- /cmd/du_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | "testing" 7 | 8 | . "github.com/agiledragon/gomonkey/v2" 9 | . "github.com/smartystreets/goconvey/convey" 10 | "github.com/tencentyun/cos-go-sdk-v5" 11 | ) 12 | 13 | func TestDuCmd(t *testing.T) { 14 | fmt.Println("TestDuCmd") 15 | testBucket = randStr(8) 16 | testAlias = testBucket + "-alias" 17 | testOfsBucket = randStr(8) 18 | testOfsBucketAlias = testOfsBucket + "-alias" 19 | testVersionBucket = randStr(8) 20 | testVersionBucketAlias = testVersionBucket + "-alias" 21 | setUp(testBucket, testAlias, testEndpoint, false, false) 22 | defer tearDown(testBucket, testAlias, testEndpoint, false) 23 | setUp(testOfsBucket, testOfsBucketAlias, testEndpoint, true, false) 24 | defer tearDown(testOfsBucket, testOfsBucketAlias, testEndpoint, false) 25 | setUp(testVersionBucket, testVersionBucketAlias, testEndpoint, false, true) 26 | //defer tearDown(testVersionBucket, testVersionBucketAlias, testEndpoint, true) 27 | clearCmd() 28 | cmd := rootCmd 29 | cmd.SilenceErrors = true 30 | cmd.SilenceUsage = true 31 | genDir(testDir, 3) 32 | defer delDir(testDir) 33 | localFileName := fmt.Sprintf("%s/small-file", testDir) 34 | 35 | cosFileName := fmt.Sprintf("cos://%s/%s", testAlias, "multi-small") 36 | args := []string{"cp", localFileName, cosFileName, "-r"} 37 | cmd.SetArgs(args) 38 | cmd.Execute() 39 | 40 | ofsFileName := fmt.Sprintf("cos://%s/%s", testOfsBucketAlias, "multi-small") 41 | 42 | args = []string{"cp", localFileName, ofsFileName, "-r"} 43 | cmd.SetArgs(args) 44 | cmd.Execute() 45 | 46 | versioningFileName := fmt.Sprintf("cos://%s/%s", testVersionBucketAlias, "multi-small") 47 | args = []string{"cp", localFileName, versioningFileName, "-r"} 48 | cmd.SetArgs(args) 49 | cmd.Execute() 50 | Convey("Test coscli du", t, func() { 51 | Convey("success", func() { 52 | Convey("duBucket", func() { 53 | clearCmd() 54 | cmd := rootCmd 55 | args = []string{"du", fmt.Sprintf("cos://%s", testAlias)} 56 | cmd.SetArgs(args) 57 | e := cmd.Execute() 58 | So(e, ShouldBeNil) 59 | }) 60 | Convey("duCosObjects", func() { 61 | clearCmd() 62 | cmd := rootCmd 63 | args = []string{"du", cosFileName} 64 | cmd.SetArgs(args) 65 | e := cmd.Execute() 66 | So(e, ShouldBeNil) 67 | }) 68 | Convey("duOfsObjects", func() { 69 | clearCmd() 70 | cmd := rootCmd 71 | args = []string{"du", ofsFileName} 72 | cmd.SetArgs(args) 73 | e := cmd.Execute() 74 | So(e, ShouldBeNil) 75 | }) 76 | Convey("duCosObjectVersions", func() { 77 | clearCmd() 78 | cmd := rootCmd 79 | args = []string{"du", versioningFileName, "--all-versions"} 80 | cmd.SetArgs(args) 81 | e := cmd.Execute() 82 | So(e, ShouldBeNil) 83 | }) 84 | }) 85 | Convey("fail", func() { 86 | Convey("not enough arguments", func() { 87 | clearCmd() 88 | cmd := rootCmd 89 | args = []string{"du"} 90 | cmd.SetArgs(args) 91 | e := cmd.Execute() 92 | fmt.Printf(" : %v", e) 93 | So(e, ShouldBeError) 94 | }) 95 | Convey("FormatUrl", func() { 96 | patches := ApplyFunc(util.FormatUrl, func(urlStr string) (util.StorageUrl, error) { 97 | return nil, fmt.Errorf("test formaturl fail") 98 | }) 99 | defer patches.Reset() 100 | clearCmd() 101 | cmd := rootCmd 102 | args = []string{"du", "invalid"} 103 | cmd.SetArgs(args) 104 | e := cmd.Execute() 105 | fmt.Printf(" : %v", e) 106 | So(e, ShouldBeError) 107 | }) 108 | Convey("not cos url", func() { 109 | clearCmd() 110 | cmd := rootCmd 111 | args = []string{"du", "invalid"} 112 | cmd.SetArgs(args) 113 | e := cmd.Execute() 114 | fmt.Printf(" : %v", e) 115 | So(e, ShouldBeError) 116 | }) 117 | Convey("NewClient", func() { 118 | patches := ApplyFunc(util.NewClient, func(config *util.Config, param *util.Param, bucketName string) (client *cos.Client, err error) { 119 | return nil, fmt.Errorf("test NewClient error") 120 | }) 121 | defer patches.Reset() 122 | clearCmd() 123 | cmd := rootCmd 124 | args = []string{"du", fmt.Sprintf("cos://%s", testAlias)} 125 | cmd.SetArgs(args) 126 | e := cmd.Execute() 127 | fmt.Printf(" : %v", e) 128 | So(e, ShouldBeError) 129 | }) 130 | }) 131 | }) 132 | } 133 | -------------------------------------------------------------------------------- /cmd/hash.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | "os" 7 | "strings" 8 | 9 | logger "github.com/sirupsen/logrus" 10 | "github.com/spf13/cobra" 11 | ) 12 | 13 | var hashCmd = &cobra.Command{ 14 | Use: "hash", 15 | Short: "Calculate local file's hash-code or show cos file's hash-code", 16 | Long: `Calculate local file's hash-code or show cos file's hash-code 17 | 18 | Format: 19 | ./coscli hash [--type ] 20 | 21 | Example: 22 | ./coscli hash cos://example --type md5`, 23 | Args: cobra.ExactArgs(1), 24 | RunE: func(cmd *cobra.Command, args []string) error { 25 | bucketName, path := util.ParsePath(args[0]) 26 | hashType, _ := cmd.Flags().GetString("type") 27 | hashType = strings.ToLower(hashType) 28 | var err error 29 | if bucketName != "" { 30 | err = showHash(bucketName, path, hashType) 31 | } else { 32 | _, err = calculateHash(path, hashType) 33 | } 34 | 35 | return err 36 | }, 37 | } 38 | 39 | func init() { 40 | rootCmd.AddCommand(hashCmd) 41 | 42 | hashCmd.Flags().StringP("type", "", "crc64", "Choose the hash type(md5 or crc64)") 43 | } 44 | 45 | func showHash(bucketName string, path string, hashType string) error { 46 | c, err := util.NewClient(&config, ¶m, bucketName) 47 | if err != nil { 48 | return err 49 | } 50 | switch hashType { 51 | case "crc64": 52 | h, _, _, err := util.ShowHash(c, path, "crc64") 53 | if err != nil { 54 | return err 55 | } 56 | logger.Infoln("crc64-ecma: ", h) 57 | case "md5": 58 | h, b, _, err := util.ShowHash(c, path, "md5") 59 | if err != nil { 60 | return err 61 | } 62 | logger.Infoln("md5: ", h) 63 | logger.Infoln("base64: ", b) 64 | default: 65 | return fmt.Errorf("--type can only be selected between MD5 and CRC64") 66 | } 67 | return nil 68 | } 69 | 70 | func calculateHash(path string, hashType string) (h string, err error) { 71 | switch hashType { 72 | case "crc64": 73 | h, _, err = util.CalculateHash(path, "crc64") 74 | if err != nil { 75 | return "", err 76 | } 77 | logger.Infoln("crc64-ecma: ", h) 78 | case "md5": 79 | f, err := os.Stat(path) 80 | if err != nil { 81 | return "", err 82 | } 83 | 84 | if (float64(f.Size()) / 1024 / 1024) > 32 { 85 | return "", fmt.Errorf("MD5 of large files is not supported") 86 | } 87 | 88 | hash, b, err := util.CalculateHash(path, "md5") 89 | if err != nil { 90 | return "", err 91 | } 92 | h = hash 93 | logger.Infof("md5: %s\n", h) 94 | logger.Infoln("base64: ", b) 95 | default: 96 | return "", fmt.Errorf("--type can only be selected between MD5 and CRC64") 97 | } 98 | return h, err 99 | } 100 | -------------------------------------------------------------------------------- /cmd/hash_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | "testing" 7 | 8 | . "github.com/agiledragon/gomonkey/v2" 9 | . "github.com/smartystreets/goconvey/convey" 10 | "github.com/tencentyun/cos-go-sdk-v5" 11 | ) 12 | 13 | func TestHashCmd(t *testing.T) { 14 | fmt.Println("TestHashCmd") 15 | testBucket = randStr(8) 16 | testAlias = testBucket + "-alias" 17 | setUp(testBucket, testAlias, testEndpoint, false, false) 18 | defer tearDown(testBucket, testAlias, testEndpoint, false) 19 | clearCmd() 20 | cmd := rootCmd 21 | cmd.SilenceErrors = true 22 | cmd.SilenceUsage = true 23 | genDir(testDir, 3) 24 | defer delDir(testDir) 25 | localFileName := fmt.Sprintf("%s/small-file", testDir) 26 | cosFileName := fmt.Sprintf("cos://%s/%s", testAlias, "multi-small") 27 | args := []string{"cp", localFileName, cosFileName, "-r"} 28 | cmd.SetArgs(args) 29 | cmd.Execute() 30 | Convey("Test coscli hash", t, func() { 31 | Convey("local file", func() { 32 | Convey("crc64", func() { 33 | clearCmd() 34 | cmd := rootCmd 35 | args := []string{"hash", fmt.Sprintf("%s/0", localFileName)} 36 | cmd.SetArgs(args) 37 | e := cmd.Execute() 38 | So(e, ShouldBeNil) 39 | }) 40 | Convey("md5", func() { 41 | clearCmd() 42 | cmd := rootCmd 43 | args := []string{"hash", fmt.Sprintf("%s/0", localFileName), "--type=md5"} 44 | cmd.SetArgs(args) 45 | e := cmd.Execute() 46 | So(e, ShouldBeNil) 47 | }) 48 | }) 49 | Convey("cos file", func() { 50 | Convey("crc64", func() { 51 | clearCmd() 52 | cmd := rootCmd 53 | args := []string{"hash", fmt.Sprintf("%s/0", cosFileName)} 54 | cmd.SetArgs(args) 55 | e := cmd.Execute() 56 | So(e, ShouldBeNil) 57 | }) 58 | Convey("md5", func() { 59 | clearCmd() 60 | cmd := rootCmd 61 | args := []string{"hash", fmt.Sprintf("%s/0", cosFileName), "--type=md5"} 62 | cmd.SetArgs(args) 63 | e := cmd.Execute() 64 | So(e, ShouldBeNil) 65 | }) 66 | }) 67 | Convey("fail", func() { 68 | Convey("New Client", func() { 69 | clearCmd() 70 | cmd := rootCmd 71 | patches := ApplyFunc(util.NewClient, func(config *util.Config, param *util.Param, bucketName string) (client *cos.Client, err error) { 72 | return nil, fmt.Errorf("test client error") 73 | }) 74 | defer patches.Reset() 75 | args := []string{"hash", fmt.Sprintf("%s/0", cosFileName)} 76 | cmd.SetArgs(args) 77 | e := cmd.Execute() 78 | fmt.Printf(" : %v", e) 79 | So(e, ShouldBeError) 80 | }) 81 | Convey("ShowHash", func() { 82 | clearCmd() 83 | cmd := rootCmd 84 | args := []string{"hash", fmt.Sprintf("%s/0", cosFileName)} 85 | Convey("crc64", func() { 86 | patches := ApplyFunc(util.ShowHash, func(c *cos.Client, path string, hashType string) (h string, b string, resp *cos.Response, err error) { 87 | return "", "", nil, fmt.Errorf("test ShowHash crc64 error") 88 | }) 89 | defer patches.Reset() 90 | cmd.SetArgs(args) 91 | e := cmd.Execute() 92 | fmt.Printf(" : %v", e) 93 | So(e, ShouldBeError) 94 | }) 95 | Convey("md5", func() { 96 | patches := ApplyFunc(util.ShowHash, func(c *cos.Client, path string, hashType string) (h string, b string, resp *cos.Response, err error) { 97 | return "", "", nil, fmt.Errorf("test ShowHash md5 error") 98 | }) 99 | defer patches.Reset() 100 | args := append(args, "--type=md5") 101 | cmd.SetArgs(args) 102 | e := cmd.Execute() 103 | fmt.Printf(" : %v", e) 104 | So(e, ShouldBeError) 105 | }) 106 | }) 107 | Convey("CalculateHash", func() { 108 | clearCmd() 109 | cmd := rootCmd 110 | args := []string{"hash", fmt.Sprintf("%s/0", localFileName)} 111 | Convey("crc64", func() { 112 | patches := ApplyFunc(util.CalculateHash, func(path string, hashType string) (h string, b string, err error) { 113 | return "", "", fmt.Errorf("test CalculateHash crc64 error") 114 | }) 115 | defer patches.Reset() 116 | cmd.SetArgs(args) 117 | e := cmd.Execute() 118 | fmt.Printf(" : %v", e) 119 | So(e, ShouldBeError) 120 | }) 121 | Convey("md5", func() { 122 | patches := ApplyFunc(util.CalculateHash, func(path string, hashType string) (h string, b string, err error) { 123 | return "", "", fmt.Errorf("test CalculateHash md5 error") 124 | }) 125 | defer patches.Reset() 126 | args := append(args, "--type=md5") 127 | cmd.SetArgs(args) 128 | e := cmd.Execute() 129 | fmt.Printf(" : %v", e) 130 | So(e, ShouldBeError) 131 | }) 132 | }) 133 | Convey("type error", func() { 134 | Convey("local file", func() { 135 | clearCmd() 136 | cmd := rootCmd 137 | args := []string{"hash", fmt.Sprintf("%s/0", localFileName), "--type=invalid"} 138 | cmd.SetArgs(args) 139 | e := cmd.Execute() 140 | fmt.Printf(" : %v", e) 141 | So(e, ShouldBeError) 142 | }) 143 | Convey("cos file", func() { 144 | clearCmd() 145 | cmd := rootCmd 146 | args := []string{"hash", fmt.Sprintf("%s/0", cosFileName), "--type=invalid"} 147 | cmd.SetArgs(args) 148 | e := cmd.Execute() 149 | fmt.Printf(" : %v", e) 150 | So(e, ShouldBeError) 151 | }) 152 | }) 153 | 154 | }) 155 | }) 156 | } 157 | -------------------------------------------------------------------------------- /cmd/ls.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "coscli/util" 6 | "fmt" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | var lsCmd = &cobra.Command{ 11 | Use: "ls", 12 | Short: "List buckets or objects", 13 | Long: `List buckets or objects 14 | 15 | Format: 16 | ./coscli ls cos://[/prefix/] [flags] 17 | 18 | Example: 19 | ./coscli ls cos://examplebucket/test/ -r`, 20 | Args: cobra.MaximumNArgs(1), 21 | RunE: func(cmd *cobra.Command, args []string) error { 22 | limit, _ := cmd.Flags().GetInt("limit") 23 | recursive, _ := cmd.Flags().GetBool("recursive") 24 | include, _ := cmd.Flags().GetString("include") 25 | exclude, _ := cmd.Flags().GetString("exclude") 26 | allVersions, _ := cmd.Flags().GetBool("all-versions") 27 | 28 | if limit == 0 { 29 | limit = 10000 30 | } else if limit < 0 { 31 | return fmt.Errorf("Flag --limit should be greater than 0") 32 | } 33 | 34 | cosPath := "" 35 | if len(args) != 0 { 36 | cosPath = args[0] 37 | } 38 | 39 | cosUrl, err := util.FormatUrl(cosPath) 40 | if err != nil { 41 | return fmt.Errorf("cos url format error:%v", err) 42 | } 43 | 44 | // 无参数，则列出当前账号下的所有存储桶 45 | if cosPath == "" { 46 | // 实例化cos client 47 | c, err := util.NewClient(&config, ¶m, "") 48 | if err != nil { 49 | return err 50 | } 51 | err = util.ListBuckets(c, limit) 52 | return err 53 | } else if cosUrl.IsCosUrl() { 54 | // 实例化cos client 55 | bucketName := cosUrl.(*util.CosUrl).Bucket 56 | c, err := util.NewClient(&config, ¶m, bucketName) 57 | if err != nil { 58 | return err 59 | } 60 | // 判断存储桶是否开启版本控制 61 | if allVersions { 62 | res, _, err := util.GetBucketVersioning(c) 63 | if err != nil { 64 | return err 65 | } 66 | if res.Status != util.VersionStatusEnabled { 67 | return fmt.Errorf("versioning is not enabled on the current bucket") 68 | } 69 | } 70 | 71 | _, filters := util.GetFilter(include, exclude) 72 | // 根据s.Header判断是否是融合桶或者普通桶 73 | s, err := c.Bucket.Head(context.Background()) 74 | if err != nil { 75 | return err 76 | } 77 | if s.Header.Get("X-Cos-Bucket-Arch") == "OFS" { 78 | if allVersions { 79 | return fmt.Errorf("the OFS bucket does not support listing multiple versions") 80 | } else { 81 | err = util.ListOfsObjects(c, cosUrl, limit, recursive, filters) 82 | } 83 | } else { 84 | if allVersions { 85 | err = util.ListObjectVersions(c, cosUrl, limit, recursive, filters) 86 | } else { 87 | err = util.ListObjects(c, cosUrl, limit, recursive, filters) 88 | } 89 | 90 | } 91 | 92 | if err != nil { 93 | return err 94 | } 95 | 96 | } else { 97 | return fmt.Errorf("cospath needs to contain cos://") 98 | } 99 | return nil 100 | }, 101 | } 102 | 103 | func init() { 104 | rootCmd.AddCommand(lsCmd) 105 | 106 | lsCmd.Flags().Int("limit", 0, "Limit the number of objects listed(0~1000)") 107 | lsCmd.Flags().BoolP("recursive", "r", false, "List objects recursively") 108 | lsCmd.Flags().String("include", "", "List files that meet the specified criteria") 109 | lsCmd.Flags().String("exclude", "", "Exclude files that meet the specified criteria") 110 | lsCmd.Flags().BoolP("all-versions", "", false, "List all versions of objects, only available if bucket versioning is enabled.") 111 | } 112 | -------------------------------------------------------------------------------- /cmd/ls_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "coscli/util" 6 | "fmt" 7 | "reflect" 8 | "testing" 9 | 10 | . "github.com/agiledragon/gomonkey/v2" 11 | . "github.com/smartystreets/goconvey/convey" 12 | "github.com/tencentyun/cos-go-sdk-v5" 13 | ) 14 | 15 | func TestLsCmd(t *testing.T) { 16 | fmt.Println("TestLsCmd") 17 | testBucket = randStr(8) 18 | testAlias = testBucket + "-alias" 19 | testOfsBucket = randStr(8) 20 | testOfsBucketAlias = testOfsBucket + "-alias" 21 | testVersionBucket = randStr(8) 22 | testVersionBucketAlias = testVersionBucket + "-alias" 23 | setUp(testBucket, testAlias, testEndpoint, false, false) 24 | defer tearDown(testBucket, testAlias, testEndpoint, false) 25 | setUp(testOfsBucket, testOfsBucketAlias, testEndpoint, true, false) 26 | defer tearDown(testOfsBucket, testOfsBucketAlias, testEndpoint, false) 27 | setUp(testVersionBucket, testVersionBucketAlias, testEndpoint, false, true) 28 | defer tearDown(testVersionBucket, testVersionBucketAlias, testEndpoint, true) 29 | clearCmd() 30 | cmd := rootCmd 31 | cmd.SilenceErrors = true 32 | cmd.SilenceUsage = true 33 | genDir(testDir, 3) 34 | defer delDir(testDir) 35 | localFileName := fmt.Sprintf("%s/small-file", testDir) 36 | 37 | cosFileName := fmt.Sprintf("cos://%s/%s", testAlias, "multi-small") 38 | args := []string{"cp", localFileName, cosFileName, "-r"} 39 | cmd.SetArgs(args) 40 | cmd.Execute() 41 | 42 | ofsFileName := fmt.Sprintf("cos://%s/%s", testOfsBucketAlias, "multi-small") 43 | args = []string{"cp", localFileName, ofsFileName, "-r"} 44 | cmd.SetArgs(args) 45 | cmd.Execute() 46 | 47 | versioningFileName := fmt.Sprintf("cos://%s/%s", testVersionBucketAlias, "multi-small") 48 | args = []string{"cp", localFileName, versioningFileName, "-r"} 49 | cmd.SetArgs(args) 50 | cmd.Execute() 51 | 52 | Convey("Test coscli ls", t, func() { 53 | Convey("success", func() { 54 | Convey("无参数", func() { 55 | clearCmd() 56 | cmd := rootCmd 57 | args := []string{"ls"} 58 | cmd.SetArgs(args) 59 | e := cmd.Execute() 60 | So(e, ShouldBeNil) 61 | }) 62 | Convey("指定桶名", func() { 63 | clearCmd() 64 | cmd := rootCmd 65 | args := []string{"ls", 66 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint, "-r"} 67 | cmd.SetArgs(args) 68 | e := cmd.Execute() 69 | So(e, ShouldBeNil) 70 | }) 71 | Convey("OFS", func() { 72 | clearCmd() 73 | cmd := rootCmd 74 | args := []string{"ls", 75 | fmt.Sprintf("cos://%s-%s", testOfsBucket, appID), "-e", testEndpoint, "-r"} 76 | cmd.SetArgs(args) 77 | e := cmd.Execute() 78 | So(e, ShouldBeNil) 79 | }) 80 | Convey("多版本桶", func() { 81 | clearCmd() 82 | cmd := rootCmd 83 | args := []string{"ls", 84 | fmt.Sprintf("cos://%s-%s", testVersionBucket, appID), "-e", testEndpoint, "--all-versions", "-r"} 85 | cmd.SetArgs(args) 86 | e := cmd.Execute() 87 | So(e, ShouldBeNil) 88 | }) 89 | }) 90 | Convey("fail", func() { 91 | Convey("参数--limit<0", func() { 92 | clearCmd() 93 | cmd := rootCmd 94 | args := []string{"ls", "--limit", "-1"} 95 | cmd.SetArgs(args) 96 | e := cmd.Execute() 97 | fmt.Printf(" : %v", e) 98 | So(e, ShouldBeError) 99 | }) 100 | Convey("FormatUrl", func() { 101 | clearCmd() 102 | cmd := rootCmd 103 | patches := ApplyFunc(util.FormatUrl, func(urlStr string) (util.StorageUrl, error) { 104 | return nil, fmt.Errorf("test formaturl error") 105 | }) 106 | defer patches.Reset() 107 | args := []string{"ls"} 108 | cmd.SetArgs(args) 109 | e := cmd.Execute() 110 | fmt.Printf(" : %v", e) 111 | So(e, ShouldBeError) 112 | }) 113 | Convey("New Client", func() { 114 | patches := ApplyFunc(util.NewClient, func(config *util.Config, param *util.Param, bucketName string) (client *cos.Client, err error) { 115 | return nil, fmt.Errorf("test new client error") 116 | }) 117 | defer patches.Reset() 118 | Convey("no cosPath", func() { 119 | clearCmd() 120 | cmd := rootCmd 121 | args := []string{"ls"} 122 | cmd.SetArgs(args) 123 | e := cmd.Execute() 124 | fmt.Printf(" : %v", e) 125 | So(e, ShouldBeError) 126 | }) 127 | Convey("cosPath", func() { 128 | clearCmd() 129 | cmd := rootCmd 130 | args := []string{"ls", 131 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint} 132 | cmd.SetArgs(args) 133 | e := cmd.Execute() 134 | fmt.Printf(" : %v", e) 135 | So(e, ShouldBeError) 136 | }) 137 | }) 138 | Convey("Head", func() { 139 | var c *cos.BucketService 140 | patches := ApplyMethodFunc(reflect.TypeOf(c), "Head", func(ctx context.Context, opt ...*cos.BucketHeadOptions) (*cos.Response, error) { 141 | return nil, fmt.Errorf("test Head error") 142 | }) 143 | defer patches.Reset() 144 | clearCmd() 145 | cmd := rootCmd 146 | args := []string{"ls", 147 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint} 148 | cmd.SetArgs(args) 149 | e := cmd.Execute() 150 | fmt.Printf(" : %v", e) 151 | So(e, ShouldBeError) 152 | }) 153 | Convey("ListObject", func() { 154 | patches := ApplyFunc(util.ListObjects, func(c *cos.Client, cosUrl util.StorageUrl, limit int, recursive bool, filters []util.FilterOptionType) error { 155 | return fmt.Errorf("test ListObject error") 156 | }) 157 | defer patches.Reset() 158 | clearCmd() 159 | cmd := rootCmd 160 | args := []string{"ls", 161 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint} 162 | cmd.SetArgs(args) 163 | e := cmd.Execute() 164 | fmt.Printf(" : %v", e) 165 | So(e, ShouldBeError) 166 | }) 167 | Convey("not cos", func() { 168 | clearCmd() 169 | cmd := rootCmd 170 | args := []string{"ls", 171 | fmt.Sprintf("/%s-%s", testBucket, appID), "-e", testEndpoint} 172 | cmd.SetArgs(args) 173 | e := cmd.Execute() 174 | fmt.Printf(" : %v", e) 175 | So(e, ShouldBeError) 176 | }) 177 | Convey("未开启多版本桶使用 --all-versions 参数", func() { 178 | clearCmd() 179 | cmd := rootCmd 180 | args := []string{"ls", 181 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint, "--all-versions"} 182 | cmd.SetArgs(args) 183 | e := cmd.Execute() 184 | fmt.Printf(" : %v", e) 185 | So(e, ShouldBeError) 186 | }) 187 | Convey("OFS桶使用 --all-versions 参数", func() { 188 | clearCmd() 189 | cmd := rootCmd 190 | args := []string{"ls", 191 | fmt.Sprintf("cos://%s-%s", testOfsBucket, appID), "-e", testEndpoint, "-r", "--all-versions"} 192 | cmd.SetArgs(args) 193 | e := cmd.Execute() 194 | fmt.Printf(" : %v", e) 195 | So(e, ShouldBeError) 196 | }) 197 | }) 198 | }) 199 | } 200 | -------------------------------------------------------------------------------- /cmd/lsdu.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | var lsduCmd = &cobra.Command{ 10 | Use: "lsdu", 11 | Short: "Displays the size of a bucket or objects", 12 | Long: `Displays the size of a bucket or objects 13 | 14 | Format: 15 | ./coscli lsdu cos://[/prefix/] [flags] 16 | 17 | Example: 18 | ./coscli lsdu cos://examplebucket/test/`, 19 | Args: cobra.ExactArgs(1), 20 | RunE: func(cmd *cobra.Command, args []string) error { 21 | include, _ := cmd.Flags().GetString("include") 22 | exclude, _ := cmd.Flags().GetString("exclude") 23 | _, filters := util.GetFilter(include, exclude) 24 | 25 | cosPath := args[0] 26 | cosUrl, err := util.FormatUrl(cosPath) 27 | if err != nil { 28 | return fmt.Errorf("cos url format error:%v", err) 29 | } 30 | 31 | if !cosUrl.IsCosUrl() { 32 | return fmt.Errorf("cospath needs to contain %s", util.SchemePrefix) 33 | } 34 | 35 | bucketName := cosUrl.(*util.CosUrl).Bucket 36 | c, err := util.NewClient(&config, ¶m, bucketName) 37 | if err != nil { 38 | return err 39 | } 40 | 41 | err = util.LsAndDuObjects(c, cosUrl, filters) 42 | return err 43 | }, 44 | } 45 | 46 | func init() { 47 | rootCmd.AddCommand(lsduCmd) 48 | lsduCmd.Flags().String("include", "", "List files that meet the specified criteria") 49 | lsduCmd.Flags().String("exclude", "", "Exclude files that meet the specified criteria") 50 | } 51 | -------------------------------------------------------------------------------- /cmd/lsdu_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | "testing" 7 | 8 | . "github.com/agiledragon/gomonkey/v2" 9 | . "github.com/smartystreets/goconvey/convey" 10 | "github.com/tencentyun/cos-go-sdk-v5" 11 | ) 12 | 13 | func TestLsduCmd(t *testing.T) { 14 | fmt.Println("TestLsduCmd") 15 | testBucket = randStr(8) 16 | testAlias = testBucket + "-alias" 17 | setUp(testBucket, testAlias, testEndpoint, false, false) 18 | defer tearDown(testBucket, testAlias, testEndpoint, false) 19 | clearCmd() 20 | cmd := rootCmd 21 | cmd.SilenceErrors = true 22 | cmd.SilenceUsage = true 23 | genDir(testDir, 3) 24 | defer delDir(testDir) 25 | localFileName := fmt.Sprintf("%s/small-file", testDir) 26 | cosFileName := fmt.Sprintf("cos://%s/%s", testAlias, "multi-small") 27 | args := []string{"cp", localFileName, cosFileName, "-r"} 28 | cmd.SetArgs(args) 29 | cmd.Execute() 30 | Convey("Test coscli lsdu", t, func() { 31 | Convey("success", func() { 32 | Convey("duBucket", func() { 33 | clearCmd() 34 | cmd := rootCmd 35 | args = []string{"lsdu", fmt.Sprintf("cos://%s", testAlias)} 36 | cmd.SetArgs(args) 37 | e := cmd.Execute() 38 | So(e, ShouldBeNil) 39 | }) 40 | Convey("duObjects", func() { 41 | clearCmd() 42 | cmd := rootCmd 43 | args = []string{"lsdu", cosFileName} 44 | cmd.SetArgs(args) 45 | e := cmd.Execute() 46 | So(e, ShouldBeNil) 47 | }) 48 | }) 49 | Convey("fail", func() { 50 | Convey("not enough arguments", func() { 51 | clearCmd() 52 | cmd := rootCmd 53 | args = []string{"lsdu"} 54 | cmd.SetArgs(args) 55 | e := cmd.Execute() 56 | fmt.Printf(" : %v", e) 57 | So(e, ShouldBeError) 58 | }) 59 | Convey("FormatUrl", func() { 60 | patches := ApplyFunc(util.FormatUrl, func(urlStr string) (util.StorageUrl, error) { 61 | return nil, fmt.Errorf("test formaturl fail") 62 | }) 63 | defer patches.Reset() 64 | clearCmd() 65 | cmd := rootCmd 66 | args = []string{"lsdu", "invalid"} 67 | cmd.SetArgs(args) 68 | e := cmd.Execute() 69 | fmt.Printf(" : %v", e) 70 | So(e, ShouldBeError) 71 | }) 72 | Convey("not cos url", func() { 73 | clearCmd() 74 | cmd := rootCmd 75 | args = []string{"lsdu", "invalid"} 76 | cmd.SetArgs(args) 77 | e := cmd.Execute() 78 | fmt.Printf(" : %v", e) 79 | So(e, ShouldBeError) 80 | }) 81 | Convey("NewClient", func() { 82 | patches := ApplyFunc(util.NewClient, func(config *util.Config, param *util.Param, bucketName string) (client *cos.Client, err error) { 83 | return nil, fmt.Errorf("test NewClient error") 84 | }) 85 | defer patches.Reset() 86 | clearCmd() 87 | cmd := rootCmd 88 | args = []string{"lsdu", fmt.Sprintf("cos://%s", testAlias)} 89 | cmd.SetArgs(args) 90 | e := cmd.Execute() 91 | fmt.Printf(" : %v", e) 92 | So(e, ShouldBeError) 93 | }) 94 | }) 95 | }) 96 | } 97 | -------------------------------------------------------------------------------- /cmd/lsparts.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | var lspartsCmd = &cobra.Command{ 10 | Use: "lsparts", 11 | Short: "List multipart uploads", 12 | Long: `List multipart uploads 13 | 14 | Format: 15 | ./coscli lsparts cos://[/] [flags] 16 | 17 | Example: 18 | ./coscli lsparts cos://examplebucket/test/`, 19 | Args: cobra.ExactArgs(1), 20 | RunE: func(cmd *cobra.Command, args []string) error { 21 | limit, _ := cmd.Flags().GetInt("limit") 22 | include, _ := cmd.Flags().GetString("include") 23 | exclude, _ := cmd.Flags().GetString("exclude") 24 | uploadId, _ := cmd.Flags().GetString("upload-id") 25 | if limit == 0 { 26 | limit = 10000 27 | } else if limit < 0 { 28 | return fmt.Errorf("Flag --limit should be greater than 0") 29 | } 30 | 31 | cosUrl, err := util.FormatUrl(args[0]) 32 | if err != nil { 33 | return fmt.Errorf("cos url format error:%v", err) 34 | } 35 | 36 | if !cosUrl.IsCosUrl() { 37 | return fmt.Errorf("cospath needs to contain cos://") 38 | } 39 | 40 | _, filters := util.GetFilter(include, exclude) 41 | 42 | bucketName := cosUrl.(*util.CosUrl).Bucket 43 | 44 | c, err := util.NewClient(&config, ¶m, bucketName) 45 | if err != nil { 46 | return err 47 | } 48 | 49 | if uploadId != "" { 50 | err = util.ListParts(c, cosUrl, limit, uploadId) 51 | } else { 52 | err = util.ListUploads(c, cosUrl, limit, filters) 53 | } 54 | 55 | return err 56 | }, 57 | } 58 | 59 | func init() { 60 | rootCmd.AddCommand(lspartsCmd) 61 | 62 | lspartsCmd.Flags().Int("limit", 0, "Limit the number of parts listed(0~1000)") 63 | lspartsCmd.Flags().String("include", "", "List files that meet the specified criteria") 64 | lspartsCmd.Flags().String("exclude", "", "Exclude files that meet the specified criteria") 65 | lspartsCmd.Flags().String("upload-id", "", "Identify the ID of this multipart upload, which is obtained when initializing the multipart upload using the Initiate Multipart Upload interface.") 66 | } 67 | -------------------------------------------------------------------------------- /cmd/lsparts_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | "testing" 7 | 8 | . "github.com/agiledragon/gomonkey/v2" 9 | . "github.com/smartystreets/goconvey/convey" 10 | "github.com/tencentyun/cos-go-sdk-v5" 11 | ) 12 | 13 | func TestLspartsCmd(t *testing.T) { 14 | fmt.Println("TestLspartsCmd") 15 | testBucket = randStr(8) 16 | testAlias = testBucket + "-alias" 17 | setUp(testBucket, testAlias, testEndpoint, false, false) 18 | defer tearDown(testBucket, testAlias, testEndpoint, false) 19 | clearCmd() 20 | cmd := rootCmd 21 | cmd.SilenceErrors = true 22 | cmd.SilenceUsage = true 23 | Convey("Test coscli lsparts", t, func() { 24 | Convey("success", func() { 25 | Convey("ls uploads", func() { 26 | clearCmd() 27 | cmd := rootCmd 28 | args := []string{"lsparts", fmt.Sprintf("cos://%s", testAlias)} 29 | cmd.SetArgs(args) 30 | e := cmd.Execute() 31 | So(e, ShouldBeNil) 32 | }) 33 | Convey("ls parts", func() { 34 | clearCmd() 35 | cmd := rootCmd 36 | patches := ApplyFunc(util.CheckUploadExist, func(c *cos.Client, cosUrl util.StorageUrl, uploadId string) (exist bool, err error) { 37 | return true, nil 38 | }) 39 | defer patches.Reset() 40 | 41 | lsPatches := ApplyFunc(util.GetPartsListForLs, func(c *cos.Client, cosUrl util.StorageUrl, uploadId, partNumberMarker string, limit int) (err error, parts []cos.Object, isTruncated bool, nextPartNumberMarker string) { 42 | return nil, []cos.Object{ 43 | { 44 | Key: "123", 45 | PartNumber: 1, 46 | LastModified: "2024-12-17T08:34:48.000Z", 47 | ETag: "58f06dd588d8ffb3beb46ada6309436b", 48 | Size: 33554432, 49 | }, 50 | }, false, "" 51 | }) 52 | defer lsPatches.Reset() 53 | 54 | args := []string{"lsparts", fmt.Sprintf("cos://%s", testAlias), "--upload-id", "123"} 55 | cmd.SetArgs(args) 56 | e := cmd.Execute() 57 | So(e, ShouldBeNil) 58 | }) 59 | 60 | }) 61 | Convey("fail", func() { 62 | Convey("limit invalid", func() { 63 | clearCmd() 64 | cmd := rootCmd 65 | args := []string{"lsparts", fmt.Sprintf("cos://%s", testAlias), "--limit", "-1"} 66 | cmd.SetArgs(args) 67 | e := cmd.Execute() 68 | fmt.Printf(" : %v", e) 69 | So(e, ShouldBeError) 70 | }) 71 | Convey("New Client", func() { 72 | clearCmd() 73 | cmd := rootCmd 74 | patches := ApplyFunc(util.NewClient, func(config *util.Config, param *util.Param, bucketName string) (client *cos.Client, err error) { 75 | return nil, fmt.Errorf("test formaturl error") 76 | }) 77 | defer patches.Reset() 78 | args := []string{"lsparts", fmt.Sprintf("cos://%s", testAlias)} 79 | cmd.SetArgs(args) 80 | e := cmd.Execute() 81 | fmt.Printf(" : %v", e) 82 | So(e, ShouldBeError) 83 | }) 84 | Convey("GetUploadsListForLs", func() { 85 | clearCmd() 86 | cmd := rootCmd 87 | patches := ApplyFunc(util.GetUploadsListForLs, func(c *cos.Client, cosUrl util.StorageUrl, uploadIDMarker, keyMarker string, limit int, recursive bool) (err error, uploads []struct { 88 | Key string 89 | UploadID string `xml:"UploadId"` 90 | StorageClass string 91 | Initiator *cos.Initiator 92 | Owner *cos.Owner 93 | Initiated string 94 | }, isTruncated bool, nextUploadIDMarker, nextKeyMarker string) { 95 | return fmt.Errorf("test GetUpload client error"), nil, false, "", "" 96 | }) 97 | defer patches.Reset() 98 | args := []string{"lsparts", fmt.Sprintf("cos://%s", testAlias)} 99 | cmd.SetArgs(args) 100 | e := cmd.Execute() 101 | fmt.Printf(" : %v", e) 102 | So(e, ShouldBeError) 103 | }) 104 | Convey("range uploads", func() { 105 | clearCmd() 106 | cmd := rootCmd 107 | patches := ApplyFunc(util.GetUploadsListForLs, func(c *cos.Client, cosUrl util.StorageUrl, uploadIDMarker, keyMarker string, limit int, recursive bool) (err error, uploads []struct { 108 | Key string 109 | UploadID string `xml:"UploadId"` 110 | StorageClass string 111 | Initiator *cos.Initiator 112 | Owner *cos.Owner 113 | Initiated string 114 | }, isTruncated bool, nextUploadIDMarker, nextKeyMarker string) { 115 | tmp := []struct { 116 | Key string 117 | UploadID string `xml:"UploadId"` 118 | StorageClass string 119 | Initiator *cos.Initiator 120 | Owner *cos.Owner 121 | Initiated string 122 | }{ 123 | { 124 | Key: "666", 125 | UploadID: "888", 126 | }, 127 | } 128 | 129 | return nil, tmp, false, "", "" 130 | }) 131 | defer patches.Reset() 132 | args := []string{"lsparts", fmt.Sprintf("cos://%s", testAlias)} 133 | cmd.SetArgs(args) 134 | e := cmd.Execute() 135 | So(e, ShouldBeNil) 136 | }) 137 | Convey("upload not exist", func() { 138 | clearCmd() 139 | cmd := rootCmd 140 | patches := ApplyFunc(util.CheckUploadExist, func(c *cos.Client, cosUrl util.StorageUrl, uploadId string) (exist bool, err error) { 141 | return false, nil 142 | }) 143 | defer patches.Reset() 144 | 145 | args := []string{"lsparts", fmt.Sprintf("cos://%s", testAlias), "--upload-id", "1"} 146 | cmd.SetArgs(args) 147 | e := cmd.Execute() 148 | fmt.Printf(" : %v", e) 149 | So(e, ShouldBeError) 150 | }) 151 | Convey("ls parts error", func() { 152 | clearCmd() 153 | cmd := rootCmd 154 | patches := ApplyFunc(util.CheckUploadExist, func(c *cos.Client, cosUrl util.StorageUrl, uploadId string) (exist bool, err error) { 155 | return true, nil 156 | }) 157 | defer patches.Reset() 158 | 159 | lsPatches := ApplyFunc(util.GetPartsListForLs, func(c *cos.Client, cosUrl util.StorageUrl, uploadId, partNumberMarker string, limit int) (err error, parts []cos.Object, isTruncated bool, nextPartNumberMarker string) { 160 | return fmt.Errorf("test GetUpload client error"), nil, false, "" 161 | }) 162 | defer lsPatches.Reset() 163 | 164 | args := []string{"lsparts", fmt.Sprintf("cos://%s", testAlias), "--upload-id", "1734424486bf8693045e9e926aa85008e3e58ddd5794fa68e0300d62d663c939e1a3b896d7"} 165 | cmd.SetArgs(args) 166 | e := cmd.Execute() 167 | fmt.Printf(" : %v", e) 168 | So(e, ShouldBeError) 169 | }) 170 | }) 171 | }) 172 | } 173 | -------------------------------------------------------------------------------- /cmd/mb.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "coscli/util" 6 | "fmt" 7 | 8 | logger "github.com/sirupsen/logrus" 9 | "github.com/spf13/cobra" 10 | "github.com/tencentyun/cos-go-sdk-v5" 11 | ) 12 | 13 | var mbCmd = &cobra.Command{ 14 | Use: "mb", 15 | Short: "Create bucket", 16 | Long: `Create bucket 17 | 18 | Format: 19 | ./coscli mb cos://- -e 20 | 21 | Example: 22 | ./coscli mb cos://examplebucket-1234567890 -e cos.ap-beijing.myqcloud.com`, 23 | Args: func(cmd *cobra.Command, args []string) error { 24 | if err := cobra.ExactArgs(1)(cmd, args); err != nil { 25 | return err 26 | } 27 | bucketIDName, cosPath := util.ParsePath(args[0]) 28 | if bucketIDName == "" || cosPath != "" { 29 | return fmt.Errorf("Invalid arguments! ") 30 | } 31 | return nil 32 | }, 33 | RunE: func(cmd *cobra.Command, args []string) error { 34 | err := createBucket(cmd, args) 35 | return err 36 | }, 37 | } 38 | 39 | func init() { 40 | rootCmd.AddCommand(mbCmd) 41 | 42 | mbCmd.Flags().StringP("region", "r", "", "Region") 43 | mbCmd.Flags().BoolP("ofs", "o", false, "Ofs") 44 | mbCmd.Flags().BoolP("maz", "m", false, "Maz") 45 | } 46 | 47 | func createBucket(cmd *cobra.Command, args []string) error { 48 | flagRegion, _ := cmd.Flags().GetString("region") 49 | flagOfs, _ := cmd.Flags().GetBool("ofs") 50 | flagMaz, _ := cmd.Flags().GetBool("maz") 51 | if param.Endpoint == "" && flagRegion != "" { 52 | param.Endpoint = fmt.Sprintf("cos.%s.myqcloud.com", flagRegion) 53 | } 54 | bucketIDName, _ := util.ParsePath(args[0]) 55 | 56 | c, err := util.CreateClient(&config, ¶m, bucketIDName) 57 | if err != nil { 58 | return err 59 | } 60 | opt := &cos.BucketPutOptions{ 61 | XCosACL: "", 62 | XCosGrantRead: "", 63 | XCosGrantWrite: "", 64 | XCosGrantFullControl: "", 65 | XCosGrantReadACP: "", 66 | XCosGrantWriteACP: "", 67 | CreateBucketConfiguration: &cos.CreateBucketConfiguration{}, 68 | } 69 | 70 | if flagOfs { 71 | opt.CreateBucketConfiguration.BucketArchConfig = "OFS" 72 | } 73 | 74 | if flagMaz { 75 | opt.CreateBucketConfiguration.BucketAZConfig = "MAZ" 76 | } 77 | 78 | _, err = c.Bucket.Put(context.Background(), opt) 79 | if err != nil { 80 | return err 81 | } 82 | logger.Infof("Create a new bucket! name: %s\n", bucketIDName) 83 | return nil 84 | } 85 | -------------------------------------------------------------------------------- /cmd/mb_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "coscli/util" 6 | "fmt" 7 | "reflect" 8 | "testing" 9 | 10 | . "github.com/agiledragon/gomonkey/v2" 11 | . "github.com/smartystreets/goconvey/convey" 12 | "github.com/tencentyun/cos-go-sdk-v5" 13 | ) 14 | 15 | func TestMbCmd(t *testing.T) { 16 | fmt.Println("TestMbCmd") 17 | testBucket = randStr(8) 18 | testAlias = testBucket + "-alias" 19 | setUp(testBucket, testAlias, testEndpoint, false, false) 20 | defer tearDown(testBucket, testAlias, testEndpoint, false) 21 | clearCmd() 22 | cmd := rootCmd 23 | cmd.SilenceErrors = true 24 | cmd.SilenceUsage = true 25 | Convey("Test coscli mb", t, func() { 26 | Convey("success", func() { 27 | Convey("Create maz bucket", func() { 28 | clearCmd() 29 | cmd := rootCmd 30 | args := []string{"mb", 31 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint, "--maz"} 32 | 33 | var c *cos.BucketService 34 | patches := ApplyMethodFunc(reflect.TypeOf(c), "Put", func(ctx context.Context, opt *cos.BucketPutOptions) (*cos.Response, error) { 35 | return nil, nil 36 | }) 37 | defer patches.Reset() 38 | cmd.SetArgs(args) 39 | e := cmd.Execute() 40 | fmt.Printf(" : %v", e) 41 | So(e, ShouldBeNil) 42 | }) 43 | }) 44 | Convey("fail", func() { 45 | Convey("Already exist", func() { 46 | clearCmd() 47 | cmd := rootCmd 48 | args := []string{"mb", 49 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint} 50 | cmd.SetArgs(args) 51 | e := cmd.Execute() 52 | fmt.Printf(" : %v", e) 53 | So(e, ShouldBeError) 54 | }) 55 | Convey("not enough arguments", func() { 56 | clearCmd() 57 | cmd := rootCmd 58 | args := []string{"mb"} 59 | cmd.SetArgs(args) 60 | e := cmd.Execute() 61 | fmt.Printf(" : %v", e) 62 | So(e, ShouldBeError) 63 | }) 64 | Convey("Invalid arguments", func() { 65 | clearCmd() 66 | cmd := rootCmd 67 | args := []string{"mb", "cos://"} 68 | cmd.SetArgs(args) 69 | e := cmd.Execute() 70 | fmt.Printf(" : %v", e) 71 | So(e, ShouldBeError) 72 | }) 73 | Convey("No Endpoint", func() { 74 | clearCmd() 75 | cmd := rootCmd 76 | patches := ApplyFunc(util.CreateClient, func(config *util.Config, param *util.Param, bucketIDName string) (client *cos.Client, err error) { 77 | return nil, fmt.Errorf(param.Endpoint) 78 | }) 79 | defer patches.Reset() 80 | args := []string{"mb", 81 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "--region", "guangzhou"} 82 | cmd.SetArgs(args) 83 | e := cmd.Execute() 84 | fmt.Printf(" : %v", e) 85 | So(e, ShouldBeError) 86 | }) 87 | Convey("Create Client", func() { 88 | clearCmd() 89 | cmd := rootCmd 90 | patches := ApplyFunc(util.CreateClient, func(config *util.Config, param *util.Param, bucketIDName string) (client *cos.Client, err error) { 91 | return nil, fmt.Errorf("test create client error") 92 | }) 93 | defer patches.Reset() 94 | args := []string{"mb", 95 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint} 96 | cmd.SetArgs(args) 97 | e := cmd.Execute() 98 | fmt.Printf(" : %v", e) 99 | So(e, ShouldBeError) 100 | }) 101 | Convey("Bucket.Put", func() { 102 | clearCmd() 103 | cmd := rootCmd 104 | var c *cos.BucketService 105 | patches := ApplyMethodFunc(reflect.TypeOf(c), "Put", func(ctx context.Context, opt *cos.BucketPutOptions) (*cos.Response, error) { 106 | return nil, fmt.Errorf("test bucket put error") 107 | }) 108 | defer patches.Reset() 109 | args := []string{"mb", 110 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint} 111 | cmd.SetArgs(args) 112 | e := cmd.Execute() 113 | fmt.Printf(" : %v", e) 114 | So(e, ShouldBeError) 115 | }) 116 | }) 117 | }) 118 | } 119 | -------------------------------------------------------------------------------- /cmd/rb.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "coscli/util" 6 | "fmt" 7 | logger "github.com/sirupsen/logrus" 8 | "github.com/spf13/cobra" 9 | ) 10 | 11 | var rbCmd = &cobra.Command{ 12 | Use: "rb", 13 | Short: "Remove bucket", 14 | Long: `Remove bucket 15 | 16 | Format: 17 | ./coscli rb cos://- -e 18 | 19 | Example: 20 | ./coscli rb cos://example-1234567890 -e cos.ap-beijing.myqcloud.com`, 21 | Args: func(cmd *cobra.Command, args []string) error { 22 | if err := cobra.ExactArgs(1)(cmd, args); err != nil { 23 | return err 24 | } 25 | bucketIDName, cosPath := util.ParsePath(args[0]) 26 | if bucketIDName == "" || cosPath != "" { 27 | return fmt.Errorf("Invalid arguments! ") 28 | } 29 | return nil 30 | }, 31 | RunE: func(cmd *cobra.Command, args []string) error { 32 | bucketIDName, _ := util.ParsePath(args[0]) 33 | flagRegion, _ := cmd.Flags().GetString("region") 34 | Force, _ := cmd.Flags().GetBool("force") 35 | failOutput, _ := cmd.Flags().GetBool("fail-output") 36 | failOutputPath, _ := cmd.Flags().GetString("fail-output-path") 37 | if param.Endpoint == "" && flagRegion != "" { 38 | param.Endpoint = fmt.Sprintf("cos.%s.myqcloud.com", flagRegion) 39 | } 40 | var choice string 41 | var err error 42 | 43 | c, err := util.NewClient(&config, ¶m, bucketIDName) 44 | if err != nil { 45 | return err 46 | } 47 | 48 | if Force { 49 | logger.Infof("Do you want to clear all inside the bucket and delete bucket %s ? (y/n)", bucketIDName) 50 | _, _ = fmt.Scanf("%s\n", &choice) 51 | if choice == "" || choice == "y" || choice == "Y" || choice == "yes" || choice == "Yes" || choice == "YES" { 52 | fo := &util.FileOperations{ 53 | Operation: util.Operation{ 54 | Force: true, 55 | FailOutput: failOutput, 56 | FailOutputPath: failOutputPath, 57 | }, 58 | Config: &config, 59 | Param: ¶m, 60 | ErrOutput: &util.ErrOutput{}, 61 | } 62 | 63 | // 根据s.Header判断是否是融合桶或者普通桶 64 | s, err := c.Bucket.Head(context.Background()) 65 | if err != nil { 66 | return err 67 | } 68 | 69 | if s.Header.Get("X-Cos-Bucket-Arch") == "OFS" { 70 | fo.Operation.AllVersions = false 71 | } else { 72 | // 判桶断是否开启版本控制，开启后需清理历史版本 73 | res, _, err := util.GetBucketVersioning(c) 74 | if err != nil { 75 | return err 76 | } 77 | if res.Status == util.VersionStatusEnabled { 78 | fo.Operation.AllVersions = true 79 | } 80 | } 81 | 82 | err = util.RemoveObjects(args, fo) 83 | if err != nil { 84 | return err 85 | } 86 | 87 | err = util.AbortUploads(args, fo) 88 | if err != nil { 89 | return err 90 | } 91 | 92 | err = util.RemoveBucket(bucketIDName, c) 93 | if err != nil { 94 | return err 95 | } 96 | } 97 | } else { 98 | logger.Infof("Do you want to delete %s? (y/n)", bucketIDName) 99 | _, _ = fmt.Scanf("%s\n", &choice) 100 | if choice == "" || choice == "y" || choice == "Y" || choice == "yes" || choice == "Yes" || choice == "YES" { 101 | err = util.RemoveBucket(bucketIDName, c) 102 | if err != nil { 103 | return err 104 | } 105 | } 106 | } 107 | return nil 108 | }, 109 | } 110 | 111 | func init() { 112 | rootCmd.AddCommand(rbCmd) 113 | rbCmd.Flags().BoolP("force", "f", false, "Clear all inside the bucket and delete bucket") 114 | rbCmd.Flags().StringP("region", "r", "", "Region") 115 | rbCmd.Flags().Bool("fail-output", true, "This option determines whether the error output for failed file uploads or downloads is enabled. If enabled, the error messages for any failed file transfers will be recorded in a file within the specified directory (if not specified, the default is coscli_output). If disabled, only the number of error files will be output to the console.") 116 | rbCmd.Flags().String("fail-output-path", "coscli_output", "This option specifies the designated error output folder where the error messages for failed file uploads or downloads will be recorded. By providing a custom folder path, you can control the location and name of the error output folder. If this option is not set, the default error log folder (coscli_output) will be used.") 117 | } 118 | -------------------------------------------------------------------------------- /cmd/rb_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "coscli/util" 6 | "fmt" 7 | . "github.com/agiledragon/gomonkey/v2" 8 | . "github.com/smartystreets/goconvey/convey" 9 | "github.com/tencentyun/cos-go-sdk-v5" 10 | "reflect" 11 | "testing" 12 | ) 13 | 14 | func TestRbCmd(t *testing.T) { 15 | fmt.Println("TestRbCmd") 16 | testBucket = randStr(8) 17 | // 仅创建桶，不添加配置 18 | setUp(testBucket, "nil", testEndpoint, false, true) 19 | clearCmd() 20 | cmd := rootCmd 21 | cmd.SilenceErrors = true 22 | cmd.SilenceUsage = true 23 | Convey("Test coscli rb", t, func() { 24 | Convey("fail", func() { 25 | Convey("Not enough arguments", func() { 26 | clearCmd() 27 | cmd := rootCmd 28 | args := []string{"rb"} 29 | cmd.SetArgs(args) 30 | e := cmd.Execute() 31 | fmt.Printf(" : %v", e) 32 | So(e, ShouldBeError) 33 | }) 34 | Convey("Invalid bukcetIDName", func() { 35 | clearCmd() 36 | cmd := rootCmd 37 | args := []string{"rb", "cos:/"} 38 | cmd.SetArgs(args) 39 | e := cmd.Execute() 40 | fmt.Printf(" : %v", e) 41 | So(e, ShouldBeError) 42 | }) 43 | Convey("Endpoint", func() { 44 | patches := ApplyFunc(util.RemoveBucket, func(string) error { 45 | return fmt.Errorf("test removeBucket error") 46 | }) 47 | defer patches.Reset() 48 | clearCmd() 49 | cmd := rootCmd 50 | args := []string{"rb", fmt.Sprintf("cos://%s-%s", testBucket, appID), "--region", "guangzhou"} 51 | cmd.SetArgs(args) 52 | e := cmd.Execute() 53 | fmt.Printf(" : %v", e) 54 | So(e, ShouldBeError) 55 | }) 56 | Convey("RemoveObjects", func() { 57 | patches := ApplyFunc(util.RemoveObjects, func(args []string, fo *util.FileOperations) error { 58 | return fmt.Errorf("test RemoveObjects error") 59 | }) 60 | defer patches.Reset() 61 | clearCmd() 62 | cmd := rootCmd 63 | args := []string{"rb", 64 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint, "-f"} 65 | cmd.SetArgs(args) 66 | e := cmd.Execute() 67 | fmt.Printf(" : %v", e) 68 | So(e, ShouldBeError) 69 | }) 70 | Convey("abortParts", func() { 71 | patches := ApplyFunc(util.AbortUploads, func(arg []string, fo *util.FileOperations) error { 72 | return fmt.Errorf("test abortParts error") 73 | }) 74 | defer patches.Reset() 75 | clearCmd() 76 | cmd := rootCmd 77 | args := []string{"rb", 78 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint, "-f"} 79 | cmd.SetArgs(args) 80 | e := cmd.Execute() 81 | fmt.Printf(" : %v", e) 82 | So(e, ShouldBeError) 83 | }) 84 | 85 | }) 86 | Convey("success and again", func() { 87 | Convey("success", func() { 88 | clearCmd() 89 | cmd := rootCmd 90 | args := []string{"rb", 91 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint} 92 | cmd.SetArgs(args) 93 | e := cmd.Execute() 94 | So(e, ShouldBeNil) 95 | }) 96 | Convey("Not exist", func() { 97 | clearCmd() 98 | cmd := rootCmd 99 | var c *cos.BucketService 100 | patches := ApplyMethodFunc(reflect.TypeOf(c), "Delete", func(ctx context.Context, opt ...*cos.BucketDeleteOptions) (*cos.Response, error) { 101 | return nil, fmt.Errorf("delete bucket error,bucket not exist") 102 | }) 103 | defer patches.Reset() 104 | args := []string{"rb", 105 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint} 106 | cmd.SetArgs(args) 107 | e := cmd.Execute() 108 | fmt.Printf(" : %v", e) 109 | So(e, ShouldBeError) 110 | }) 111 | }) 112 | Convey("removeBucket", func() { 113 | patches := ApplyFunc(util.RemoveBucket, func(string) error { 114 | return fmt.Errorf("test removeBucket error") 115 | }) 116 | defer patches.Reset() 117 | clearCmd() 118 | cmd := rootCmd 119 | args := []string{"rb", 120 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint, "-f"} 121 | cmd.SetArgs(args) 122 | e := cmd.Execute() 123 | fmt.Printf(" : %v", e) 124 | So(e, ShouldBeError) 125 | }) 126 | }) 127 | } 128 | -------------------------------------------------------------------------------- /cmd/restore.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | var restoreCmd = &cobra.Command{ 11 | Use: "restore", 12 | Short: "Restore objects", 13 | Long: `Restore objects 14 | 15 | Format: 16 | ./coscli restore cos://[/] [flags] 17 | 18 | Example: 19 | ./coscli restore cos://examplebucket/test/ -r -d 3 -m Expedited`, 20 | Args: cobra.ExactArgs(1), 21 | RunE: func(cmd *cobra.Command, args []string) error { 22 | recursive, _ := cmd.Flags().GetBool("recursive") 23 | include, _ := cmd.Flags().GetString("include") 24 | exclude, _ := cmd.Flags().GetString("exclude") 25 | days, _ := cmd.Flags().GetInt("days") 26 | mode, _ := cmd.Flags().GetString("mode") 27 | failOutput, _ := cmd.Flags().GetBool("fail-output") 28 | failOutputPath, _ := cmd.Flags().GetString("fail-output-path") 29 | 30 | if days < 1 || days > 365 { 31 | return fmt.Errorf("Flag --days should in range 1~365") 32 | } 33 | 34 | _, filters := util.GetFilter(include, exclude) 35 | 36 | fo := &util.FileOperations{ 37 | Operation: util.Operation{ 38 | Recursive: recursive, 39 | Filters: filters, 40 | FailOutput: failOutput, 41 | FailOutputPath: failOutputPath, 42 | Days: days, 43 | RestoreMode: mode, 44 | }, 45 | ErrOutput: &util.ErrOutput{}, 46 | Command: util.CommandRestore, 47 | } 48 | 49 | cosPath := "" 50 | if len(args) != 0 { 51 | cosPath = args[0] 52 | } 53 | cosUrl, err := util.FormatUrl(cosPath) 54 | if err != nil { 55 | return fmt.Errorf("cos url format error:%v", err) 56 | } 57 | if !cosUrl.IsCosUrl() { 58 | return fmt.Errorf("cospath needs to contain %s", util.SchemePrefix) 59 | } 60 | 61 | bucketName := cosUrl.(*util.CosUrl).Bucket 62 | c, err := util.NewClient(&config, ¶m, bucketName) 63 | if err != nil { 64 | return err 65 | } 66 | 67 | if recursive { 68 | err = util.RestoreObjects(c, cosUrl, fo) 69 | } else { 70 | _, err = util.TryRestoreObject(c, bucketName, cosUrl.(*util.CosUrl).Object, days, mode) 71 | } 72 | return err 73 | }, 74 | } 75 | 76 | func init() { 77 | rootCmd.AddCommand(restoreCmd) 78 | 79 | restoreCmd.Flags().BoolP("recursive", "r", false, "Restore objects recursively") 80 | restoreCmd.Flags().String("include", "", "Include files that meet the specified criteria") 81 | restoreCmd.Flags().String("exclude", "", "Exclude files that meet the specified criteria") 82 | restoreCmd.Flags().IntP("days", "d", 3, "Specifies the expiration time of temporary files") 83 | restoreCmd.Flags().StringP("mode", "m", "Standard", "Specifies the mode for fetching temporary files") 84 | restoreCmd.Flags().Bool("fail-output", true, "This option determines whether error output for failed file restore is enabled. If enabled, any error messages for failed file reheats will be recorded in a file within the specified directory (if not specified, the default directory is coscli_output). If disabled, only the number of error files will be output to the console.") 85 | restoreCmd.Flags().String("fail-output-path", "coscli_output", "This option specifies the error output folder where error messages for file restore failures will be recorded. By providing a custom folder path, you can control the location and name of the error output folder. If this option is not set, the default error log folder (coscli_output) will be used.") 86 | } 87 | -------------------------------------------------------------------------------- /cmd/restore_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | "testing" 7 | 8 | . "github.com/agiledragon/gomonkey/v2" 9 | . "github.com/smartystreets/goconvey/convey" 10 | "github.com/tencentyun/cos-go-sdk-v5" 11 | ) 12 | 13 | func TestRestoreCmd(t *testing.T) { 14 | fmt.Println("TestRestoreCmd") 15 | testBucket = randStr(8) 16 | testAlias = testBucket + "-alias" 17 | setUp(testBucket, testAlias, testEndpoint, false, false) 18 | defer tearDown(testBucket, testAlias, testEndpoint, false) 19 | genDir(testDir, 3) 20 | defer delDir(testDir) 21 | localObject := fmt.Sprintf("%s/small-file/0", testDir) 22 | localFileName := fmt.Sprintf("%s/small-file", testDir) 23 | cosObject := fmt.Sprintf("cos://%s", testAlias) 24 | cosFileName := fmt.Sprintf("cos://%s/%s", testAlias, "multi-small") 25 | clearCmd() 26 | cmd := rootCmd 27 | cmd.SilenceUsage = true 28 | cmd.SilenceErrors = true 29 | args1 := []string{"cp", localObject, cosObject, "--storage-class", "ARCHIVE"} 30 | args2 := []string{"cp", localFileName, cosFileName, "-r", "--storage-class", "ARCHIVE"} 31 | cmd.SetArgs(args1) 32 | cmd.Execute() 33 | clearCmd() 34 | cmd = rootCmd 35 | cmd.SetArgs(args2) 36 | cmd.Execute() 37 | Convey("Test coscli restore", t, func() { 38 | Convey("success", func() { 39 | Convey("RestoreObject", func() { 40 | clearCmd() 41 | cmd := rootCmd 42 | args := []string{"restore", 43 | fmt.Sprintf("%s/0", cosObject)} 44 | cmd.SetArgs(args) 45 | e := cmd.Execute() 46 | So(e, ShouldBeNil) 47 | }) 48 | Convey("RestoreObjects", func() { 49 | clearCmd() 50 | cmd := rootCmd 51 | args := []string{"restore", cosFileName, "-r"} 52 | cmd.SetArgs(args) 53 | e := cmd.Execute() 54 | So(e, ShouldBeNil) 55 | }) 56 | }) 57 | Convey("fail", func() { 58 | Convey("Not enough arguments", func() { 59 | clearCmd() 60 | cmd := rootCmd 61 | args := []string{"restore"} 62 | cmd.SetArgs(args) 63 | e := cmd.Execute() 64 | fmt.Printf(" : %v", e) 65 | So(e, ShouldBeError) 66 | }) 67 | Convey("days over range", func() { 68 | clearCmd() 69 | cmd := rootCmd 70 | args := []string{"restore", fmt.Sprintf("%s/0", cosObject), "--days", "366"} 71 | cmd.SetArgs(args) 72 | e := cmd.Execute() 73 | fmt.Printf(" : %v", e) 74 | So(e, ShouldBeError) 75 | }) 76 | Convey("FormatUrl", func() { 77 | patches := ApplyFunc(util.FormatUrl, func(urlStr string) (util.StorageUrl, error) { 78 | return nil, fmt.Errorf("test formaturl fail") 79 | }) 80 | defer patches.Reset() 81 | clearCmd() 82 | cmd := rootCmd 83 | args := []string{"restore", "invalid"} 84 | cmd.SetArgs(args) 85 | e := cmd.Execute() 86 | fmt.Printf(" : %v", e) 87 | So(e, ShouldBeError) 88 | }) 89 | Convey("not cos url", func() { 90 | clearCmd() 91 | cmd := rootCmd 92 | args := []string{"restore", "invalid"} 93 | cmd.SetArgs(args) 94 | e := cmd.Execute() 95 | fmt.Printf(" : %v", e) 96 | So(e, ShouldBeError) 97 | }) 98 | Convey("New Client", func() { 99 | clearCmd() 100 | cmd := rootCmd 101 | patches := ApplyFunc(util.NewClient, func(config *util.Config, param *util.Param, bucketName string) (client *cos.Client, err error) { 102 | return nil, fmt.Errorf("test new client error") 103 | }) 104 | defer patches.Reset() 105 | args := []string{"restore", cosFileName, "-r"} 106 | cmd.SetArgs(args) 107 | e := cmd.Execute() 108 | fmt.Printf(" : %v", e) 109 | So(e, ShouldBeError) 110 | }) 111 | 112 | Convey("RestoreObjects", func() { 113 | clearCmd() 114 | cmd := rootCmd 115 | patches := ApplyFunc(util.RestoreObjects, func(c *cos.Client, cosUrl util.StorageUrl, fo *util.FileOperations) error { 116 | return fmt.Errorf("test RestoreObjects error") 117 | }) 118 | defer patches.Reset() 119 | args := []string{"restore", cosFileName, "-r"} 120 | cmd.SetArgs(args) 121 | e := cmd.Execute() 122 | fmt.Printf(" : %v", e) 123 | So(e, ShouldBeError) 124 | }) 125 | }) 126 | }) 127 | } 128 | -------------------------------------------------------------------------------- /cmd/rm.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | var rmCmd = &cobra.Command{ 11 | Use: "rm", 12 | Short: "Remove objects", 13 | Long: `Remove objects 14 | 15 | Format: 16 | ./coscli rm cos://[/prefix/] [cos://[/prefix/]...] [flags] 17 | 18 | Example: 19 | ./coscli rm cos://example/test/ -r`, 20 | Args: func(cmd *cobra.Command, args []string) error { 21 | if err := cobra.MinimumNArgs(1)(cmd, args); err != nil { 22 | return err 23 | } 24 | for _, arg := range args { 25 | bucketName, _ := util.ParsePath(arg) 26 | if bucketName == "" { 27 | return fmt.Errorf("Invalid arguments! ") 28 | } 29 | } 30 | return nil 31 | }, 32 | RunE: func(cmd *cobra.Command, args []string) error { 33 | recursive, _ := cmd.Flags().GetBool("recursive") 34 | force, _ := cmd.Flags().GetBool("force") 35 | onlyCurrentDir, _ := cmd.Flags().GetBool("only-current-dir") 36 | retryNum, _ := cmd.Flags().GetInt("retry-num") 37 | include, _ := cmd.Flags().GetString("include") 38 | exclude, _ := cmd.Flags().GetString("exclude") 39 | failOutput, _ := cmd.Flags().GetBool("fail-output") 40 | failOutputPath, _ := cmd.Flags().GetString("fail-output-path") 41 | allVersions, _ := cmd.Flags().GetBool("all-versions") 42 | versionId, _ := cmd.Flags().GetString("version-id") 43 | 44 | _, filters := util.GetFilter(include, exclude) 45 | 46 | if versionId != "" && recursive { 47 | return fmt.Errorf("version-id can only be used to delete a single version of an object") 48 | } 49 | 50 | if allVersions && !recursive { 51 | return fmt.Errorf("all-versions can not be used to delete single object") 52 | } 53 | 54 | fo := &util.FileOperations{ 55 | Operation: util.Operation{ 56 | Recursive: recursive, 57 | Filters: filters, 58 | OnlyCurrentDir: onlyCurrentDir, 59 | Force: force, 60 | RetryNum: retryNum, 61 | FailOutput: failOutput, 62 | FailOutputPath: failOutputPath, 63 | AllVersions: allVersions, 64 | VersionId: versionId, 65 | }, 66 | Monitor: &util.FileProcessMonitor{}, 67 | Config: &config, 68 | Param: ¶m, 69 | ErrOutput: &util.ErrOutput{}, 70 | Command: util.CommandRm, 71 | } 72 | var err error 73 | if recursive { 74 | err = util.RemoveObjects(args, fo) 75 | } else { 76 | err = util.RemoveObject(args, fo) 77 | } 78 | return err 79 | }, 80 | } 81 | 82 | func init() { 83 | rootCmd.AddCommand(rmCmd) 84 | 85 | rmCmd.Flags().BoolP("recursive", "r", false, "Delete object recursively") 86 | rmCmd.Flags().BoolP("force", "f", false, "Force delete") 87 | rmCmd.Flags().Bool("only-current-dir", false, "Upload only the files in the current directory, ignoring subdirectories and their contents") 88 | rmCmd.Flags().Int("retry-num", 0, "Rate-limited retry. Specify 1-10 times. When multiple machines concurrently execute download operations on the same COS directory, rate-limited retry can be performed by specifying this parameter.") 89 | rmCmd.Flags().String("include", "", "List files that meet the specified criteria") 90 | rmCmd.Flags().String("exclude", "", "Exclude files that meet the specified criteria") 91 | rmCmd.Flags().Bool("fail-output", true, "This option determines whether error output for failed file deletions is enabled. If enabled, any error messages for failed file deletions will be recorded in a file within the specified directory (if not specified, the default directory is coscli_output). If disabled, only the number of error files will be output to the console.") 92 | rmCmd.Flags().String("fail-output-path", "coscli_output", "This option specifies the error output folder where error messages for failed file deletions will be recorded. By providing a custom folder path, you can control the location and name of the error output folder. If this option is not set, the default error log folder (coscli_output) will be used.") 93 | rmCmd.Flags().BoolP("all-versions", "", false, "remove all versions of objects, only available if bucket versioning is enabled.") 94 | rmCmd.Flags().String("version-id", "", "remove Downloading a specified version of a object, only available if bucket versioning is enabled.") 95 | } 96 | -------------------------------------------------------------------------------- /cmd/rm_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | . "github.com/agiledragon/gomonkey/v2" 7 | . "github.com/smartystreets/goconvey/convey" 8 | "github.com/tencentyun/cos-go-sdk-v5" 9 | "testing" 10 | ) 11 | 12 | func TestRmCmd(t *testing.T) { 13 | fmt.Println("TestRmCmd") 14 | testBucket = randStr(8) 15 | testAlias = testBucket + "-alias" 16 | testOfsBucket = randStr(8) 17 | testOfsBucketAlias = testOfsBucket + "-alias" 18 | testVersionBucket = randStr(8) 19 | testVersionBucketAlias = testVersionBucket + "-alias" 20 | setUp(testBucket, testAlias, testEndpoint, false, false) 21 | defer tearDown(testBucket, testAlias, testEndpoint, false) 22 | setUp(testOfsBucket, testOfsBucketAlias, testEndpoint, true, false) 23 | defer tearDown(testOfsBucket, testOfsBucketAlias, testEndpoint, false) 24 | setUp(testVersionBucket, testVersionBucketAlias, testEndpoint, false, true) 25 | defer tearDown(testVersionBucket, testVersionBucketAlias, testEndpoint, true) 26 | clearCmd() 27 | cmd := rootCmd 28 | cmd.SilenceErrors = true 29 | cmd.SilenceUsage = true 30 | genDir(testDir, 3) 31 | defer delDir(testDir) 32 | localFileName := fmt.Sprintf("%s/small-file", testDir) 33 | 34 | cosFileName := fmt.Sprintf("cos://%s/%s", testAlias, "multi-small") 35 | args := []string{"cp", localFileName, cosFileName, "-r"} 36 | cmd.SetArgs(args) 37 | cmd.Execute() 38 | 39 | ofsFileName := fmt.Sprintf("cos://%s/%s", testOfsBucketAlias, "multi-small") 40 | args = []string{"cp", localFileName, ofsFileName, "-r"} 41 | cmd.SetArgs(args) 42 | cmd.Execute() 43 | 44 | versioningFileName := fmt.Sprintf("cos://%s/%s", testVersionBucketAlias, "multi-small") 45 | args = []string{"cp", localFileName, versioningFileName, "-r"} 46 | cmd.SetArgs(args) 47 | cmd.Execute() 48 | Convey("Test coscli rm", t, func() { 49 | Convey("success", func() { 50 | Convey("rm single object", func() { 51 | clearCmd() 52 | cmd := rootCmd 53 | patches := ApplyFunc(util.GetBucketVersioning, func(c *cos.Client) (res *cos.BucketGetVersionResult, resp *cos.Response, err error) { 54 | res = &cos.BucketGetVersionResult{ 55 | Status: util.VersionStatusEnabled, 56 | } 57 | return res, nil, nil 58 | }) 59 | defer patches.Reset() 60 | patches = ApplyFunc(util.CheckCosObjectExist, func(c *cos.Client, prefix string, id ...string) (exist bool, err error) { 61 | return true, nil 62 | }) 63 | args := []string{"rm", versioningFileName, "--version-id", "123"} 64 | cmd.SetArgs(args) 65 | e := cmd.Execute() 66 | fmt.Printf(" : %v", e) 67 | So(e, ShouldBeNil) 68 | }) 69 | Convey("rm versions", func() { 70 | clearCmd() 71 | cmd := rootCmd 72 | args := []string{"rm", versioningFileName, "--all-versions", "-r"} 73 | cmd.SetArgs(args) 74 | e := cmd.Execute() 75 | fmt.Printf(" : %v", e) 76 | So(e, ShouldBeNil) 77 | }) 78 | Convey("rm cos objects", func() { 79 | clearCmd() 80 | cmd := rootCmd 81 | args := []string{"rm", cosFileName, "-r"} 82 | cmd.SetArgs(args) 83 | e := cmd.Execute() 84 | fmt.Printf(" : %v", e) 85 | So(e, ShouldBeNil) 86 | }) 87 | Convey("rm ofs objects", func() { 88 | clearCmd() 89 | cmd := rootCmd 90 | args := []string{"rm", ofsFileName, "-r"} 91 | cmd.SetArgs(args) 92 | e := cmd.Execute() 93 | fmt.Printf(" : %v", e) 94 | So(e, ShouldBeNil) 95 | }) 96 | }) 97 | Convey("fail", func() { 98 | Convey("Not enough arguments", func() { 99 | clearCmd() 100 | cmd := rootCmd 101 | args := []string{"rm"} 102 | cmd.SetArgs(args) 103 | e := cmd.Execute() 104 | fmt.Printf(" : %v", e) 105 | So(e, ShouldBeError) 106 | }) 107 | Convey("Invalid arguments", func() { 108 | clearCmd() 109 | cmd := rootCmd 110 | args := []string{"rm", "invaild"} 111 | cmd.SetArgs(args) 112 | e := cmd.Execute() 113 | fmt.Printf(" : %v", e) 114 | So(e, ShouldBeError) 115 | }) 116 | Convey("versionId use in recursive", func() { 117 | clearCmd() 118 | cmd := rootCmd 119 | args := []string{"rm", versioningFileName, "--version-id", "123", "-r"} 120 | cmd.SetArgs(args) 121 | e := cmd.Execute() 122 | fmt.Printf(" : %v", e) 123 | So(e, ShouldBeError) 124 | }) 125 | Convey("all-versions use in single object", func() { 126 | clearCmd() 127 | cmd := rootCmd 128 | args := []string{"rm", versioningFileName, "--all-versions"} 129 | cmd.SetArgs(args) 130 | e := cmd.Execute() 131 | fmt.Printf(" : %v", e) 132 | So(e, ShouldBeError) 133 | }) 134 | }) 135 | 136 | }) 137 | } 138 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | clilog "coscli/logger" 5 | "coscli/util" 6 | "fmt" 7 | logger "github.com/sirupsen/logrus" 8 | "log" 9 | "os" 10 | "strings" 11 | 12 | homedir "github.com/mitchellh/go-homedir" 13 | "github.com/spf13/cobra" 14 | "github.com/spf13/viper" 15 | ) 16 | 17 | var cfgFile string 18 | var initSkip bool 19 | var logPath string 20 | var config util.Config 21 | var param util.Param 22 | var cmdCnt int //控制某些函数在一个命令中被调用的次数 23 | 24 | var rootCmd = &cobra.Command{ 25 | Use: "coscli", 26 | Short: "Welcome to use coscli", 27 | Long: "Welcome to use coscli!", 28 | Run: func(cmd *cobra.Command, args []string) { 29 | _ = cmd.Help() 30 | }, 31 | Version: util.Version, 32 | } 33 | 34 | func Execute() error { 35 | rootCmd.SilenceErrors = true 36 | rootCmd.SilenceUsage = true 37 | return rootCmd.Execute() 38 | } 39 | 40 | func init() { 41 | cobra.OnInitialize(initConfig) 42 | 43 | rootCmd.PersistentFlags().StringVarP(&cfgFile, "config-path", "c", "", "config file path(default is $HOME/.cos.yaml)") 44 | rootCmd.PersistentFlags().StringVarP(¶m.SecretID, "secret-id", "i", "", "config secretId") 45 | rootCmd.PersistentFlags().StringVarP(¶m.SecretKey, "secret-key", "k", "", "config secretKey") 46 | rootCmd.PersistentFlags().StringVarP(¶m.SessionToken, "token", "", "", "config sessionToken") 47 | rootCmd.PersistentFlags().StringVarP(¶m.Endpoint, "endpoint", "e", "", "config endpoint") 48 | rootCmd.PersistentFlags().BoolVarP(¶m.Customized, "customized", "", false, "config customized") 49 | rootCmd.PersistentFlags().StringVarP(¶m.Protocol, "protocol", "p", "", "config protocol") 50 | rootCmd.PersistentFlags().BoolVarP(&initSkip, "init-skip", "", false, "skip config init") 51 | rootCmd.PersistentFlags().StringVarP(&logPath, "log-path", "", "", "coscli log dir") 52 | } 53 | 54 | func initConfig() { 55 | // 初始化日志路径 56 | clilog.InitLoggerWithDir(logPath) 57 | 58 | home, err := homedir.Dir() 59 | cobra.CheckErr(err) 60 | viper.SetConfigType("yaml") 61 | firstArg := "" 62 | if len(os.Args) > 1 { 63 | firstArg = os.Args[1] 64 | } 65 | 66 | if cfgFile != "" { 67 | if cfgFile[0] == '~' { 68 | cfgFile = home + cfgFile[1:] 69 | } 70 | if !strings.HasSuffix(cfgFile, ".yaml") { 71 | fmt.Println("config file need end with .yaml ") 72 | os.Exit(1) 73 | } 74 | viper.SetConfigFile(cfgFile) 75 | } else { 76 | _, err = os.Stat(home + "/.cos.yaml") 77 | if os.IsNotExist(err) { 78 | if firstArg != "config" { 79 | if !initSkip { 80 | log.Println("Welcome to coscli!\nWhen you use coscli for the first time, you need to input some necessary information to generate the default configuration file of coscli.") 81 | initConfigFile(false) 82 | cmdCnt++ 83 | } else { 84 | // 若无配置文件，则需有输入ak，sk及endpoint 85 | if param.SecretID == "" { 86 | logger.Fatalln("missing parameter SecretID") 87 | os.Exit(1) 88 | } 89 | if param.SecretKey == "" { 90 | logger.Fatalln("missing parameter SecretKey") 91 | os.Exit(1) 92 | } 93 | if param.Endpoint == "" { 94 | logger.Fatalln("missing parameter Endpoint") 95 | os.Exit(1) 96 | } 97 | return 98 | } 99 | } else { 100 | if !initSkip { 101 | log.Println("Welcome to coscli!\nWhen you use coscli for the first time, you need to input some necessary information to generate the default configuration file of coscli.") 102 | initConfigFile(false) 103 | cmdCnt++ 104 | } else { 105 | return 106 | } 107 | } 108 | 109 | } 110 | 111 | viper.AddConfigPath(home) 112 | viper.SetConfigName(".cos") 113 | } 114 | 115 | viper.AutomaticEnv() 116 | if err := viper.ReadInConfig(); err == nil { 117 | if err := viper.UnmarshalKey("cos", &config); err != nil { 118 | fmt.Println(err) 119 | os.Exit(1) 120 | } 121 | if config.Base.Protocol == "" { 122 | config.Base.Protocol = "https" 123 | } 124 | // 若未关闭秘钥加密，则先解密秘钥 125 | if config.Base.DisableEncryption != "true" { 126 | secretKey, err := util.DecryptSecret(config.Base.SecretKey) 127 | if err == nil { 128 | config.Base.SecretKey = secretKey 129 | } 130 | secretId, err := util.DecryptSecret(config.Base.SecretID) 131 | if err == nil { 132 | config.Base.SecretID = secretId 133 | } 134 | sessionToken, err := util.DecryptSecret(config.Base.SessionToken) 135 | if err == nil { 136 | config.Base.SessionToken = sessionToken 137 | } 138 | } 139 | 140 | } else { 141 | fmt.Println(err) 142 | os.Exit(1) 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /cmd/root_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | . "github.com/smartystreets/goconvey/convey" 8 | ) 9 | 10 | func TestSkipCmd(t *testing.T) { 11 | fmt.Println("TestSkipCmd") 12 | testBucket = randStr(8) 13 | testAlias = testBucket + "-alias" 14 | setUp(testBucket, testAlias, testEndpoint, false, false) 15 | defer tearDown(testBucket, testAlias, testEndpoint, false) 16 | clearCmd() 17 | cmd := rootCmd 18 | cmd.SilenceErrors = true 19 | cmd.SilenceUsage = true 20 | Convey("success", t, func() { 21 | cmd := rootCmd 22 | cmd.SilenceErrors = true 23 | cmd.SilenceUsage = true 24 | cosFileName := fmt.Sprintf("cos://%s", testAlias) 25 | args := []string{"ls", cosFileName, "--init-skip", "-i", "123", "-k", "456"} 26 | cmd.SetArgs(args) 27 | e := cmd.Execute() 28 | fmt.Printf(" : %v", e) 29 | So(e, ShouldBeError) 30 | }) 31 | } 32 | -------------------------------------------------------------------------------- /cmd/signurl.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "coscli/util" 6 | "fmt" 7 | "net/http" 8 | "net/url" 9 | "time" 10 | 11 | logger "github.com/sirupsen/logrus" 12 | "github.com/spf13/cobra" 13 | "github.com/tencentyun/cos-go-sdk-v5" 14 | ) 15 | 16 | var signurlCmd = &cobra.Command{ 17 | Use: "signurl", 18 | Short: "Gets the signed download URL", 19 | Long: `Gets the signed download URL 20 | 21 | Format: 22 | ./coscli signurl cos:/// [flags] 23 | 24 | Example: 25 | ./coscli signurl cos://examplebucket/test.jpg -t 100`, 26 | Args: cobra.ExactArgs(1), 27 | RunE: func(cmd *cobra.Command, args []string) error { 28 | time, _ := cmd.Flags().GetInt("time") 29 | var err error 30 | if util.IsCosPath(args[0]) { 31 | err = GetSignedURL(args[0], time) 32 | } else { 33 | return fmt.Errorf("cospath needs to contain cos://") 34 | } 35 | 36 | return err 37 | }, 38 | } 39 | 40 | func init() { 41 | rootCmd.AddCommand(signurlCmd) 42 | 43 | signurlCmd.Flags().IntP("time", "t", 10000, "Set the validity time of the signature(Default 10000)") 44 | } 45 | 46 | func GetSignedURL(path string, t int) error { 47 | bucketName, cosPath := util.ParsePath(path) 48 | c, err := util.NewClient(&config, ¶m, bucketName) 49 | if err != nil { 50 | return err 51 | } 52 | 53 | opt := &cos.PresignedURLOptions{ 54 | Query: &url.Values{}, 55 | Header: &http.Header{}, 56 | } 57 | // 格式化参数 58 | secretID, secretKey, secretToken := config.Base.SecretID, config.Base.SecretKey, config.Base.SessionToken 59 | if param.SecretID != "" { 60 | secretID = param.SecretID 61 | secretToken = "" 62 | } 63 | if param.SecretKey != "" { 64 | secretKey = param.SecretKey 65 | secretToken = "" 66 | } 67 | if param.SessionToken != "" { 68 | secretToken = param.SessionToken 69 | } 70 | if secretToken != "" { 71 | opt.Query.Add("x-cos-security-token", secretToken) 72 | } 73 | 74 | presignedURL, err := c.Object.GetPresignedURL(context.Background(), http.MethodGet, cosPath, 75 | secretID, secretKey, time.Second*time.Duration(t), opt) 76 | if err != nil { 77 | return err 78 | } 79 | 80 | logger.Infoln("Signed URL:") 81 | logger.Infoln(presignedURL) 82 | 83 | return nil 84 | } 85 | -------------------------------------------------------------------------------- /cmd/signurl_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "coscli/util" 6 | "fmt" 7 | "net/url" 8 | "reflect" 9 | "testing" 10 | "time" 11 | 12 | . "github.com/agiledragon/gomonkey/v2" 13 | . "github.com/smartystreets/goconvey/convey" 14 | "github.com/tencentyun/cos-go-sdk-v5" 15 | ) 16 | 17 | func TestSignurlCmd(t *testing.T) { 18 | fmt.Println("TestSignurlCmd") 19 | testBucket = randStr(8) 20 | testAlias = testBucket + "-alias" 21 | setUp(testBucket, testAlias, testEndpoint, false, false) 22 | defer tearDown(testBucket, testAlias, testEndpoint, false) 23 | genDir(testDir, 3) 24 | defer delDir(testDir) 25 | localFileName := fmt.Sprintf("%s/small-file/0", testDir) 26 | cosFileName := fmt.Sprintf("cos://%s", testAlias) 27 | clearCmd() 28 | cmd := rootCmd 29 | cmd.SilenceUsage = true 30 | cmd.SilenceErrors = true 31 | args := []string{"cp", localFileName, cosFileName} 32 | cmd.SetArgs(args) 33 | cmd.Execute() 34 | Convey("Test coscli signurl", t, func() { 35 | Convey("success", func() { 36 | clearCmd() 37 | cmd := rootCmd 38 | args := []string{"signurl", 39 | fmt.Sprintf("cos://%s/0", testAlias)} 40 | cmd.SetArgs(args) 41 | e := cmd.Execute() 42 | So(e, ShouldBeNil) 43 | }) 44 | Convey("failed", func() { 45 | Convey("Not enough arguments", func() { 46 | clearCmd() 47 | cmd := rootCmd 48 | args := []string{"abort"} 49 | cmd.SetArgs(args) 50 | e := cmd.Execute() 51 | fmt.Printf(" : %v", e) 52 | So(e, ShouldBeError) 53 | }) 54 | Convey("not cos", func() { 55 | clearCmd() 56 | cmd := rootCmd 57 | args := []string{"signurl", 58 | fmt.Sprintf("co//%s/0", testAlias)} 59 | cmd.SetArgs(args) 60 | e := cmd.Execute() 61 | fmt.Printf(" : %v", e) 62 | So(e, ShouldBeError) 63 | }) 64 | Convey("New Client", func() { 65 | clearCmd() 66 | cmd := rootCmd 67 | patches := ApplyFunc(util.NewClient, func(config *util.Config, param *util.Param, bucketName string) (client *cos.Client, err error) { 68 | return nil, fmt.Errorf("test new client error") 69 | }) 70 | defer patches.Reset() 71 | args := []string{"signurl", 72 | fmt.Sprintf("cos://%s/0", testAlias)} 73 | cmd.SetArgs(args) 74 | e := cmd.Execute() 75 | fmt.Printf(" : %v", e) 76 | So(e, ShouldBeError) 77 | }) 78 | Convey("Cover arguments and GetPresignedURL", func() { 79 | clearCmd() 80 | cmd := rootCmd 81 | var c *cos.ObjectService 82 | patches := ApplyMethodFunc(reflect.TypeOf(c), "GetPresignedURL", func(ctx context.Context, httpMethod string, name string, ak string, sk string, expired time.Duration, opt interface{}, signHost ...bool) (*url.URL, error) { 83 | return nil, fmt.Errorf("test getpresignedurl error") 84 | }) 85 | defer patches.Reset() 86 | patches.ApplyFunc(util.NewClient, func(config *util.Config, param *util.Param, bucketName string) (client *cos.Client, err error) { 87 | return &cos.Client{}, nil 88 | }) 89 | args := []string{"signurl", 90 | fmt.Sprintf("cos://%s/0", testAlias), "-i", "123", "-k", "123", "--token", "123"} 91 | cmd.SetArgs(args) 92 | e := cmd.Execute() 93 | fmt.Printf(" : %v", e) 94 | So(e, ShouldBeError) 95 | }) 96 | }) 97 | }) 98 | } 99 | -------------------------------------------------------------------------------- /cmd/symlink.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | logger "github.com/sirupsen/logrus" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | var symlinkCmd = &cobra.Command{ 11 | Use: "symlink", 12 | Short: "Create/Get symlink ", 13 | Long: `Create/Get symlink 14 | 15 | Format: 16 | ./coscli symlink --method create cos://-/test1 --link linkKey 17 | 18 | Example: 19 | ./coscli symlink --method create cos://examplebucket-1234567890/test1 --link linkKey`, 20 | Args: func(cmd *cobra.Command, args []string) error { 21 | if err := cobra.ExactArgs(1)(cmd, args); err != nil { 22 | return err 23 | } 24 | return nil 25 | }, 26 | RunE: func(cmd *cobra.Command, args []string) error { 27 | method, _ := cmd.Flags().GetString("method") 28 | linkKey, _ := cmd.Flags().GetString("link") 29 | 30 | //linkKey = strings.ToLower(linkKey) 31 | 32 | cosUrl, err := util.FormatUrl(args[0]) 33 | if err != nil { 34 | return err 35 | } 36 | if !cosUrl.IsCosUrl() { 37 | return fmt.Errorf("cospath needs to contain cos://") 38 | } 39 | 40 | // 实例化cos client 41 | bucketName := cosUrl.(*util.CosUrl).Bucket 42 | c, err := util.NewClient(&config, ¶m, bucketName) 43 | if err != nil { 44 | return err 45 | } 46 | 47 | if method == "create" { 48 | err = util.CreateSymlink(c, cosUrl, linkKey) 49 | if err != nil { 50 | return err 51 | } 52 | logger.Infof("Create symlink successfully! object: %s, symlink: %s", cosUrl.(*util.CosUrl).Object, linkKey) 53 | } else if method == "get" { 54 | res, err := util.GetSymlink(c, linkKey) 55 | if err != nil { 56 | return err 57 | } 58 | logger.Infof("Link-object: %s", res) 59 | } else { 60 | return fmt.Errorf("--method can only be selected create get and get") 61 | } 62 | 63 | return err 64 | }, 65 | } 66 | 67 | func init() { 68 | rootCmd.AddCommand(symlinkCmd) 69 | symlinkCmd.Flags().StringP("method", "", "", "Create/Get") 70 | symlinkCmd.Flags().StringP("link", "", "", "link key") 71 | } 72 | -------------------------------------------------------------------------------- /cmd/symlink_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "coscli/util" 5 | "fmt" 6 | "testing" 7 | 8 | . "github.com/agiledragon/gomonkey/v2" 9 | . "github.com/smartystreets/goconvey/convey" 10 | "github.com/tencentyun/cos-go-sdk-v5" 11 | ) 12 | 13 | func TestSymLinkCmd(t *testing.T) { 14 | fmt.Println("TestSymLinkCmd") 15 | testBucket = randStr(8) 16 | testAlias = testBucket + "-alias" 17 | linkKey := randStr(5) 18 | setUp(testBucket, testAlias, testEndpoint, false, false) 19 | defer tearDown(testBucket, testAlias, testEndpoint, false) 20 | genDir(testDir, 3) 21 | defer delDir(testDir) 22 | localFileName := fmt.Sprintf("%s/small-file/0", testDir) 23 | cosFileName := fmt.Sprintf("cos://%s", testAlias) 24 | clearCmd() 25 | cmd := rootCmd 26 | cmd.SilenceErrors = true 27 | cmd.SilenceUsage = true 28 | args := []string{"cp", localFileName, cosFileName, "-r"} 29 | clearCmd() 30 | cmd = rootCmd 31 | cmd.SetArgs(args) 32 | cmd.Execute() 33 | Convey("test coscli symlink", t, func() { 34 | Convey("fail", func() { 35 | Convey("Not enough argument", func() { 36 | clearCmd() 37 | cmd := rootCmd 38 | args := []string{"symlink"} 39 | cmd.SetArgs(args) 40 | e := cmd.Execute() 41 | fmt.Printf(" : %v", e) 42 | So(e, ShouldBeError) 43 | }) 44 | Convey("FormatUrl", func() { 45 | patches := ApplyFunc(util.FormatUrl, func(urlStr string) (util.StorageUrl, error) { 46 | return nil, fmt.Errorf("test FormatUrl error") 47 | }) 48 | defer patches.Reset() 49 | clearCmd() 50 | cmd := rootCmd 51 | args := []string{"symlink", "--method", "create", fmt.Sprintf("cos://%s/0", testAlias)} 52 | cmd.SetArgs(args) 53 | e := cmd.Execute() 54 | fmt.Printf(" : %v", e) 55 | So(e, ShouldBeError) 56 | }) 57 | Convey("Not CosUrl", func() { 58 | clearCmd() 59 | cmd := rootCmd 60 | args := []string{"symlink", "--method", "create", testAlias} 61 | cmd.SetArgs(args) 62 | e := cmd.Execute() 63 | fmt.Printf(" : %v", e) 64 | So(e, ShouldBeError) 65 | }) 66 | Convey("NewClient", func() { 67 | patches := ApplyFunc(util.NewClient, func(config *util.Config, param *util.Param, bucketName string) (client *cos.Client, err error) { 68 | return nil, fmt.Errorf("test NewClient error") 69 | }) 70 | defer patches.Reset() 71 | clearCmd() 72 | cmd := rootCmd 73 | args := []string{"symlink", "--method", "create", fmt.Sprintf("cos://%s/0", testAlias)} 74 | cmd.SetArgs(args) 75 | e := cmd.Execute() 76 | fmt.Printf(" : %v", e) 77 | So(e, ShouldBeError) 78 | }) 79 | Convey("CreateSymlink", func() { 80 | patches := ApplyFunc(util.CreateSymlink, func(c *cos.Client, cosUrl util.StorageUrl, linkKey string) error { 81 | return fmt.Errorf("test CreateSymlink error") 82 | }) 83 | defer patches.Reset() 84 | clearCmd() 85 | cmd := rootCmd 86 | args := []string{"symlink", "--method", "create", fmt.Sprintf("cos://%s/0", testAlias)} 87 | cmd.SetArgs(args) 88 | e := cmd.Execute() 89 | fmt.Printf(" : %v", e) 90 | So(e, ShouldBeError) 91 | }) 92 | Convey("GetSymlink", func() { 93 | patches := ApplyFunc(util.GetSymlink, func(c *cos.Client, linkKey string) (res string, err error) { 94 | return "", fmt.Errorf("test GetSymlink error") 95 | }) 96 | defer patches.Reset() 97 | clearCmd() 98 | cmd := rootCmd 99 | args := []string{"symlink", "--method", "get", fmt.Sprintf("cos://%s/0", testAlias)} 100 | cmd.SetArgs(args) 101 | e := cmd.Execute() 102 | fmt.Printf(" : %v", e) 103 | So(e, ShouldBeError) 104 | }) 105 | Convey("Invalid argument", func() { 106 | clearCmd() 107 | cmd := rootCmd 108 | args := []string{"symlink", "--method", "invalid", fmt.Sprintf("cos://%s/0", testAlias)} 109 | cmd.SetArgs(args) 110 | e := cmd.Execute() 111 | fmt.Printf(" : %v", e) 112 | So(e, ShouldBeError) 113 | }) 114 | }) 115 | Convey("success", func() { 116 | Convey("create", func() { 117 | clearCmd() 118 | cmd := rootCmd 119 | args := []string{"symlink", "--method", "create", fmt.Sprintf("cos://%s/0", testAlias), "--link", linkKey} 120 | cmd.SetArgs(args) 121 | e := cmd.Execute() 122 | So(e, ShouldBeNil) 123 | }) 124 | Convey("get", func() { 125 | clearCmd() 126 | cmd := rootCmd 127 | args := []string{"symlink", "--method", "get", fmt.Sprintf("cos://%s", testAlias), "--link", linkKey} 128 | cmd.SetArgs(args) 129 | e := cmd.Execute() 130 | So(e, ShouldBeNil) 131 | }) 132 | }) 133 | }) 134 | } 135 | -------------------------------------------------------------------------------- /cmd/testconfig_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "math/rand" 7 | "os" 8 | "time" 9 | 10 | "github.com/mitchellh/go-homedir" 11 | logger "github.com/sirupsen/logrus" 12 | "github.com/spf13/pflag" 13 | "github.com/spf13/viper" 14 | ) 15 | 16 | var testDir = "test-tmp-dir" 17 | 18 | var appID string 19 | 20 | var testBucket string 21 | var testAlias string 22 | var testBucket1 string 23 | var testAlias1 string 24 | var testBucket2 string 25 | var testAlias2 string 26 | var testEndpoint = "cos.ap-guangzhou.myqcloud.com" 27 | 28 | var testOfsBucket string 29 | var testOfsBucketAlias string 30 | 31 | var testVersionBucket string 32 | var testVersionBucketAlias string 33 | 34 | func init() { 35 | // 读取配置文件 36 | getConfig() 37 | // 初始化 app-id 38 | name := config.Buckets[0].Name 39 | if len(name) > 10 { 40 | appID = name[len(name)-10:] 41 | } else { 42 | logger.Errorln("请先配置一个桶") 43 | return 44 | } 45 | } 46 | 47 | func getConfig() { 48 | home, err := homedir.Dir() 49 | if err != nil { 50 | logger.Errorln(err) 51 | } 52 | viper.SetConfigFile(home + "/.cos.yaml") 53 | 54 | if err = viper.ReadInConfig(); err != nil { 55 | logger.Errorln(err) 56 | } 57 | if err = viper.UnmarshalKey("cos", &config); err != nil { 58 | logger.Errorln(err) 59 | } 60 | } 61 | 62 | func randStr(length int) string { 63 | str := "0123456789abcdefghijklmnopqrstuvwxyz" 64 | bytes := []byte(str) 65 | result := []byte{} 66 | rand.Seed(time.Now().UnixNano() + int64(rand.Intn(100))) 67 | for i := 0; i < length; i++ { 68 | result = append(result, bytes[rand.Intn(len(bytes))]) 69 | } 70 | return string(result) 71 | } 72 | 73 | func setUp(testBucket, testAlias, testEndpoint string, ofs bool, versioning bool) { 74 | // 创建测试桶 75 | logger.Infoln(fmt.Sprintf("创建测试桶：%s-%s %s", testBucket, appID, testEndpoint)) 76 | clearCmd() 77 | cmd := rootCmd 78 | var args []string 79 | if ofs { 80 | args = []string{"mb", 81 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint, "-o"} 82 | } else { 83 | args = []string{"mb", 84 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint} 85 | } 86 | cmd.SetArgs(args) 87 | err := cmd.Execute() 88 | if err != nil { 89 | logger.Errorln(err) 90 | } 91 | 92 | // 开启多版本 93 | if versioning { 94 | args := []string{"bucket-versioning", "--method", "put", 95 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "Enabled"} 96 | cmd.SetArgs(args) 97 | err := cmd.Execute() 98 | if err != nil { 99 | logger.Errorln(err) 100 | } 101 | } 102 | 103 | if testAlias == "nil" { 104 | return 105 | } 106 | 107 | // 更新配置文件 108 | logger.Infoln(fmt.Sprintf("更新配置文件：%s", testBucket)) 109 | if testAlias == "" { 110 | args = []string{"config", "add", "-b", 111 | fmt.Sprintf("%s-%s", testBucket, appID), "-e", testEndpoint} 112 | } else { 113 | args = []string{"config", "add", "-b", 114 | fmt.Sprintf("%s-%s", testBucket, appID), "-e", testEndpoint, "-a", testAlias} 115 | } 116 | if ofs { 117 | args = append(args, "-o") 118 | } 119 | clearCmd() 120 | cmd = rootCmd 121 | cmd.SetArgs(args) 122 | err = cmd.Execute() 123 | if err != nil { 124 | logger.Errorln(err) 125 | } 126 | 127 | // 更新 Config 128 | getConfig() 129 | } 130 | 131 | func tearDown(testBucket, testAlias, testEndpoint string, versioning bool) { 132 | if testAlias == "" { 133 | testAlias = testBucket + "-" + appID 134 | } 135 | // 清空测试桶 136 | logger.Infoln(fmt.Sprintf("清空测试桶文件：%s", testAlias)) 137 | var args []string 138 | 139 | if versioning { 140 | args = []string{"rm", 141 | fmt.Sprintf("cos://%s", testAlias), "-r", "-f", "--all-versions"} 142 | } else { 143 | args = []string{"rm", 144 | fmt.Sprintf("cos://%s", testAlias), "-r", "-f"} 145 | } 146 | 147 | clearCmd() 148 | cmd := rootCmd 149 | cmd.SetArgs(args) 150 | err := cmd.Execute() 151 | if err != nil { 152 | logger.Errorln(err) 153 | } 154 | logger.Infoln(fmt.Sprintf("清空测试桶碎片：%s", testAlias)) 155 | args = []string{"abort", 156 | fmt.Sprintf("cos://%s", testAlias)} 157 | clearCmd() 158 | cmd = rootCmd 159 | cmd.SetArgs(args) 160 | err = cmd.Execute() 161 | if err != nil { 162 | logger.Errorln(err) 163 | } 164 | 165 | // 删除测试桶 166 | logger.Infoln(fmt.Sprintf("删除测试桶：%s-%s %s", testBucket, appID, testEndpoint)) 167 | args = []string{"rb", 168 | fmt.Sprintf("cos://%s-%s", testBucket, appID), "-e", testEndpoint} 169 | clearCmd() 170 | cmd = rootCmd 171 | cmd.SetArgs(args) 172 | err = cmd.Execute() 173 | if err != nil { 174 | logger.Errorln(err) 175 | } 176 | 177 | // 更新配置文件 178 | logger.Infoln(fmt.Sprintf("更新配置文件：%s", testAlias)) 179 | args = []string{"config", "delete", "-a", testAlias} 180 | clearCmd() 181 | cmd = rootCmd 182 | cmd.SetArgs(args) 183 | err = cmd.Execute() 184 | if err != nil { 185 | logger.Errorln(err) 186 | } 187 | } 188 | 189 | func copyYaml() { 190 | // 打开源文件 191 | sourceFile, err := os.Open("/root/.cos.yaml") 192 | if err != nil { 193 | logger.Errorln("failed to open source file: %w", err) 194 | } 195 | defer sourceFile.Close() 196 | 197 | // 创建目标文件 198 | destinationFile, err := os.Create("test.yaml") 199 | if err != nil { 200 | logger.Errorln("failed to create destination file: %w", err) 201 | } 202 | defer destinationFile.Close() 203 | 204 | // 复制文件内容 205 | _, err = io.Copy(destinationFile, sourceFile) 206 | if err != nil { 207 | logger.Errorln("failed to copy file content: %w", err) 208 | } 209 | 210 | } 211 | 212 | func delYaml() { 213 | if err := os.RemoveAll("test.yaml"); err != nil { 214 | logger.Errorln("delDir error: 删除文件夹失败") 215 | } 216 | } 217 | 218 | // 创建文件 219 | func genFile(fileName string, size int) { 220 | data := make([]byte, 0) 221 | 222 | rand.Seed(time.Now().Unix()) 223 | for i := 0; i < size; i++ { 224 | u := uint8(rand.Intn(256)) 225 | data = append(data, u) 226 | } 227 | 228 | f, err := os.Create(fileName) 229 | if err != nil { 230 | logger.Errorln("genFile error: 创建文件失败") 231 | } 232 | defer f.Close() 233 | 234 | n, err := f.Write(data) 235 | if err != nil || n != size { 236 | logger.Errorln("genFile error: 数据写入失败") 237 | } 238 | } 239 | 240 | // 创建目录，有 num 个小文件和3个大文件 241 | func genDir(dirName string, num int) { 242 | if err := os.MkdirAll(fmt.Sprintf("%s/small-file", dirName), os.ModePerm); err != nil { 243 | logger.Errorln("genDir error: 创建文件夹失败") 244 | } 245 | if err := os.MkdirAll(fmt.Sprintf("%s/big-file", dirName), os.ModePerm); err != nil { 246 | logger.Errorln("genDir error: 创建文件夹失败") 247 | } 248 | 249 | logger.Infoln(fmt.Sprintf("生成小文件：%s/small-file", dirName)) 250 | for i := 0; i < num; i++ { 251 | genFile(fmt.Sprintf("%s/small-file/%d", dirName, i), 30*1024) 252 | } 253 | logger.Infoln(fmt.Sprintf("生成大文件：%s/big-file", dirName)) 254 | for i := 0; i < 3; i++ { 255 | genFile(fmt.Sprintf("%s/big-file/%d", dirName, i), 5*1024*1024) 256 | } 257 | } 258 | 259 | func delDir(dirName string) { 260 | logger.Infoln(fmt.Sprintf("删除测试临时文件夹：%s", dirName)) 261 | if err := os.RemoveAll(dirName); err != nil { 262 | logger.Errorln("delDir error: 删除文件夹失败") 263 | } 264 | } 265 | 266 | func clearCmd() { 267 | rootCmd.Flags().VisitAll(func(flag *pflag.Flag) { 268 | flag.Value.Set(flag.DefValue) 269 | }) 270 | 271 | // 重置子命令的状态 272 | for _, subCmd := range rootCmd.Commands() { 273 | subCmd.Flags().VisitAll(func(flag *pflag.Flag) { 274 | flag.Value.Set(flag.DefValue) 275 | }) 276 | } 277 | } 278 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module coscli 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/agiledragon/gomonkey/v2 v2.12.0 7 | github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible 8 | github.com/mitchellh/go-homedir v1.1.0 9 | github.com/mozillazg/go-httpheader v0.4.0 10 | github.com/olekukonko/tablewriter v0.0.5 11 | github.com/sirupsen/logrus v1.2.0 12 | github.com/smartystreets/goconvey v1.6.4 13 | github.com/spf13/cobra v1.1.3 14 | github.com/spf13/viper v1.10.0 15 | github.com/syndtr/goleveldb v1.0.0 16 | github.com/tencentyun/cos-go-sdk-v5 v0.7.62 17 | ) 18 | 19 | require golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect 20 | 21 | require ( 22 | github.com/clbanning/mxj v1.8.4 // indirect 23 | github.com/fsnotify/fsnotify v1.5.1 // indirect 24 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect 25 | github.com/google/go-querystring v1.1.0 // indirect 26 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect 27 | github.com/hashicorp/hcl v1.0.0 // indirect 28 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 29 | github.com/jtolds/gls v4.20.0+incompatible // indirect 30 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect 31 | github.com/lestrrat-go/strftime v1.0.6 // indirect 32 | github.com/magiconair/properties v1.8.5 // indirect 33 | github.com/mattn/go-runewidth v0.0.9 // indirect 34 | github.com/mitchellh/mapstructure v1.5.0 // indirect 35 | github.com/pelletier/go-toml v1.9.4 // indirect 36 | github.com/pkg/errors v0.9.1 // indirect 37 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d // indirect 38 | github.com/spf13/afero v1.6.0 // indirect 39 | github.com/spf13/cast v1.4.1 // indirect 40 | github.com/spf13/jwalterweatherman v1.1.0 // indirect 41 | github.com/spf13/pflag v1.0.5 42 | github.com/subosito/gotenv v1.2.0 // indirect 43 | golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect 44 | golang.org/x/sys v0.0.0-20211205182925-97ca703d548d // indirect 45 | golang.org/x/text v0.3.7 // indirect 46 | gopkg.in/ini.v1 v1.66.2 // indirect 47 | gopkg.in/yaml.v2 v2.4.0 // indirect 48 | ) 49 | -------------------------------------------------------------------------------- /logger/logger.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "path/filepath" 7 | "runtime" 8 | "strings" 9 | "time" 10 | 11 | rotatelogs "github.com/lestrrat-go/file-rotatelogs" 12 | "github.com/sirupsen/logrus" 13 | log "github.com/sirupsen/logrus" 14 | ) 15 | 16 | var logName = "coscli.log" 17 | 18 | func InitLoggerWithDir(path string) { 19 | if path == "" { 20 | var err error 21 | path, err = filepath.Abs(filepath.Dir(os.Args[0])) 22 | if err != nil { 23 | path = "." 24 | } 25 | } 26 | 27 | logPath := "" 28 | 29 | if strings.HasSuffix(path, ".log") { 30 | logPath = path 31 | } else { 32 | logPath = filepath.Join(path, logName) 33 | } 34 | 35 | fsWriter, err := rotatelogs.New( 36 | logPath, 37 | rotatelogs.WithMaxAge(time.Duration(168)*time.Hour), 38 | rotatelogs.WithRotationTime(time.Duration(24)*time.Hour), 39 | ) 40 | if err != nil { 41 | panic(err) 42 | } 43 | 44 | multiWriter := io.MultiWriter(fsWriter, os.Stdout) 45 | log.SetOutput(multiWriter) 46 | log.SetLevel(log.InfoLevel) 47 | forceColors := true 48 | if runtime.GOOS == "windows" { 49 | forceColors = false 50 | } 51 | log.SetFormatter(&logrus.TextFormatter{ 52 | ForceColors: forceColors, 53 | TimestampFormat: "2006-01-02 15:04:05", //时间格式 54 | FullTimestamp: true, 55 | }) 56 | } 57 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2021 NAME HERE 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package main 17 | 18 | import ( 19 | "coscli/cmd" 20 | logger "github.com/sirupsen/logrus" 21 | "os" 22 | ) 23 | 24 | func main() { 25 | if err := cmd.Execute(); err != nil { 26 | logger.Errorln(err) 27 | //logger.Infoln(cmd.UsageString()) 28 | os.Exit(1) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /util/bucket.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | func FindBucket(config *Config, bucketName string) (Bucket, int, error) { 4 | for i, b := range config.Buckets { 5 | if b.Alias == bucketName { 6 | return b, i, nil 7 | } 8 | } 9 | for i, b := range config.Buckets { 10 | if b.Name == bucketName { 11 | return b, i, nil 12 | } 13 | } 14 | var tmpBucket Bucket 15 | tmpBucket.Name = bucketName 16 | return tmpBucket, -1, nil 17 | // return Bucket{}, -1, errors.New("Bucket not exist! Use \"./coscli config show\" to check config file please! ") 18 | } 19 | -------------------------------------------------------------------------------- /util/cam_auth.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "time" 10 | ) 11 | 12 | const ( 13 | CamUrl = "http://metadata.tencentyun.com/meta-data/cam/security-credentials/" 14 | ) 15 | 16 | type Data struct { 17 | TmpSecretId string `json:"TmpSecretId"` 18 | TmpSecretKey string `json:"TmpSecretKey"` 19 | ExpiredTime int `json:"ExpiredTime"` 20 | Expiration string `json:"Expiration"` 21 | Token string `json:"Token"` 22 | Code string `json:"Code"` 23 | } 24 | 25 | var data Data 26 | 27 | func CamAuth(roleName string) (data Data, err error) { 28 | if roleName == "" { 29 | return data, fmt.Errorf("Get cam auth error : roleName not set") 30 | } 31 | 32 | // 创建HTTP客户端 33 | client := &http.Client{} 34 | 35 | // 创建一个5秒的超时上下文 36 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 37 | defer cancel() 38 | 39 | // 创建一个HTTP GET请求并将上下文与其关联 40 | req, err := http.NewRequest("GET", CamUrl+roleName, nil) 41 | if err != nil { 42 | return data, fmt.Errorf("Get cam auth error : create request error[%v]", err) 43 | } 44 | req = req.WithContext(ctx) 45 | 46 | // 发起HTTP GET请求 47 | res, err := client.Do(req) 48 | if err != nil { 49 | // 检查是否超时错误 50 | if ctx.Err() == context.DeadlineExceeded { 51 | return data, fmt.Errorf("Get cam auth timeout[%v]", ctx.Err()) 52 | } else { 53 | return data, fmt.Errorf("Get cam auth error : request error[%v]", err) 54 | } 55 | } 56 | 57 | defer res.Body.Close() 58 | 59 | body, err := ioutil.ReadAll(res.Body) 60 | if err != nil { 61 | return data, fmt.Errorf("Get cam auth error : get response error[%v]", err) 62 | } 63 | 64 | err = json.Unmarshal(body, &data) 65 | if err != nil { 66 | return data, fmt.Errorf("Get cam auth error : auth error[%v]", err) 67 | } 68 | 69 | if data.Code != "Success" { 70 | return data, fmt.Errorf("Get cam auth error : response error[%v]", err) 71 | } 72 | 73 | return data, nil 74 | } 75 | -------------------------------------------------------------------------------- /util/cat_object.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/tencentyun/cos-go-sdk-v5" 7 | "io" 8 | "os" 9 | ) 10 | 11 | func CatObject(c *cos.Client, cosUrl StorageUrl) error { 12 | opt := &cos.ObjectGetOptions{ 13 | ResponseContentType: "text/html", 14 | } 15 | res, err := c.Object.Get(context.Background(), cosUrl.(*CosUrl).Object, opt) 16 | if err != nil { 17 | return err 18 | } 19 | 20 | io.Copy(os.Stdout, res.Body) 21 | fmt.Printf("\n") 22 | return nil 23 | } 24 | -------------------------------------------------------------------------------- /util/client.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "github.com/tencentyun/cos-go-sdk-v5" 6 | "net/http" 7 | "time" 8 | ) 9 | 10 | var secretID, secretKey, secretToken string 11 | 12 | // 根据桶别名，从配置文件中加载信息，创建客户端 13 | func NewClient(config *Config, param *Param, bucketName string, options ...*FileOperations) (client *cos.Client, err error) { 14 | if config.Base.Mode == "CvmRole" { 15 | // 若使用 CvmRole 方式，则需请求请求CAM的服务，获取临时密钥 16 | data, err = CamAuth(config.Base.CvmRoleName) 17 | if err != nil { 18 | return client, err 19 | } 20 | secretID = data.TmpSecretId 21 | secretKey = data.TmpSecretKey 22 | secretToken = data.Token 23 | } else { 24 | // SecretKey 方式则直接获取用户配置文件中设置的密钥 25 | secretID = config.Base.SecretID 26 | secretKey = config.Base.SecretKey 27 | secretToken = config.Base.SessionToken 28 | } 29 | // 若参数中有传 SecretID 或 SecretKey ，需将之前赋值的SessionToken置为空，否则会出现使用参数的 SecretID 和 SecretKey ，却使用了CvmRole方式返回的token，导致鉴权失败 30 | if param.SecretID != "" { 31 | secretID = param.SecretID 32 | secretToken = "" 33 | } 34 | if param.SecretKey != "" { 35 | secretKey = param.SecretKey 36 | secretToken = "" 37 | } 38 | if param.SessionToken != "" { 39 | secretToken = param.SessionToken 40 | } 41 | 42 | if secretID == "" { 43 | return client, fmt.Errorf("secretID is missing ") 44 | } 45 | 46 | if secretKey == "" { 47 | return client, fmt.Errorf("secretKey is missing") 48 | } 49 | 50 | if bucketName == "" { // 不指定 bucket，则创建用于发送 Service 请求的客户端 51 | client = cos.NewClient(GenBaseURL(config, param), &http.Client{ 52 | Transport: &cos.AuthorizationTransport{ 53 | SecretID: secretID, 54 | SecretKey: secretKey, 55 | SessionToken: secretToken, 56 | }, 57 | }) 58 | } else { 59 | url, err := GenURL(config, param, bucketName) 60 | if err != nil { 61 | return client, err 62 | } 63 | 64 | var httpClient *http.Client 65 | // 如果使用长链接则调整连接池大小至并发数 66 | if len(options) > 0 && options[0] != nil && !options[0].Operation.DisableLongLinks { 67 | longLinksNums := 0 68 | if options[0].Operation.LongLinksNums > 0 { 69 | longLinksNums = options[0].Operation.LongLinksNums 70 | } else { 71 | longLinksNums = options[0].Operation.Routines 72 | } 73 | httpClient = &http.Client{ 74 | Transport: &cos.AuthorizationTransport{ 75 | SecretID: secretID, 76 | SecretKey: secretKey, 77 | SessionToken: secretToken, 78 | Transport: &http.Transport{ 79 | MaxIdleConnsPerHost: longLinksNums, 80 | MaxIdleConns: longLinksNums, 81 | }, 82 | }, 83 | } 84 | } else { 85 | // 若没有传递 options 或者没有设置 DisableLongLinks 86 | httpClient = &http.Client{ 87 | Transport: &cos.AuthorizationTransport{ 88 | SecretID: secretID, 89 | SecretKey: secretKey, 90 | SessionToken: secretToken, 91 | }, 92 | } 93 | } 94 | 95 | client = cos.NewClient(url, httpClient) 96 | } 97 | 98 | // 切换备用域名开关 99 | if config.Base.CloseAutoSwitchHost == "true" { 100 | client.Conf.RetryOpt.AutoSwitchHost = false 101 | } 102 | 103 | // 服务端错误重试（默认10次，每次间隔1s） 104 | if len(options) > 0 && options[0] != nil && options[0].Operation.ErrRetryNum > 0 { 105 | client.Conf.RetryOpt.Count = options[0].Operation.ErrRetryNum 106 | if options[0].Operation.ErrRetryInterval > 0 { 107 | client.Conf.RetryOpt.Interval = time.Duration(options[0].Operation.ErrRetryInterval) 108 | } else { 109 | client.Conf.RetryOpt.Interval = time.Duration(1) 110 | } 111 | } else { 112 | client.Conf.RetryOpt.Count = 10 113 | client.Conf.RetryOpt.Interval = time.Duration(1) 114 | } 115 | 116 | // 修改 UserAgent 117 | client.UserAgent = Package + "-" + Version 118 | 119 | return client, nil 120 | } 121 | 122 | // 根据函数参数创建客户端 123 | func CreateClient(config *Config, param *Param, bucketIDName string) (client *cos.Client, err error) { 124 | if config.Base.Mode == "CvmRole" { 125 | // 若使用 CvmRole 方式，则需请求请求CAM的服务，获取临时密钥 126 | data, err = CamAuth(config.Base.CvmRoleName) 127 | if err != nil { 128 | return client, err 129 | } 130 | 131 | secretID = data.TmpSecretId 132 | secretKey = data.TmpSecretKey 133 | secretToken = data.Token 134 | } else { 135 | // SecretKey 方式则直接获取用户配置文件中设置的密钥 136 | secretID = config.Base.SecretID 137 | secretKey = config.Base.SecretKey 138 | secretToken = config.Base.SessionToken 139 | } 140 | 141 | // 若参数中有传 SecretID 或 SecretKey ，需将之前赋值的SessionToken置为空，否则会出现使用参数的 SecretID 和 SecretKey ，却使用了CvmRole方式返回的token，导致鉴权失败 142 | if param.SecretID != "" { 143 | secretID = param.SecretID 144 | secretToken = "" 145 | } 146 | if param.SecretKey != "" { 147 | secretKey = param.SecretKey 148 | secretToken = "" 149 | } 150 | if param.SessionToken != "" { 151 | secretToken = param.SessionToken 152 | } 153 | 154 | protocol := "https" 155 | if config.Base.Protocol != "" { 156 | protocol = config.Base.Protocol 157 | } 158 | if param.Protocol != "" { 159 | protocol = param.Protocol 160 | } 161 | 162 | client = cos.NewClient(CreateURL(bucketIDName, protocol, param.Endpoint, false), &http.Client{ 163 | Transport: &cos.AuthorizationTransport{ 164 | SecretID: secretID, 165 | SecretKey: secretKey, 166 | SessionToken: secretToken, 167 | }, 168 | }) 169 | 170 | // 切换备用域名开关 171 | if config.Base.CloseAutoSwitchHost == "true" { 172 | client.Conf.RetryOpt.AutoSwitchHost = false 173 | } 174 | 175 | // 错误重试 176 | client.Conf.RetryOpt.Count = 10 177 | client.Conf.RetryOpt.Interval = 2 178 | 179 | // 修改 UserAgent 180 | client.UserAgent = Package + "-" + Version 181 | 182 | return client, nil 183 | } 184 | -------------------------------------------------------------------------------- /util/const.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | const ( 4 | Standard = "STANDARD" 5 | StandardIA = "STANDARD_IA" 6 | IntelligentTiering = "INTELLIGENT_TIERING" 7 | Archive = "ARCHIVE" 8 | DeepArchive = "DEEP_ARCHIVE" 9 | MAZStandard = "MAZ_STANDARD" 10 | MAZStandardIA = "MAZ_STANDARD_IA" 11 | MAZIntelligentTiering = "MAZ_INTELLIGENT_TIERING" 12 | MAZArchive = "MAZ_ARCHIVE" 13 | ) 14 | 15 | const ( 16 | CommandCP = "cp" 17 | CommandSync = "sync" 18 | CommandLs = "ls" 19 | CommandRm = "rm" 20 | CommandRestore = "restore" 21 | ) 22 | 23 | const ( 24 | TypeSnapshotPath = "snapshotPath" 25 | TypeFailOutputPath = "failOutputPath" 26 | ) 27 | 28 | const ( 29 | Version string = "v1.0.6" 30 | Package string = "coscli" 31 | SchemePrefix string = "cos://" 32 | CosSeparator string = "/" 33 | IncludePrompt = "--include" 34 | ExcludePrompt = "--exclude" 35 | ChannelSize int = 1000 36 | MaxSyncNumbers = 1000000 37 | MaxDeleteBatchCount int = 1000 38 | SnapshotConnector = "==>" 39 | OfsMaxRenderNum int = 100 40 | ) 41 | 42 | const ( 43 | CpTypeUpload CpType = iota 44 | CpTypeDownload 45 | CpTypeCopy 46 | ) 47 | 48 | const ( 49 | DU_TYPE_TOTAL = 1 50 | DU_TYPE_CATEGORIZATION = 2 51 | ) 52 | 53 | // 版本控制状态 54 | const ( 55 | VersionStatusSuspended = "Suspended" 56 | VersionStatusEnabled = "Enabled" 57 | ) 58 | -------------------------------------------------------------------------------- /util/copy.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/tencentyun/cos-go-sdk-v5" 7 | "math/rand" 8 | "strings" 9 | "time" 10 | ) 11 | 12 | func CosCopy(srcClient, destClient *cos.Client, srcUrl, destUrl StorageUrl, fo *FileOperations) error { 13 | startT := time.Now().UnixNano() / 1000 / 1000 14 | 15 | fo.Monitor.init(fo.CpType) 16 | chProgressSignal = make(chan chProgressSignalType, 10) 17 | go progressBar(fo) 18 | 19 | if srcUrl.(*CosUrl).Object != "" && !strings.HasSuffix(srcUrl.(*CosUrl).Object, CosSeparator) { 20 | // 单对象copy 21 | index := strings.LastIndex(srcUrl.(*CosUrl).Object, "/") 22 | prefix := "" 23 | relativeKey := srcUrl.(*CosUrl).Object 24 | if index > 0 { 25 | prefix = srcUrl.(*CosUrl).Object[:index+1] 26 | relativeKey = srcUrl.(*CosUrl).Object[index+1:] 27 | } 28 | // 获取文件信息 29 | resp, err := GetHead(srcClient, srcUrl.(*CosUrl).Object, fo.Operation.VersionId) 30 | if err != nil { 31 | if resp != nil && resp.StatusCode == 404 { 32 | // 源文件不在cos上 33 | return fmt.Errorf("Object not found : %v", err) 34 | } 35 | return fmt.Errorf("Head object err : %v", err) 36 | } 37 | 38 | // copy文件 39 | skip, err, isDir, size, msg := singleCopy(srcClient, destClient, fo, objectInfoType{prefix, relativeKey, resp.ContentLength, resp.Header.Get("Last-Modified")}, srcUrl, destUrl, fo.Operation.VersionId) 40 | 41 | fo.Monitor.updateMonitor(skip, err, isDir, size) 42 | if err != nil { 43 | return fmt.Errorf("%s failed: %v", msg, err) 44 | } 45 | 46 | } else { 47 | // 多对象copy 48 | batchCopyFiles(srcClient, destClient, srcUrl, destUrl, fo) 49 | } 50 | 51 | CloseErrorOutputFile(fo) 52 | closeProgress() 53 | fmt.Printf(fo.Monitor.progressBar(true, normalExit)) 54 | 55 | endT := time.Now().UnixNano() / 1000 / 1000 56 | PrintTransferStats(startT, endT, fo) 57 | 58 | return nil 59 | } 60 | 61 | func batchCopyFiles(srcClient, destClient *cos.Client, srcUrl, destUrl StorageUrl, fo *FileOperations) { 62 | chObjects := make(chan objectInfoType, ChannelSize) 63 | chError := make(chan error, fo.Operation.Routines) 64 | chListError := make(chan error, 1) 65 | 66 | if fo.BucketType == "OFS" { 67 | // 扫描ofs对象大小及数量 68 | go getOfsObjectList(srcClient, srcUrl, nil, nil, fo, true, false) 69 | // 获取ofs对象列表 70 | go getOfsObjectList(srcClient, srcUrl, chObjects, chListError, fo, false, true) 71 | } else { 72 | // 扫描cos对象大小及数量 73 | go getCosObjectList(srcClient, srcUrl, nil, nil, fo, true, false) 74 | // 获取cos对象列表 75 | go getCosObjectList(srcClient, srcUrl, chObjects, chListError, fo, false, true) 76 | } 77 | 78 | for i := 0; i < fo.Operation.Routines; i++ { 79 | go copyFiles(srcClient, destClient, srcUrl, destUrl, fo, chObjects, chError) 80 | } 81 | 82 | completed := 0 83 | for completed <= fo.Operation.Routines { 84 | select { 85 | case err := <-chListError: 86 | if err != nil { 87 | if fo.Operation.FailOutput { 88 | writeError(err.Error(), fo) 89 | } 90 | } 91 | completed++ 92 | case err := <-chError: 93 | if err == nil { 94 | completed++ 95 | } else { 96 | if fo.Operation.FailOutput { 97 | writeError(err.Error(), fo) 98 | } 99 | } 100 | } 101 | } 102 | } 103 | 104 | func copyFiles(srcClient, destClient *cos.Client, srcUrl, destUrl StorageUrl, fo *FileOperations, chObjects <-chan objectInfoType, chError chan<- error) { 105 | for object := range chObjects { 106 | var skip, isDir bool 107 | var err error 108 | var size int64 109 | var msg string 110 | for retry := 0; retry <= fo.Operation.ErrRetryNum; retry++ { 111 | skip, err, isDir, size, msg = singleCopy(srcClient, destClient, fo, object, srcUrl, destUrl) 112 | if err == nil { 113 | break // Copy succeeded, break the loop 114 | } else { 115 | if retry < fo.Operation.ErrRetryNum { 116 | if fo.Operation.ErrRetryInterval == 0 { 117 | // If the retry interval is not specified, retry after a random interval of 1~10 seconds. 118 | time.Sleep(time.Duration(rand.Intn(10)+1) * time.Second) 119 | } else { 120 | time.Sleep(time.Duration(fo.Operation.ErrRetryInterval) * time.Second) 121 | } 122 | } 123 | } 124 | } 125 | 126 | fo.Monitor.updateMonitor(skip, err, isDir, size) 127 | if err != nil { 128 | chError <- fmt.Errorf("%s failed: %w", msg, err) 129 | continue 130 | } 131 | } 132 | 133 | chError <- nil 134 | } 135 | 136 | func singleCopy(srcClient, destClient *cos.Client, fo *FileOperations, objectInfo objectInfoType, srcUrl, destUrl StorageUrl, VersionId ...string) (skip bool, rErr error, isDir bool, size int64, msg string) { 137 | skip = false 138 | rErr = nil 139 | isDir = false 140 | size = objectInfo.size 141 | object := objectInfo.prefix + objectInfo.relativeKey 142 | 143 | destPath := copyPathFixed(objectInfo.relativeKey, destUrl.(*CosUrl).Object) 144 | msg = fmt.Sprintf("\nCopy %s to %s", getCosUrl(srcUrl.(*CosUrl).Bucket, object), getCosUrl(destUrl.(*CosUrl).Bucket, destPath)) 145 | 146 | var err error 147 | // 是文件夹则直接创建并退出 148 | if size == 0 && strings.HasSuffix(object, "/") { 149 | isDir = true 150 | } 151 | 152 | // 仅sync命令执行skip 153 | if fo.Command == CommandSync && !isDir { 154 | skip, err = skipCopy(srcClient, destClient, object, destPath) 155 | if err != nil { 156 | rErr = err 157 | return 158 | } 159 | } 160 | 161 | if skip { 162 | return 163 | } 164 | 165 | // copy暂不支持监听进度 166 | // size = 0 167 | 168 | url, err := GenURL(fo.Config, fo.Param, srcUrl.(*CosUrl).Bucket) 169 | 170 | srcURL := fmt.Sprintf("%s/%s", url.BucketURL.Host, object) 171 | 172 | opt := &cos.MultiCopyOptions{ 173 | OptCopy: &cos.ObjectCopyOptions{ 174 | &cos.ObjectCopyHeaderOptions{ 175 | CacheControl: fo.Operation.Meta.CacheControl, 176 | ContentDisposition: fo.Operation.Meta.ContentDisposition, 177 | ContentEncoding: fo.Operation.Meta.ContentEncoding, 178 | ContentType: fo.Operation.Meta.ContentType, 179 | Expires: fo.Operation.Meta.Expires, 180 | XCosStorageClass: fo.Operation.StorageClass, 181 | XCosMetaXXX: fo.Operation.Meta.XCosMetaXXX, 182 | }, 183 | nil, 184 | }, 185 | PartSize: fo.Operation.PartSize, 186 | ThreadPoolSize: fo.Operation.ThreadNum, 187 | } 188 | if fo.Operation.Meta.CacheControl != "" || fo.Operation.Meta.ContentDisposition != "" || fo.Operation.Meta.ContentEncoding != "" || 189 | fo.Operation.Meta.ContentType != "" || fo.Operation.Meta.Expires != "" || fo.Operation.Meta.MetaChange { 190 | } 191 | { 192 | opt.OptCopy.ObjectCopyHeaderOptions.XCosMetadataDirective = "Replaced" 193 | } 194 | 195 | _, _, err = destClient.Object.MultiCopy(context.Background(), destPath, srcURL, opt, VersionId...) 196 | 197 | if err != nil { 198 | rErr = err 199 | return 200 | } 201 | 202 | if fo.Operation.Move { 203 | if err == nil { 204 | _, err = srcClient.Object.Delete(context.Background(), object, nil) 205 | rErr = err 206 | return 207 | } 208 | } 209 | 210 | return 211 | } 212 | -------------------------------------------------------------------------------- /util/error_output.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | logger "github.com/sirupsen/logrus" 5 | "os" 6 | "path/filepath" 7 | "sync" 8 | "time" 9 | ) 10 | 11 | const ( 12 | ErrTypeUpload string = "upload" 13 | ErrTypeDownload string = "download" 14 | ErrTypeList string = "list" 15 | ) 16 | 17 | // 开启错误输出 18 | var ( 19 | outputMu sync.Mutex 20 | ) 21 | 22 | func writeError(errString string, fo *FileOperations) { 23 | var err error 24 | if fo.ErrOutput.Path == "" { 25 | fo.ErrOutput.Path = filepath.Join(fo.Operation.FailOutputPath, time.Now().Format("20060102_150405")) 26 | _, err := os.Stat(fo.ErrOutput.Path) 27 | if os.IsNotExist(err) { 28 | err := os.MkdirAll(fo.ErrOutput.Path, 0755) 29 | if err != nil { 30 | logger.Errorf("Failed to create error output dir: %v", err) 31 | return 32 | } 33 | } 34 | } 35 | 36 | if fo.ErrOutput.outputFile == nil { 37 | // 创建错误日志文件 38 | failOutputFilePath := filepath.Join(fo.ErrOutput.Path, "error.report") 39 | fo.ErrOutput.outputFile, err = os.OpenFile(failOutputFilePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) 40 | if err != nil { 41 | logger.Errorf("Failed to create error error output file:%v", err) 42 | return 43 | } 44 | } 45 | 46 | outputMu.Lock() 47 | _, writeErr := fo.ErrOutput.outputFile.WriteString(errString) 48 | 49 | if writeErr != nil { 50 | logger.Errorf("Failed to write error output file : %v\n", writeErr) 51 | } 52 | outputMu.Unlock() 53 | } 54 | 55 | func CloseErrorOutputFile(fo *FileOperations) { 56 | if fo.ErrOutput.outputFile != nil { 57 | defer fo.ErrOutput.outputFile.Close() 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /util/filter.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "regexp" 5 | "strings" 6 | ) 7 | 8 | func matchPatterns(filename string, filters []FilterOptionType) bool { 9 | if len(filters) == 0 { 10 | return true 11 | } 12 | 13 | files := []string{filename} 14 | vsf := matchFiltersForStrs(files, filters) 15 | 16 | if len(vsf) > 0 { 17 | return true 18 | } 19 | return false 20 | } 21 | 22 | func matchFiltersForStrs(strs []string, filters []FilterOptionType) []string { 23 | if len(filters) == 0 { 24 | return strs 25 | } 26 | 27 | vsf := make([]string, 0) 28 | 29 | for _, str := range strs { 30 | if matchFiltersForStr(str, filters) { 31 | vsf = append(vsf, str) 32 | } 33 | } 34 | 35 | return vsf 36 | } 37 | 38 | func matchFiltersForStr(str string, filters []FilterOptionType) bool { 39 | if len(filters) == 0 { 40 | return true 41 | } 42 | 43 | var res bool 44 | if filters[0].name == IncludePrompt { 45 | res = filterSingleStr(str, filters[0].pattern, true) 46 | } else { 47 | res = filterSingleStr(str, filters[0].pattern, false) 48 | } 49 | for _, filter := range filters[1:] { 50 | if filter.name == IncludePrompt { 51 | res = res || filterSingleStr(str, filter.pattern, true) 52 | } else { 53 | res = res && filterSingleStr(str, filter.pattern, false) 54 | } 55 | } 56 | 57 | return res 58 | } 59 | 60 | func filterSingleStr(v, p string, include bool) bool { 61 | //_, name := filepath.Split(v) 62 | res, _ := regexp.MatchString(p, v) 63 | 64 | if include { 65 | return res 66 | } else { 67 | return !res 68 | } 69 | } 70 | 71 | func GetFilter(include, exclude string) (bool, []FilterOptionType) { 72 | filters := make([]FilterOptionType, 0) 73 | 74 | if include != "" { 75 | ok, filter := createFilter(IncludePrompt, include) 76 | if !ok { 77 | return false, filters 78 | } 79 | filters = append(filters, filter) 80 | } 81 | 82 | if exclude != "" { 83 | ok, filter := createFilter(ExcludePrompt, exclude) 84 | if !ok { 85 | return false, filters 86 | } 87 | filters = append(filters, filter) 88 | } 89 | 90 | return true, filters 91 | } 92 | 93 | func createFilter(name, pattern string) (bool, FilterOptionType) { 94 | var filter FilterOptionType 95 | filter.name = name 96 | filter.pattern = strings.Replace(pattern, "[!", "[^", -1) 97 | //dir, _ := filepath.Split(filter.pattern) 98 | //if dir != "" { 99 | // return false, filter 100 | //} 101 | return true, filter 102 | } 103 | 104 | func cosObjectMatchPatterns(object string, filters []FilterOptionType) bool { 105 | if len(filters) == 0 { 106 | return true 107 | } 108 | 109 | return matchPatterns(object, filters) 110 | } 111 | -------------------------------------------------------------------------------- /util/get_ofs_object.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "github.com/tencentyun/cos-go-sdk-v5" 5 | "net/url" 6 | "strings" 7 | ) 8 | 9 | func getOfsObjectList(c *cos.Client, cosUrl StorageUrl, chObjects chan<- objectInfoType, chError chan<- error, fo *FileOperations, scanSizeNum bool, withFinishSignal bool) { 10 | if chObjects != nil { 11 | defer close(chObjects) 12 | } 13 | 14 | prefix := cosUrl.(*CosUrl).Object 15 | marker := "" 16 | limit := 0 17 | 18 | delimiter := "" 19 | if fo.Operation.OnlyCurrentDir { 20 | delimiter = "/" 21 | } 22 | 23 | err := getOfsObjectListRecursion(c, cosUrl, chObjects, chError, fo, scanSizeNum, prefix, marker, limit, delimiter) 24 | 25 | if err != nil && scanSizeNum { 26 | fo.Monitor.setScanError(err) 27 | } 28 | 29 | if scanSizeNum { 30 | fo.Monitor.setScanEnd() 31 | freshProgress() 32 | } 33 | 34 | if withFinishSignal { 35 | chError <- err 36 | } 37 | } 38 | 39 | func getOfsObjectListRecursion(c *cos.Client, cosUrl StorageUrl, chObjects chan<- objectInfoType, chError chan<- error, fo *FileOperations, scanSizeNum bool, prefix string, marker string, limit int, delimiter string) error { 40 | isTruncated := true 41 | for isTruncated { 42 | // 实例化请求参数 43 | opt := &cos.BucketGetOptions{ 44 | Prefix: prefix, 45 | Delimiter: delimiter, 46 | EncodingType: "url", 47 | Marker: marker, 48 | MaxKeys: limit, 49 | } 50 | res, err := tryGetObjects(c, opt) 51 | if err != nil { 52 | return err 53 | } 54 | 55 | for _, object := range res.Contents { 56 | object.Key, _ = url.QueryUnescape(object.Key) 57 | if cosObjectMatchPatterns(object.Key, fo.Operation.Filters) { 58 | if scanSizeNum { 59 | fo.Monitor.updateScanSizeNum(object.Size, 1) 60 | } else { 61 | objPrefix := "" 62 | objKey := object.Key 63 | index := strings.LastIndex(cosUrl.(*CosUrl).Object, "/") 64 | if index > 0 { 65 | objPrefix = object.Key[:index+1] 66 | objKey = object.Key[index+1:] 67 | } 68 | chObjects <- objectInfoType{objPrefix, objKey, int64(object.Size), object.LastModified} 69 | } 70 | } 71 | } 72 | 73 | if len(res.CommonPrefixes) > 0 { 74 | for _, commonPrefix := range res.CommonPrefixes { 75 | commonPrefix, _ = url.QueryUnescape(commonPrefix) 76 | 77 | if cosObjectMatchPatterns(commonPrefix, fo.Operation.Filters) { 78 | if scanSizeNum { 79 | fo.Monitor.updateScanSizeNum(0, 1) 80 | } else { 81 | objPrefix := "" 82 | objKey := commonPrefix 83 | index := strings.LastIndex(cosUrl.(*CosUrl).Object, "/") 84 | if index > 0 { 85 | objPrefix = commonPrefix[:index+1] 86 | objKey = commonPrefix[index+1:] 87 | } 88 | chObjects <- objectInfoType{objPrefix, objKey, int64(0), ""} 89 | } 90 | } 91 | 92 | if delimiter == "" { 93 | err = getOfsObjectListRecursion(c, cosUrl, chObjects, chError, fo, scanSizeNum, commonPrefix, marker, limit, delimiter) 94 | if err != nil { 95 | return err 96 | } 97 | } 98 | } 99 | } 100 | 101 | isTruncated = res.IsTruncated 102 | marker, _ = url.QueryUnescape(res.NextMarker) 103 | } 104 | return nil 105 | } 106 | 107 | func getOfsObjectListForLs(c *cos.Client, prefix string, marker string, limit int, recursive bool) (err error, objects []cos.Object, commonPrefixes []string, isTruncated bool, nextMarker string) { 108 | 109 | delimiter := "" 110 | if !recursive { 111 | delimiter = "/" 112 | } 113 | 114 | // 实例化请求参数 115 | opt := &cos.BucketGetOptions{ 116 | Prefix: prefix, 117 | Delimiter: delimiter, 118 | EncodingType: "url", 119 | Marker: marker, 120 | MaxKeys: limit, 121 | } 122 | res, err := tryGetObjects(c, opt) 123 | if err != nil { 124 | return 125 | } 126 | 127 | objects = res.Contents 128 | commonPrefixes = res.CommonPrefixes 129 | isTruncated = res.IsTruncated 130 | nextMarker, _ = url.QueryUnescape(res.NextMarker) 131 | return 132 | } 133 | 134 | func GetOfsKeys(c *cos.Client, cosUrl StorageUrl, keys map[string]string, fo *FileOperations) error { 135 | 136 | chFiles := make(chan objectInfoType, ChannelSize) 137 | chFinish := make(chan error, 2) 138 | go ReadCosKeys(keys, cosUrl, chFiles, chFinish) 139 | go getOfsObjectList(c, cosUrl, chFiles, chFinish, fo, false, false) 140 | select { 141 | case err := <-chFinish: 142 | if err != nil { 143 | return err 144 | } 145 | } 146 | 147 | return nil 148 | } 149 | -------------------------------------------------------------------------------- /util/hash.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "context" 5 | "crypto/md5" 6 | "encoding/base64" 7 | "encoding/hex" 8 | "fmt" 9 | "hash" 10 | "hash/crc64" 11 | "io" 12 | "os" 13 | 14 | "github.com/tencentyun/cos-go-sdk-v5" 15 | ) 16 | 17 | func ShowHash(c *cos.Client, path string, hashType string) (h string, b string, resp *cos.Response, err error) { 18 | opt := &cos.ObjectHeadOptions{ 19 | IfModifiedSince: "", 20 | XCosSSECustomerAglo: "", 21 | XCosSSECustomerKey: "", 22 | XCosSSECustomerKeyMD5: "", 23 | XOptionHeader: nil, 24 | } 25 | 26 | resp, err = c.Object.Head(context.Background(), path, opt) 27 | if err != nil { 28 | return "", "", nil, err 29 | } 30 | 31 | switch hashType { 32 | case "crc64": 33 | h = resp.Header.Get("x-cos-hash-crc64ecma") 34 | case "md5": 35 | m := resp.Header.Get("etag") 36 | h = m[1 : len(m)-1] 37 | 38 | encode, _ := hex.DecodeString(h) 39 | b = base64.StdEncoding.EncodeToString(encode) 40 | default: 41 | return "", "", nil, fmt.Errorf("--type can only be selected between MD5 and CRC64") 42 | } 43 | return h, b, resp, nil 44 | } 45 | 46 | func CalculateHash(path string, hashType string) (h string, b string, err error) { 47 | f, err := os.Open(path) 48 | if err != nil { 49 | return "", "", err 50 | } 51 | defer f.Close() 52 | _, _ = f.Seek(0, 0) 53 | 54 | switch hashType { 55 | case "crc64": 56 | ecma := crc64.New(crc64.MakeTable(crc64.ECMA)) 57 | w, _ := ecma.(hash.Hash) 58 | if _, err := io.Copy(w, f); err != nil { 59 | return "", "", err 60 | } 61 | 62 | res := ecma.Sum64() 63 | h = fmt.Sprintf("%d", res) 64 | case "md5": 65 | m := md5.New() 66 | w, _ := m.(hash.Hash) 67 | if _, err := io.Copy(w, f); err != nil { 68 | return "", "", err 69 | } 70 | 71 | res := m.Sum(nil) 72 | h = fmt.Sprintf("%x", res) 73 | b = base64.StdEncoding.EncodeToString(res) 74 | default: 75 | return "", "", fmt.Errorf("Wrong args!") 76 | } 77 | return h, b, nil 78 | } 79 | 80 | func GetHead(c *cos.Client, cosPath string, id ...string) (*cos.Response, error) { 81 | headOpt := &cos.ObjectHeadOptions{ 82 | IfModifiedSince: "", 83 | XCosSSECustomerAglo: "", 84 | XCosSSECustomerKey: "", 85 | XCosSSECustomerKeyMD5: "", 86 | XOptionHeader: nil, 87 | } 88 | return c.Object.Head(context.Background(), cosPath, headOpt, id...) 89 | } 90 | -------------------------------------------------------------------------------- /util/list_iterator.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "context" 5 | "github.com/tencentyun/cos-go-sdk-v5" 6 | "net/url" 7 | ) 8 | 9 | func GetObjectsListIterator(c *cos.Client, prefix, marker string, include, exclude string) (objects []cos.Object, 10 | isTruncated bool, nextMarker string, commonPrefixes []string, err error) { 11 | opt := &cos.BucketGetOptions{ 12 | Prefix: prefix, 13 | Delimiter: "", 14 | EncodingType: "", 15 | Marker: marker, 16 | MaxKeys: 0, 17 | } 18 | 19 | res, _, err := c.Bucket.Get(context.Background(), opt) 20 | if err != nil { 21 | return objects, isTruncated, nextMarker, commonPrefixes, err 22 | } 23 | 24 | objects = append(objects, res.Contents...) 25 | commonPrefixes = res.CommonPrefixes 26 | 27 | isTruncated = res.IsTruncated 28 | nextMarker, _ = url.QueryUnescape(res.NextMarker) 29 | 30 | if len(include) > 0 { 31 | objects = MatchCosPattern(objects, include, true) 32 | } 33 | if len(exclude) > 0 { 34 | objects = MatchCosPattern(objects, exclude, false) 35 | } 36 | 37 | return objects, isTruncated, nextMarker, commonPrefixes, err 38 | } 39 | -------------------------------------------------------------------------------- /util/listener.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "github.com/tencentyun/cos-go-sdk-v5" 6 | ) 7 | 8 | type CosListener struct { 9 | fo *FileOperations 10 | counter *Counter 11 | } 12 | 13 | func (l *CosListener) ProgressChangedCallback(event *cos.ProgressEvent) { 14 | switch event.EventType { 15 | case cos.ProgressStartedEvent: 16 | case cos.ProgressDataEvent: 17 | l.fo.Monitor.updateTransferSize(event.RWBytes) 18 | l.fo.Monitor.updateDealSize(event.RWBytes) 19 | l.counter.TransferSize += event.RWBytes 20 | case cos.ProgressCompletedEvent: 21 | case cos.ProgressFailedEvent: 22 | l.fo.Monitor.updateDealSize(-event.ConsumedBytes) 23 | default: 24 | fmt.Printf("Progress Changed Error: unknown progress event type\n") 25 | } 26 | freshProgress() 27 | } 28 | -------------------------------------------------------------------------------- /util/meta.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "strconv" 7 | "strings" 8 | "time" 9 | ) 10 | 11 | func MetaStringToHeader(meta string) (result Meta, err error) { 12 | if meta == "" { 13 | return 14 | } 15 | meta = strings.TrimSpace(meta) 16 | kvs := strings.Split(meta, "#") 17 | header := http.Header{} 18 | metaXXX := &http.Header{} 19 | var metaChange bool 20 | for _, kv := range kvs { 21 | if kv == "" { 22 | continue 23 | } 24 | item := strings.Split(kv, ":") 25 | if len(item) < 2 { 26 | return result, fmt.Errorf("invalid meta item %v", item) 27 | } 28 | 29 | k := strings.ToLower(item[0]) 30 | v := strings.Join(item[1:], ":") 31 | if strings.HasPrefix(k, "x-cos-meta-") { 32 | metaXXX.Set(k, v) 33 | metaChange = true 34 | } else { 35 | header.Set(k, v) 36 | } 37 | } 38 | 39 | expires := header.Get("Expires") 40 | if expires != "" { 41 | extime, err := time.Parse(time.RFC3339, expires) 42 | if err != nil { 43 | return result, fmt.Errorf("invalid meta expires format, %v", err) 44 | } 45 | 46 | expires = extime.Format(time.RFC1123) 47 | } 48 | result = Meta{ 49 | CacheControl: header.Get("Cache-Control"), 50 | ContentDisposition: header.Get("Content-Disposition"), 51 | ContentEncoding: header.Get("Content-Encoding"), 52 | ContentType: header.Get("Content-Type"), 53 | ContentMD5: header.Get("Content-MD5"), 54 | ContentLength: 0, 55 | ContentLanguage: header.Get("Content-Language"), 56 | Expires: expires, 57 | XCosMetaXXX: metaXXX, 58 | MetaChange: metaChange, 59 | } 60 | 61 | cl := header.Get("Content-Length") 62 | if cl != "" { 63 | var clInt int64 64 | clInt, err = strconv.ParseInt(cl, 10, 64) 65 | if err != nil { 66 | return result, fmt.Errorf("parse meta ContentLength invalid, %v", err) 67 | } 68 | 69 | result.ContentLength = clInt 70 | } 71 | 72 | return result, nil 73 | } 74 | -------------------------------------------------------------------------------- /util/path.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "github.com/mitchellh/go-homedir" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | ) 10 | 11 | func IsCosPath(path string) bool { 12 | if len(path) <= 6 { 13 | return false 14 | } 15 | if path[:6] == "cos://" { 16 | return true 17 | } else { 18 | return false 19 | } 20 | } 21 | 22 | func ParsePath(url string) (bucketName string, path string) { 23 | if IsCosPath(url) { 24 | res := strings.SplitN(url[6:], "/", 2) 25 | if len(res) < 2 { 26 | return res[0], "" 27 | } else { 28 | return res[0], res[1] 29 | } 30 | } else { 31 | if url[0] == '~' { 32 | home, _ := homedir.Dir() 33 | path = home + url[1:] 34 | } else { 35 | path = url 36 | } 37 | return "", path 38 | } 39 | } 40 | 41 | func UploadPathFixed(file fileInfoType, cosPath string) (string, string) { 42 | // cos路径不全则补充文件名 43 | if cosPath == "" || strings.HasSuffix(cosPath, "/") { 44 | filePath := file.filePath 45 | filePath = strings.Replace(file.filePath, string(os.PathSeparator), "/", -1) 46 | filePath = strings.Replace(file.filePath, "\\", "/", -1) 47 | cosPath += filePath 48 | } 49 | 50 | localFilePath := filepath.Join(file.dir, file.filePath) 51 | 52 | return localFilePath, cosPath 53 | } 54 | 55 | func DownloadPathFixed(relativeObject, filePath string) string { 56 | if strings.HasSuffix(filePath, "/") || strings.HasSuffix(filePath, "\\") { 57 | return filePath + relativeObject 58 | } 59 | // 兼容windows路径 60 | filePath = strings.Replace(filePath, "/", string(os.PathSeparator), -1) 61 | return filePath 62 | } 63 | 64 | func copyPathFixed(relativeObject, destPath string) string { 65 | if destPath == "" || strings.HasSuffix(destPath, "/") { 66 | return destPath + relativeObject 67 | } 68 | 69 | return destPath 70 | } 71 | 72 | func getAbsPath(strPath string) (string, error) { 73 | if filepath.IsAbs(strPath) { 74 | return strPath, nil 75 | } 76 | 77 | currentDir, err := os.Getwd() 78 | if err != nil { 79 | return "", err 80 | } 81 | 82 | if !strings.HasSuffix(strPath, string(os.PathSeparator)) { 83 | strPath += string(os.PathSeparator) 84 | } 85 | 86 | strPath = currentDir + string(os.PathSeparator) + strPath 87 | absPath, err := filepath.Abs(strPath) 88 | if err != nil { 89 | return "", err 90 | } 91 | 92 | if !strings.HasSuffix(absPath, string(os.PathSeparator)) { 93 | absPath += string(os.PathSeparator) 94 | } 95 | return absPath, err 96 | } 97 | 98 | // 检查路径是否是本地文件路径的子路径 99 | func CheckPath(fileUrl StorageUrl, fo *FileOperations, pathType string) error { 100 | absFileDir, err := getAbsPath(fileUrl.ToString()) 101 | if err != nil { 102 | return err 103 | } 104 | 105 | var path string 106 | if pathType == TypeSnapshotPath { 107 | path = fo.Operation.SnapshotPath 108 | } else if pathType == TypeFailOutputPath { 109 | if fo.Operation.FailOutput { 110 | path = fo.Operation.FailOutputPath 111 | } else { 112 | return nil 113 | } 114 | 115 | } else { 116 | return fmt.Errorf("check path failed , invalid pathType %s", pathType) 117 | } 118 | 119 | absPath, err := getAbsPath(path) 120 | if err != nil { 121 | return err 122 | } 123 | 124 | if strings.Index(absPath, absFileDir) >= 0 { 125 | return fmt.Errorf("%s %s is subdirectory of %s", pathType, fo.Operation.SnapshotPath, fileUrl.ToString()) 126 | } 127 | return nil 128 | } 129 | 130 | func createParentDirectory(localFilePath string) error { 131 | dir, err := filepath.Abs(filepath.Dir(localFilePath)) 132 | if err != nil { 133 | return err 134 | } 135 | dir = strings.Replace(dir, "\\", "/", -1) 136 | return os.MkdirAll(dir, 0755) 137 | } 138 | -------------------------------------------------------------------------------- /util/print.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | ) 7 | 8 | func PrintTransferStats(startT, endT int64, fo *FileOperations) { 9 | if fo.Monitor.ErrNum > 0 && fo.Operation.FailOutput { 10 | absErrOutputPath, _ := filepath.Abs(fo.ErrOutput.Path) 11 | fmt.Printf("Some file upload failed, please check the detailed information in dir %s.\n", absErrOutputPath) 12 | } 13 | 14 | // 计算上传速度 15 | if endT-startT > 0 { 16 | averSpeed := (float64(fo.Monitor.TransferSize) / float64(endT-startT)) * 1000 17 | formattedSpeed := formatBytes(averSpeed) 18 | fmt.Printf("\nAvgSpeed: %s/s\n", formattedSpeed) 19 | } 20 | } 21 | 22 | func PrintCostTime(startT, endT int64) { 23 | // 计算并输出花费时间 24 | elapsedTime := float64(endT-startT) / 1000 25 | fmt.Printf("\ncost %.6f(s)\n", elapsedTime) 26 | } 27 | -------------------------------------------------------------------------------- /util/processbar.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | // 文件上传进度 8 | func drawBar(percent int) (bar string) { 9 | bar1 := "##############################" 10 | bar2 := "------------------------------" 11 | total := 30 12 | doneNum := total * percent / 100 13 | remainNum := total - doneNum 14 | bar = bar1[:doneNum] + bar2[:remainNum] 15 | return bar 16 | } 17 | 18 | func freshProgress() { 19 | if len(chProgressSignal) <= signalNum { 20 | chProgressSignal <- chProgressSignalType{false, normalExit} 21 | } 22 | } 23 | 24 | func progressBar(fo *FileOperations) { 25 | for signal := range chProgressSignal { 26 | fmt.Printf(fo.Monitor.progressBar(signal.finish, signal.exitStat)) 27 | } 28 | } 29 | 30 | func closeProgress() { 31 | signalNum = -1 32 | } 33 | -------------------------------------------------------------------------------- /util/rename.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "encoding/xml" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "io/ioutil" 11 | "net/http" 12 | "reflect" 13 | "strconv" 14 | "strings" 15 | 16 | "github.com/mozillazg/go-httpheader" 17 | cosgo "github.com/tencentyun/cos-go-sdk-v5" 18 | ) 19 | 20 | /** 21 | 1. 首先源地址 /ofs 22 | 2. 目标地址 https://tina-coscli-test-1253960454.cos.ap-chengdu.myqcloud.com/x?rename 23 | 3. header头部 24 | */ 25 | type ObjectMoveOptions struct { 26 | *cosgo.BucketHeadOptions 27 | *cosgo.ACLHeaderOptions 28 | XCosRenameSource string `header:"x-cos-rename-source" url:"-" xml:"-"` 29 | } 30 | type jsonError struct { 31 | Code int `json:"code,omitempty"` 32 | Message string `json:"message,omitempty"` 33 | RequestID string `json:"request_id,omitempty"` 34 | } 35 | 36 | func PutRename(ctx context.Context, config *Config, param *Param, c *cosgo.Client, name, dstURL string, 37 | closeBody bool) (resp *http.Response, err error) { 38 | var cancel context.CancelFunc 39 | if closeBody { 40 | ctx, cancel = context.WithCancel(ctx) 41 | defer cancel() 42 | } 43 | durl := strings.SplitN(dstURL, "/", 2) 44 | if len(durl) < 2 { 45 | return nil, errors.New(fmt.Sprintf("x-cos-rename-source format error: #{dstURL}")) 46 | } 47 | var u string 48 | u = fmt.Sprintf("%s/%s?rename", c.BaseURL.BucketURL, durl[1]) 49 | 50 | req, err := http.NewRequest("PUT", u, nil) 51 | if err != nil { 52 | return 53 | } 54 | copyOpt := &ObjectMoveOptions{ 55 | &cosgo.BucketHeadOptions{}, 56 | &cosgo.ACLHeaderOptions{}, 57 | "/" + name, 58 | } 59 | 60 | //header 61 | req.Header, err = addHeaderOptions(req.Header, copyOpt) 62 | if err != nil { 63 | return 64 | } 65 | if v := req.Header.Get("Content-Length"); req.ContentLength == 0 && v != "" && v != "0" { 66 | req.ContentLength, _ = strconv.ParseInt(v, 10, 64) 67 | } 68 | if c.Host != "" { 69 | req.Host = c.Host 70 | } 71 | if c.Conf.RequestBodyClose { 72 | req.Close = true 73 | } 74 | secretID := config.Base.SecretID 75 | secretKey := config.Base.SecretKey 76 | secretToken := config.Base.SessionToken 77 | if param.SecretID != "" { 78 | secretID = param.SecretID 79 | } 80 | if param.SecretKey != "" { 81 | secretKey = param.SecretKey 82 | } 83 | if param.SessionToken != "" { 84 | secretToken = param.SessionToken 85 | } 86 | client := &http.Client{ 87 | Transport: &cosgo.AuthorizationTransport{ 88 | SecretID: secretID, 89 | SecretKey: secretKey, 90 | SessionToken: secretToken, 91 | }, 92 | } 93 | req = req.WithContext(ctx) 94 | resp, err = client.Do(req) 95 | if err != nil { 96 | // If we got an error, and the context has been canceled, 97 | // the context's error is probably more useful. 98 | select { 99 | case <-ctx.Done(): 100 | return nil, ctx.Err() 101 | default: 102 | } 103 | return nil, err 104 | } 105 | defer func() { 106 | if closeBody { 107 | // Close the body to let the Transport reuse the connection 108 | io.Copy(ioutil.Discard, resp.Body) 109 | resp.Body.Close() 110 | } 111 | }() 112 | 113 | err = checkResponse(resp) 114 | if err != nil { 115 | // StatusCode != 2xx when Get Object 116 | if !closeBody { 117 | resp.Body.Close() 118 | } 119 | // even though there was an error, we still return the response 120 | // in case the caller wants to inspect it further 121 | return resp, err 122 | } 123 | 124 | return resp, err 125 | } 126 | 127 | // addHeaderOptions adds the parameters in opt as Header fields to req 128 | func addHeaderOptions(header http.Header, opt interface{}) (http.Header, error) { 129 | v := reflect.ValueOf(opt) 130 | if v.Kind() == reflect.Ptr && v.IsNil() { 131 | return header, nil 132 | } 133 | 134 | h, err := httpheader.Header(opt) 135 | if err != nil { 136 | return nil, err 137 | } 138 | 139 | for key, values := range h { 140 | for _, value := range values { 141 | header.Add(key, value) 142 | } 143 | } 144 | return header, nil 145 | } 146 | 147 | func checkResponse(r *http.Response) error { 148 | if c := r.StatusCode; 200 <= c && c <= 299 { 149 | return nil 150 | } 151 | errorResponse := &cosgo.ErrorResponse{Response: r} 152 | data, err := ioutil.ReadAll(r.Body) 153 | if err == nil && data != nil { 154 | xml.Unmarshal(data, errorResponse) 155 | } 156 | // 是否为 json 格式 157 | if errorResponse.Code == "" { 158 | ctype := strings.TrimLeft(r.Header.Get("Content-Type"), " ") 159 | if strings.HasPrefix(ctype, "application/json") { 160 | var jerror jsonError 161 | json.Unmarshal(data, &jerror) 162 | errorResponse.Code = strconv.Itoa(jerror.Code) 163 | errorResponse.Message = jerror.Message 164 | errorResponse.RequestID = jerror.RequestID 165 | } 166 | 167 | } 168 | return errorResponse 169 | } 170 | -------------------------------------------------------------------------------- /util/restore_objects.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "context" 5 | "encoding/xml" 6 | "fmt" 7 | logger "github.com/sirupsen/logrus" 8 | "github.com/tencentyun/cos-go-sdk-v5" 9 | "math/rand" 10 | "net/url" 11 | "path/filepath" 12 | "time" 13 | ) 14 | 15 | var succeedNum, failedNum, errTypeNum int 16 | 17 | func RestoreObjects(c *cos.Client, cosUrl StorageUrl, fo *FileOperations) error { 18 | // 根据s.Header判断是否是融合桶或者普通桶 19 | s, err := c.Bucket.Head(context.Background()) 20 | if err != nil { 21 | return err 22 | } 23 | logger.Infof("Start Restore %s", cosUrl.(*CosUrl).Bucket+cosUrl.(*CosUrl).Object) 24 | if s.Header.Get("X-Cos-Bucket-Arch") == "OFS" { 25 | bucketName := cosUrl.(*CosUrl).Bucket 26 | prefix := cosUrl.(*CosUrl).Object 27 | err = restoreOfsObjects(c, bucketName, prefix, fo, "") 28 | } else { 29 | err = restoreCosObjects(c, cosUrl, fo) 30 | } 31 | 32 | absErrOutputPath, _ := filepath.Abs(fo.ErrOutput.Path) 33 | 34 | if failedNum > 0 { 35 | logger.Warningf("Restore %s completed, total num: %d,success num: %d,restore error num: %d,error type num: %d,Some objects restore failed, please check the detailed information in dir %s.\n", cosUrl.(*CosUrl).Bucket+cosUrl.(*CosUrl).Object, succeedNum+failedNum+errTypeNum, succeedNum, failedNum, errTypeNum, absErrOutputPath) 36 | } else { 37 | logger.Infof("Restore %s completed,total num: %d,success num: %d,restore error num: %d,error type num: %d", cosUrl.(*CosUrl).Bucket+cosUrl.(*CosUrl).Object, succeedNum+failedNum+errTypeNum, succeedNum, failedNum, errTypeNum) 38 | } 39 | 40 | return nil 41 | } 42 | 43 | func restoreCosObjects(c *cos.Client, cosUrl StorageUrl, fo *FileOperations) error { 44 | var err error 45 | var objects []cos.Object 46 | marker := "" 47 | isTruncated := true 48 | 49 | for isTruncated { 50 | err, objects, _, isTruncated, marker = getCosObjectListForLs(c, cosUrl, marker, 0, true) 51 | if err != nil { 52 | return fmt.Errorf("list objects error : %v", err) 53 | } 54 | 55 | for _, object := range objects { 56 | if object.StorageClass == Archive || object.StorageClass == MAZArchive || object.StorageClass == DeepArchive { 57 | object.Key, _ = url.QueryUnescape(object.Key) 58 | if cosObjectMatchPatterns(object.Key, fo.Operation.Filters) { 59 | if object.RestoreStatus == "ONGOING" || object.RestoreStatus == "ONGING" { 60 | succeedNum += 1 61 | } else { 62 | resp, err := TryRestoreObject(c, cosUrl.(*CosUrl).Bucket, object.Key, fo.Operation.Days, fo.Operation.RestoreMode) 63 | if err != nil { 64 | if resp != nil && resp.StatusCode == 409 { 65 | succeedNum += 1 66 | } else { 67 | failedNum += 1 68 | writeError(fmt.Sprintf("restore %s failed , errMsg:%v\n", object.Key, err), fo) 69 | } 70 | } else { 71 | succeedNum += 1 72 | } 73 | } 74 | 75 | } 76 | } else { 77 | errTypeNum += 1 78 | } 79 | 80 | } 81 | } 82 | 83 | return nil 84 | } 85 | 86 | func TryRestoreObject(c *cos.Client, bucketName, objectKey string, days int, mode string) (resp *cos.Response, err error) { 87 | 88 | logger.Infof("Restore cos://%s/%s\n", bucketName, objectKey) 89 | opt := &cos.ObjectRestoreOptions{ 90 | XMLName: xml.Name{}, 91 | Days: days, 92 | Tier: &cos.CASJobParameters{Tier: mode}, 93 | XOptionHeader: nil, 94 | } 95 | 96 | for i := 0; i <= 10; i++ { 97 | resp, err = c.Object.PostRestore(context.Background(), objectKey, opt) 98 | if err != nil { 99 | if resp != nil && resp.StatusCode == 503 { 100 | if i == 10 { 101 | return resp, err 102 | } else { 103 | fmt.Println("Error 503: Service rate limiting. Retrying...") 104 | waitTime := time.Duration(rand.Intn(10)+1) * time.Second 105 | time.Sleep(waitTime) 106 | continue 107 | } 108 | } else { 109 | return resp, err 110 | } 111 | } else { 112 | return resp, err 113 | } 114 | } 115 | return resp, err 116 | } 117 | 118 | func restoreOfsObjects(c *cos.Client, bucketName, prefix string, fo *FileOperations, marker string) error { 119 | var err error 120 | var objects []cos.Object 121 | var commonPrefixes []string 122 | isTruncated := true 123 | 124 | for isTruncated { 125 | err, objects, commonPrefixes, isTruncated, marker = getOfsObjectListForLs(c, prefix, marker, 0, true) 126 | if err != nil { 127 | return fmt.Errorf("list objects error : %v", err) 128 | } 129 | 130 | for _, object := range objects { 131 | if object.StorageClass == Archive || object.StorageClass == MAZArchive || object.StorageClass == DeepArchive { 132 | object.Key, _ = url.QueryUnescape(object.Key) 133 | if cosObjectMatchPatterns(object.Key, fo.Operation.Filters) { 134 | if object.RestoreStatus == "ONGOING" || object.RestoreStatus == "ONGING" { 135 | succeedNum += 1 136 | } else { 137 | resp, err := TryRestoreObject(c, bucketName, object.Key, fo.Operation.Days, fo.Operation.RestoreMode) 138 | if err != nil { 139 | if resp != nil && resp.StatusCode == 409 { 140 | succeedNum += 1 141 | } else { 142 | failedNum += 1 143 | writeError(fmt.Sprintf("restore %s failed , errMsg:%v\n", object.Key, err), fo) 144 | } 145 | } else { 146 | succeedNum += 1 147 | } 148 | } 149 | 150 | } 151 | } else { 152 | errTypeNum += 1 153 | } 154 | } 155 | 156 | if len(commonPrefixes) > 0 { 157 | for _, commonPrefix := range commonPrefixes { 158 | commonPrefix, _ = url.QueryUnescape(commonPrefix) 159 | // 递归目录 160 | err = restoreOfsObjects(c, bucketName, commonPrefix, fo, "") 161 | if err != nil { 162 | return err 163 | } 164 | } 165 | } 166 | } 167 | 168 | return nil 169 | } 170 | -------------------------------------------------------------------------------- /util/secret.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "bytes" 5 | "crypto/aes" 6 | "crypto/cipher" 7 | "crypto/rand" 8 | "encoding/base64" 9 | "fmt" 10 | "io" 11 | ) 12 | 13 | const ( 14 | AesKey = "coscli-secret" 15 | AesBlockSize = 16 16 | ) 17 | 18 | func DecryptSecret(encode string) (decryptStr string, err error) { 19 | decode, err := base64.StdEncoding.DecodeString(encode) 20 | if err != nil { 21 | return "", err 22 | } 23 | tool := NewAesTool([]byte(AesKey), AesBlockSize, ECB) 24 | decrypt, err := tool.Decrypt(decode) 25 | decryptStr = string(decrypt) 26 | return decryptStr, err 27 | } 28 | 29 | func EncryptSecret(src string) (encode string, err error) { 30 | tool := NewAesTool([]byte(AesKey), AesBlockSize, ECB) 31 | encrypt, err := tool.Encrypt([]byte(src)) 32 | encode = base64.StdEncoding.EncodeToString(encrypt) 33 | return encode, err 34 | } 35 | 36 | const ( 37 | ECB = 1 38 | CBC = 2 39 | ) 40 | 41 | // AES ECB模式的加密解密 42 | type AesTool struct { 43 | // 128 192 256位的其中一个 长度 对应分别是 16 24 32字节长度 44 | Key []byte 45 | BlockSize int 46 | Mode int 47 | } 48 | 49 | func NewAesTool(key []byte, blockSize int, mode int) *AesTool { 50 | return &AesTool{Key: key, BlockSize: blockSize, Mode: mode} 51 | } 52 | 53 | /** 54 | 注意：0填充方式 55 | */ 56 | func (this *AesTool) padding(src []byte) []byte { 57 | // 填充个数 58 | paddingCount := aes.BlockSize - len(src)%aes.BlockSize 59 | if paddingCount == 0 { 60 | return src 61 | } else { 62 | // 填充数据 63 | return append(src, bytes.Repeat([]byte{byte(0)}, paddingCount)...) 64 | } 65 | } 66 | 67 | // unpadding 68 | func (this *AesTool) unPadding(src []byte) []byte { 69 | for i := len(src) - 1; i >= 0; i-- { 70 | if src[i] != 0 { 71 | return src[:i+1] 72 | } 73 | } 74 | return nil 75 | } 76 | 77 | func (this *AesTool) Encrypt(src []byte) ([]byte, error) { 78 | var encryptData []byte 79 | // key只能是 16 24 32长度 80 | key := this.padding(this.Key) 81 | block, err := aes.NewCipher([]byte(key)) 82 | if err != nil { 83 | return nil, err 84 | } 85 | // padding 86 | src = this.padding(src) 87 | 88 | switch this.Mode { 89 | case ECB: 90 | encryptData = make([]byte, len(src)) 91 | mode := NewECBEncrypter(block) 92 | mode.CryptBlocks(encryptData, src) 93 | break 94 | case CBC: 95 | // The IV needs to be unique, but not secure. Therefore it's common to 96 | // include it at the beginning of the ciphertext. 97 | encryptData := make([]byte, aes.BlockSize+len(src)) 98 | iv := encryptData[:aes.BlockSize] 99 | if _, err := io.ReadFull(rand.Reader, iv); err != nil { 100 | panic(err) 101 | } 102 | 103 | mode := cipher.NewCBCEncrypter(block, iv) 104 | mode.CryptBlocks(encryptData[aes.BlockSize:], src) 105 | break 106 | } 107 | 108 | return encryptData, nil 109 | 110 | } 111 | func (this *AesTool) Decrypt(src []byte) (res []byte, err error) { 112 | defer func() { 113 | if err1 := recover(); err1 != nil { 114 | err = fmt.Errorf(fmt.Sprintf("%v", err1)) 115 | } 116 | }() 117 | 118 | // key只能是 16 24 32长度 119 | key := this.padding(this.Key) 120 | 121 | block, err := aes.NewCipher([]byte(key)) 122 | if err != nil { 123 | return nil, err 124 | } 125 | 126 | switch this.Mode { 127 | case ECB: 128 | mode := NewECBDecrypter(block) 129 | 130 | // CryptBlocks can work in-place if the two arguments are the same. 131 | mode.CryptBlocks(src, src) 132 | break 133 | case CBC: 134 | iv := src[:aes.BlockSize] 135 | src = src[aes.BlockSize:] 136 | fmt.Printf("decode iv :%x\n", iv) 137 | // CBC mode always works in whole blocks. 138 | if len(src)%aes.BlockSize != 0 { 139 | panic("ciphertext is not a multiple of the block size") 140 | } 141 | 142 | mode := cipher.NewCBCDecrypter(block, iv) 143 | 144 | // CryptBlocks can work in-place if the two arguments are the same. 145 | mode.CryptBlocks(src, src) 146 | break 147 | } 148 | 149 | return this.unPadding(src), nil 150 | } 151 | 152 | // Copyright 2013 The Go Authors. All rights reserved. 153 | // Use of this source code is governed by a BSD-style 154 | // license that can be found in the LICENSE file. 155 | 156 | // Electronic Code Book (ECB) mode. 157 | 158 | // ECB provides confidentiality by assigning a fixed ciphertext block to each 159 | // plaintext block. 160 | 161 | // See NIST SP 800-38A, pp 08-09 162 | 163 | type ecb struct { 164 | b cipher.Block 165 | blockSize int 166 | } 167 | 168 | func newECB(b cipher.Block) *ecb { 169 | return &ecb{ 170 | b: b, 171 | blockSize: b.BlockSize(), 172 | } 173 | } 174 | 175 | type ecbEncrypter ecb 176 | 177 | // NewECBEncrypter returns a BlockMode which encrypts in electronic code book 178 | // mode, using the given Block. 179 | func NewECBEncrypter(b cipher.Block) cipher.BlockMode { 180 | return (*ecbEncrypter)(newECB(b)) 181 | } 182 | 183 | func (x *ecbEncrypter) BlockSize() int { return x.blockSize } 184 | 185 | func (x *ecbEncrypter) CryptBlocks(dst, src []byte) { 186 | if len(src)%x.blockSize != 0 { 187 | panic("crypto/cipher: input not full blocks") 188 | } 189 | if len(dst) < len(src) { 190 | panic("crypto/cipher: output smaller than input") 191 | } 192 | for len(src) > 0 { 193 | x.b.Encrypt(dst, src[:x.blockSize]) 194 | src = src[x.blockSize:] 195 | dst = dst[x.blockSize:] 196 | } 197 | } 198 | 199 | type ecbDecrypter ecb 200 | 201 | // NewECBDecrypter returns a BlockMode which decrypts in electronic code book 202 | // mode, using the given Block. 203 | func NewECBDecrypter(b cipher.Block) cipher.BlockMode { 204 | return (*ecbDecrypter)(newECB(b)) 205 | } 206 | 207 | func (x *ecbDecrypter) BlockSize() int { return x.blockSize } 208 | 209 | func (x *ecbDecrypter) CryptBlocks(dst, src []byte) { 210 | if len(src)%x.blockSize != 0 { 211 | panic("crypto/cipher: input not full blocks") 212 | } 213 | if len(dst) < len(src) { 214 | panic("crypto/cipher: output smaller than input") 215 | } 216 | for len(src) > 0 { 217 | x.b.Decrypt(dst, src[:x.blockSize]) 218 | src = src[x.blockSize:] 219 | dst = dst[x.blockSize:] 220 | } 221 | } 222 | -------------------------------------------------------------------------------- /util/size.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | func FormatSize(b int64) string { 9 | if b < 1024 { 10 | return fmt.Sprintf("%d B", b) 11 | } else if b < 1024*1024 { 12 | return fmt.Sprintf("%.2f KB", float64(b)/1024) 13 | } else if b < 1024*1024*1024 { 14 | return fmt.Sprintf("%.2f MB", float64(b)/(1024*1024)) 15 | } else if b < 1024*1024*1024*1024 { 16 | return fmt.Sprintf("%.2f GB", float64(b)/(1024*1024*1024)) 17 | } else { 18 | return fmt.Sprintf("%.2f TB", float64(b)/(1024*1024*1024*1024)) 19 | } 20 | } 21 | 22 | func formatBytes(bytes float64) string { 23 | const ( 24 | KB = 1024 25 | MB = KB * 1024 26 | GB = MB * 1024 27 | TB = GB * 1024 28 | ) 29 | 30 | switch { 31 | case bytes < KB: 32 | return fmt.Sprintf("%.2f B", bytes) 33 | case bytes < MB: 34 | return fmt.Sprintf("%.2f KB", bytes/KB) 35 | case bytes < GB: 36 | return fmt.Sprintf("%.2f MB", bytes/MB) 37 | case bytes < TB: 38 | return fmt.Sprintf("%.2f GB", bytes/GB) 39 | default: 40 | return fmt.Sprintf("%.2f TB", bytes/TB) 41 | } 42 | } 43 | func getSizeString(size int64) string { 44 | prefix := "" 45 | str := fmt.Sprintf("%d", size) 46 | if size < 0 { 47 | prefix = "-" 48 | str = str[1:] 49 | } 50 | len := len(str) 51 | strList := []string{} 52 | i := len % 3 53 | if i != 0 { 54 | strList = append(strList, str[0:i]) 55 | } 56 | for ; i < len; i += 3 { 57 | strList = append(strList, str[i:i+3]) 58 | } 59 | 60 | sizeStr := formatBytes(float64(size)) 61 | return fmt.Sprintf("%s%s Byte (%s)", prefix, strings.Join(strList, ","), sizeStr) 62 | } 63 | 64 | func max(a, b int64) int64 { 65 | if a >= b { 66 | return a 67 | } 68 | return b 69 | } 70 | -------------------------------------------------------------------------------- /util/symlink.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "context" 5 | "github.com/tencentyun/cos-go-sdk-v5" 6 | ) 7 | 8 | func CreateSymlink(c *cos.Client, cosUrl StorageUrl, linkKey string) error { 9 | opt := &cos.ObjectPutSymlinkOptions{ 10 | SymlinkTarget: cosUrl.(*CosUrl).Object, 11 | } 12 | _, err := c.Object.PutSymlink(context.Background(), linkKey, opt) 13 | return err 14 | } 15 | 16 | func GetSymlink(c *cos.Client, linkKey string) (res string, err error) { 17 | res, _, err = c.Object.GetSymlink(context.Background(), linkKey, nil) 18 | return res, err 19 | } 20 | -------------------------------------------------------------------------------- /util/types.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "github.com/olekukonko/tablewriter" 5 | "github.com/syndtr/goleveldb/leveldb" 6 | "net/http" 7 | "os" 8 | ) 9 | 10 | type Config struct { 11 | Base BaseCfg `yaml:"base"` 12 | Buckets []Bucket `yaml:"buckets"` 13 | } 14 | 15 | type BaseCfg struct { 16 | SecretID string `yaml:"secretid"` 17 | SecretKey string `yaml:"secretkey"` 18 | SessionToken string `yaml:"sessiontoken"` 19 | Protocol string `yaml:"protocol"` 20 | Mode string `yaml:"mode"` 21 | CvmRoleName string `yaml:"cvmrolename"` 22 | CloseAutoSwitchHost string `yaml:"closeautoswitchhost"` 23 | DisableEncryption string `yaml:"disableencryption"` 24 | } 25 | 26 | type Bucket struct { 27 | Name string `yaml:"name"` 28 | Alias string `yaml:"alias"` 29 | Region string `yaml:"region"` 30 | Endpoint string `yaml:"endpoint"` 31 | Ofs bool `yaml:"ofs"` 32 | } 33 | type Param struct { 34 | SecretID string 35 | SecretKey string 36 | SessionToken string 37 | Endpoint string 38 | Customized bool 39 | Protocol string 40 | } 41 | 42 | type UploadInfo struct { 43 | Key string `xml:"Key,omitempty"` 44 | UploadID string `xml:"UploadId,omitempty"` 45 | Initiated string `xml:"Initiated,omitempty"` 46 | } 47 | 48 | type fileInfoType struct { 49 | filePath string 50 | dir string 51 | } 52 | 53 | type objectInfoType struct { 54 | prefix string 55 | relativeKey string 56 | size int64 57 | lastModified string 58 | } 59 | 60 | type CpType int 61 | 62 | type FileOperations struct { 63 | Operation Operation 64 | Monitor *FileProcessMonitor 65 | ErrOutput *ErrOutput 66 | Config *Config 67 | Param *Param 68 | SnapshotDb *leveldb.DB 69 | CpType CpType 70 | Command string 71 | DeleteCount int 72 | BucketType string 73 | } 74 | 75 | type Operation struct { 76 | Recursive bool 77 | Filters []FilterOptionType 78 | StorageClass string 79 | RateLimiting float32 80 | PartSize int64 81 | ThreadNum int 82 | Routines int 83 | FailOutput bool 84 | FailOutputPath string 85 | Meta Meta 86 | RetryNum int 87 | ErrRetryNum int 88 | ErrRetryInterval int 89 | OnlyCurrentDir bool 90 | DisableAllSymlink bool 91 | EnableSymlinkDir bool 92 | DisableCrc64 bool 93 | DisableChecksum bool 94 | DisableLongLinks bool 95 | LongLinksNums int 96 | VersionId string 97 | AllVersions bool 98 | SnapshotPath string 99 | Delete bool 100 | BackupDir string 101 | Force bool 102 | Days int 103 | RestoreMode string 104 | Move bool 105 | } 106 | 107 | type ErrOutput struct { 108 | Path string 109 | outputFile *os.File 110 | } 111 | 112 | type FilterOptionType struct { 113 | name string 114 | pattern string 115 | } 116 | 117 | type Meta struct { 118 | CacheControl string 119 | ContentDisposition string 120 | ContentEncoding string 121 | ContentType string 122 | ContentMD5 string 123 | ContentLength int64 124 | ContentLanguage string 125 | Expires string 126 | // 自定义的 x-cos-meta-* header 127 | XCosMetaXXX *http.Header 128 | MetaChange bool 129 | } 130 | 131 | type LsCounter struct { 132 | TotalLimit int 133 | RenderNum int 134 | Table *tablewriter.Table 135 | } 136 | -------------------------------------------------------------------------------- /util/upload.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/tencentyun/cos-go-sdk-v5" 7 | "math/rand" 8 | "os" 9 | "path/filepath" 10 | "strconv" 11 | "strings" 12 | "sync" 13 | "time" 14 | ) 15 | 16 | var ( 17 | mu sync.Mutex 18 | ) 19 | 20 | // 定义一个结构体类型 21 | type Counter struct { 22 | TransferSize int64 23 | } 24 | 25 | func Upload(c *cos.Client, fileUrl StorageUrl, cosUrl StorageUrl, fo *FileOperations) { 26 | startT := time.Now().UnixNano() / 1000 / 1000 27 | localPath := fileUrl.ToString() 28 | 29 | fo.Monitor.init(fo.CpType) 30 | chProgressSignal = make(chan chProgressSignalType, 10) 31 | go progressBar(fo) 32 | 33 | chFiles := make(chan fileInfoType, ChannelSize) 34 | chError := make(chan error, fo.Operation.Routines) 35 | chListError := make(chan error, 1) 36 | // 统计文件数量及大小数据 37 | go fileStatistic(localPath, fo) 38 | // 生成文件列表 39 | go generateFileList(localPath, chFiles, chListError, fo) 40 | 41 | for i := 0; i < fo.Operation.Routines; i++ { 42 | go uploadFiles(c, cosUrl, fo, chFiles, chError) 43 | } 44 | 45 | completed := 0 46 | for completed <= fo.Operation.Routines { 47 | select { 48 | case err := <-chListError: 49 | if err != nil { 50 | if fo.Operation.FailOutput { 51 | writeError(err.Error(), fo) 52 | } 53 | } 54 | completed++ 55 | case err := <-chError: 56 | if err == nil { 57 | completed++ 58 | } else { 59 | if fo.Operation.FailOutput { 60 | writeError(err.Error(), fo) 61 | } 62 | } 63 | } 64 | } 65 | 66 | closeProgress() 67 | fmt.Printf(fo.Monitor.progressBar(true, normalExit)) 68 | 69 | endT := time.Now().UnixNano() / 1000 / 1000 70 | PrintTransferStats(startT, endT, fo) 71 | } 72 | 73 | func uploadFiles(c *cos.Client, cosUrl StorageUrl, fo *FileOperations, chFiles <-chan fileInfoType, chError chan<- error) { 74 | for file := range chFiles { 75 | var skip, isDir bool 76 | var err error 77 | var size, transferSize int64 78 | var msg string 79 | for retry := 0; retry <= fo.Operation.ErrRetryNum; retry++ { 80 | skip, err, isDir, size, transferSize, msg = SingleUpload(c, fo, file, cosUrl) 81 | if err == nil { 82 | break // Upload succeeded, break the loop 83 | } else { 84 | // 服务端重试在go sdk内部进行，客户端仅重试文件上传完完整性校验不通过的case 85 | if retry < fo.Operation.ErrRetryNum && strings.HasPrefix(err.Error(), "verification failed, want:") { 86 | if fo.Operation.ErrRetryInterval == 0 { 87 | // If the retry interval is not specified, retry after a random interval of 1~10 seconds. 88 | time.Sleep(time.Duration(rand.Intn(10)+1) * time.Second) 89 | } else { 90 | time.Sleep(time.Duration(fo.Operation.ErrRetryInterval) * time.Second) 91 | } 92 | 93 | fo.Monitor.updateDealSize(-transferSize) 94 | } 95 | } 96 | } 97 | 98 | fo.Monitor.updateMonitor(skip, err, isDir, size) 99 | if err != nil { 100 | chError <- fmt.Errorf("%s failed: %w", msg, err) 101 | continue 102 | } 103 | } 104 | 105 | chError <- nil 106 | } 107 | 108 | func SingleUpload(c *cos.Client, fo *FileOperations, file fileInfoType, cosUrl StorageUrl) (skip bool, rErr error, isDir bool, size, transferSize int64, msg string) { 109 | skip = false 110 | rErr = nil 111 | isDir = false 112 | size = 0 113 | transferSize = 0 114 | 115 | localFilePath, cosPath := UploadPathFixed(file, cosUrl.(*CosUrl).Object) 116 | 117 | fileInfo, err := os.Stat(localFilePath) 118 | if err != nil { 119 | rErr = err 120 | return 121 | } 122 | 123 | var snapshotKey string 124 | 125 | msg = fmt.Sprintf("\nUpload %s to %s", localFilePath, getCosUrl(cosUrl.(*CosUrl).Bucket, cosPath)) 126 | if fileInfo.IsDir() { 127 | isDir = true 128 | // 在cos创建文件夹 129 | _, err = c.Object.Put(context.Background(), cosPath, strings.NewReader(""), nil) 130 | if err != nil { 131 | rErr = err 132 | return 133 | } 134 | } else { 135 | size = fileInfo.Size() 136 | 137 | // 仅sync命令执行skip 138 | if fo.Command == CommandSync { 139 | absLocalFilePath, _ := filepath.Abs(localFilePath) 140 | snapshotKey = getUploadSnapshotKey(absLocalFilePath, cosUrl.(*CosUrl).Bucket, cosUrl.(*CosUrl).Object) 141 | skip, err = skipUpload(snapshotKey, c, fo, fileInfo.ModTime().Unix(), cosPath, localFilePath) 142 | if err != nil { 143 | rErr = err 144 | return 145 | } 146 | } 147 | 148 | if skip { 149 | return 150 | } 151 | 152 | opt := &cos.MultiUploadOptions{ 153 | OptIni: &cos.InitiateMultipartUploadOptions{ 154 | ACLHeaderOptions: &cos.ACLHeaderOptions{ 155 | XCosACL: "", 156 | XCosGrantRead: "", 157 | XCosGrantWrite: "", 158 | XCosGrantFullControl: "", 159 | XCosGrantReadACP: "", 160 | XCosGrantWriteACP: "", 161 | }, 162 | ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{ 163 | CacheControl: fo.Operation.Meta.CacheControl, 164 | ContentDisposition: fo.Operation.Meta.ContentDisposition, 165 | ContentEncoding: fo.Operation.Meta.ContentEncoding, 166 | ContentType: fo.Operation.Meta.ContentType, 167 | ContentMD5: fo.Operation.Meta.ContentMD5, 168 | ContentLength: fo.Operation.Meta.ContentLength, 169 | ContentLanguage: fo.Operation.Meta.ContentLanguage, 170 | Expect: "", 171 | Expires: fo.Operation.Meta.Expires, 172 | XCosContentSHA1: "", 173 | XCosMetaXXX: fo.Operation.Meta.XCosMetaXXX, 174 | XCosStorageClass: fo.Operation.StorageClass, 175 | XCosServerSideEncryption: "", 176 | XCosSSECustomerAglo: "", 177 | XCosSSECustomerKey: "", 178 | XCosSSECustomerKeyMD5: "", 179 | XOptionHeader: nil, 180 | XCosTrafficLimit: (int)(fo.Operation.RateLimiting * 1024 * 1024 * 8), 181 | }, 182 | }, 183 | PartSize: fo.Operation.PartSize, 184 | ThreadPoolSize: fo.Operation.ThreadNum, 185 | CheckPoint: true, 186 | DisableChecksum: fo.Operation.DisableChecksum, 187 | } 188 | 189 | counter := &Counter{TransferSize: 0} 190 | // 未跳过则通过监听更新size(仅需要分块文件的通过sdk监听进度) 191 | if size > fo.Operation.PartSize*1024*1024 { 192 | opt.OptIni.Listener = &CosListener{fo, counter} 193 | size = 0 194 | } 195 | 196 | _, _, err = c.Object.Upload(context.Background(), cosPath, localFilePath, opt) 197 | 198 | if err != nil { 199 | if strings.HasPrefix(err.Error(), "verification failed, want:") { 200 | transferSize = counter.TransferSize 201 | } 202 | rErr = err 203 | return 204 | } 205 | } 206 | 207 | if snapshotKey != "" && fo.Operation.SnapshotPath != "" { 208 | // 上传成功后添加快照 209 | fo.SnapshotDb.Put([]byte(snapshotKey), []byte(strconv.FormatInt(fileInfo.ModTime().Unix(), 10)), nil) 210 | } 211 | 212 | return 213 | } 214 | -------------------------------------------------------------------------------- /util/url.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "github.com/tencentyun/cos-go-sdk-v5" 6 | "net/url" 7 | ) 8 | 9 | func GenBucketURL(bucketIDName string, protocol string, endpoint string, customized bool) string { 10 | b := "" 11 | if customized { 12 | b = fmt.Sprintf("%s://%s", protocol, endpoint) 13 | } else { 14 | b = fmt.Sprintf("%s://%s.%s", protocol, bucketIDName, endpoint) 15 | } 16 | 17 | return b 18 | } 19 | 20 | func GenServiceURL(protocol string, endpoint string) string { 21 | s := fmt.Sprintf("%s://%s", protocol, endpoint) 22 | return s 23 | } 24 | 25 | func GenCiURL(bucketIDName string, protocol string, endpoint string) string { 26 | c := fmt.Sprintf("%s://%s.%s", protocol, bucketIDName, endpoint) 27 | return c 28 | } 29 | 30 | // 根据函数参数生成URL 31 | func CreateURL(idName string, protocol string, endpoint string, customized bool) *cos.BaseURL { 32 | b := GenBucketURL(idName, protocol, endpoint, customized) 33 | s := GenServiceURL(protocol, endpoint) 34 | c := GenCiURL(idName, protocol, endpoint) 35 | 36 | bucketURL, _ := url.Parse(b) 37 | serviceURL, _ := url.Parse(s) 38 | ciURL, _ := url.Parse(c) 39 | 40 | return &cos.BaseURL{ 41 | BucketURL: bucketURL, 42 | ServiceURL: serviceURL, 43 | CIURL: ciURL, 44 | } 45 | } 46 | 47 | // 根据配置文件生成ServiceURL 48 | func GenBaseURL(config *Config, param *Param) *cos.BaseURL { 49 | if param.Endpoint == "" { 50 | return nil 51 | } 52 | endpoint := param.Endpoint 53 | 54 | protocol := "https" 55 | if config.Base.Protocol != "" { 56 | protocol = config.Base.Protocol 57 | } 58 | if param.Protocol != "" { 59 | protocol = param.Protocol 60 | } 61 | 62 | return CreateBaseURL(protocol, endpoint) 63 | } 64 | 65 | // 根据函数参数生成ServiceURL 66 | func CreateBaseURL(protocol string, endpoint string) *cos.BaseURL { 67 | service := GenServiceURL(protocol, endpoint) 68 | serviceURL, _ := url.Parse(service) 69 | 70 | return &cos.BaseURL{ 71 | ServiceURL: serviceURL, 72 | } 73 | } 74 | 75 | // 根据配置文件生成URL 76 | func GenURL(config *Config, param *Param, bucketName string) (url *cos.BaseURL, err error) { 77 | bucket, _, err := FindBucket(config, bucketName) 78 | if err != nil { 79 | return url, err 80 | } 81 | 82 | idName := bucket.Name 83 | endpoint := bucket.Endpoint 84 | if param.Endpoint != "" { 85 | endpoint = param.Endpoint 86 | } 87 | if endpoint == "" && bucket.Region != "" { 88 | endpoint = fmt.Sprintf("cos.%s.myqcloud.com", bucket.Region) 89 | } 90 | 91 | if endpoint == "" { 92 | return nil, fmt.Errorf("endpoint is missing") 93 | } 94 | 95 | protocol := "https" 96 | if config.Base.Protocol != "" { 97 | protocol = config.Base.Protocol 98 | } 99 | if param.Protocol != "" { 100 | protocol = param.Protocol 101 | } 102 | 103 | customized := param.Customized 104 | 105 | return CreateURL(idName, protocol, endpoint, customized), nil 106 | } 107 | -------------------------------------------------------------------------------- /util/versioning.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "context" 5 | "github.com/tencentyun/cos-go-sdk-v5" 6 | ) 7 | 8 | func GetBucketVersioning(c *cos.Client) (res *cos.BucketGetVersionResult, resp *cos.Response, err error) { 9 | res, resp, err = c.Bucket.GetVersioning(context.Background()) 10 | if err != nil { 11 | return nil, nil, err 12 | } 13 | return res, resp, err 14 | } 15 | 16 | func PutBucketVersioning(c *cos.Client, status string) (resp *cos.Response, err error) { 17 | opt := &cos.BucketPutVersionOptions{ 18 | Status: status, 19 | } 20 | resp, err = c.Bucket.PutVersioning(context.Background(), opt) 21 | return resp, err 22 | } 23 | --------------------------------------------------------------------------------