├── .github └── workflows │ ├── build.yml │ ├── goreleaser.yml │ ├── lint.yml │ └── test.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── VERSION ├── basic_test.go ├── cmd └── upx │ └── upx.go ├── commands.go ├── config.go ├── copy_test.go ├── db.go ├── fscmds_test.go ├── fsutil ├── ignore.go ├── ignore_unix.go └── ignore_windows.go ├── get_test.go ├── go.mod ├── go.sum ├── install.sh ├── io.go ├── match.go ├── move_test.go ├── partial ├── chunk.go ├── downloader.go └── downloader_test.go ├── processbar ├── bar.go └── fmt.go ├── putgetrm_test.go ├── putiginore_test.go ├── session.go ├── tree_test.go ├── upgrade.go ├── upx.go ├── upx_test.go ├── utils.go └── xerrors └── errors.go /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | on: 3 | pull_request: 4 | push: 5 | 6 | jobs: 7 | build: 8 | strategy: 9 | matrix: 10 | go-version: [^1] 11 | os: [ubuntu-latest, macos-latest, windows-latest] 12 | runs-on: ${{ matrix.os }} 13 | env: 14 | GO111MODULE: "on" 15 | steps: 16 | - name: Install Go 17 | uses: actions/setup-go@v5 18 | with: 19 | go-version: ${{ matrix.go-version }} 20 | 21 | - name: Checkout code 22 | uses: actions/checkout@v4 23 | - name: Download Go modules 24 | run: go mod download 25 | 26 | - name: Build 27 | run: | 28 | go build -v ./cmd/upx 29 | ./upx -v 30 | -------------------------------------------------------------------------------- /.github/workflows/goreleaser.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | goreleaser: 10 | runs-on: ubuntu-latest 11 | env: 12 | UPYUN_RELEASE_BUCKET: ${{ secrets.UPYUN_RELEASE_BUCKET }} 13 | UPYUN_RELEASE_USERNAME: ${{ secrets.UPYUN_RELEASE_USERNAME }} 14 | UPYUN_RELEASE_PASSWORD: ${{ secrets.UPYUN_RELEASE_PASSWORD }} 15 | steps: 16 | - 17 | name: Checkout 18 | uses: actions/checkout@v4 19 | with: 20 | fetch-depth: 0 21 | - 22 | name: Set up Go 23 | uses: actions/setup-go@v5 24 | - name: Build Upx 25 | run: | 26 | go build ./cmd/upx 27 | ./upx login "${{ env.UPYUN_RELEASE_BUCKET }}" "${{ env.UPYUN_RELEASE_USERNAME }}" "${{ env.UPYUN_RELEASE_PASSWORD }}" 28 | - 29 | name: Run GoReleaser 30 | uses: goreleaser/goreleaser-action@v6 31 | with: 32 | distribution: goreleaser 33 | version: '~> v2' 34 | args: release --clean 35 | env: 36 | GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} 37 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | on: 3 | pull_request: 4 | push: 5 | 6 | jobs: 7 | golangci: 8 | name: lint 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v4 12 | - name: golangci-lint 13 | uses: golangci/golangci-lint-action@v7 14 | with: 15 | # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. 16 | version: latest 17 | # Optional: golangci-lint command line arguments. 18 | args: 19 | # Optional: working directory, useful for monorepos 20 | # working-directory: somedir 21 | # Optional: show only new issues if it's a pull request. The default value is `false`. 22 | only-new-issues: true 23 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: 3 | pull_request_target: 4 | types: [labeled] 5 | paths-ignore: 6 | - "**.md" 7 | push: 8 | paths-ignore: 9 | - "**.md" 10 | 11 | jobs: 12 | build: 13 | strategy: 14 | matrix: 15 | go-version: [^1] 16 | os: [ubuntu-latest, macos-latest, windows-latest] 17 | runs-on: ${{ matrix.os }} 18 | if: ${{ github.event_name != 'pull_request_target' || (contains(github.event.pull_request.labels.*.name, 'ok to test') && github.event.pull_request.state == 'open') }} 19 | env: 20 | GO111MODULE: "on" 21 | UPYUN_BUCKET1: ${{ secrets.UPYUN_BUCKET1 }} 22 | UPYUN_BUCKET2: ${{ secrets.UPYUN_BUCKET2 }} 23 | UPYUN_PASSWORD: ${{ secrets.UPYUN_PASSWORD }} 24 | UPYUN_USERNAME: ${{ secrets.UPYUN_USERNAME }} 25 | steps: 26 | - name: Remove 'ok to test' Label 27 | if: ${{ github.event_name == 'pull_request_target' }} 28 | uses: actions-ecosystem/action-remove-labels@v1.3.0 29 | with: 30 | labels: 'ok to test' 31 | - name: Install Go 32 | uses: actions/setup-go@v5 33 | with: 34 | go-version: ${{ matrix.go-version }} 35 | 36 | - name: Checkout code 37 | uses: actions/checkout@v4 38 | with: 39 | ref: ${{github.event.pull_request.head.ref}} 40 | repository: ${{github.event.pull_request.head.repo.full_name}} 41 | 42 | - uses: actions/cache@v4 43 | with: 44 | path: ~/go/pkg/mod 45 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 46 | restore-keys: | 47 | ${{ runner.os }}-go- 48 | 49 | - name: Download Go modules 50 | run: go mod download 51 | - name: Test 52 | run: | 53 | go build -v ./cmd/upx 54 | go test -v ./... 55 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /upx 2 | *.swo 3 | *.swp 4 | *.swn 5 | .vscode 6 | tmp 7 | upx-* 8 | release 9 | dist 10 | FILE 11 | list 12 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | run: 3 | tests: false 4 | linters: 5 | default: none 6 | enable: 7 | - dogsled 8 | - dupl 9 | - gochecknoinits 10 | - gocyclo 11 | - goprintffuncname 12 | - misspell 13 | - nolintlint 14 | - unconvert 15 | - unparam 16 | - whitespace 17 | exclusions: 18 | generated: lax 19 | presets: 20 | - comments 21 | - common-false-positives 22 | - legacy 23 | - std-error-handling 24 | paths: 25 | - third_party$ 26 | - builtin$ 27 | - examples$ 28 | issues: 29 | max-issues-per-linter: 0 30 | max-same-issues: 0 31 | formatters: 32 | enable: 33 | - gofmt 34 | - goimports 35 | exclusions: 36 | generated: lax 37 | paths: 38 | - third_party$ 39 | - builtin$ 40 | - examples$ 41 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | before: 2 | hooks: 3 | - go mod download 4 | builds: 5 | - id: "upx" 6 | main: ./cmd/upx 7 | env: 8 | - CGO_ENABLED=0 9 | binary: upx 10 | flags: 11 | - -trimpath 12 | goos: 13 | - linux 14 | - darwin 15 | - windows 16 | goarch: 17 | - amd64 18 | - arm64 19 | - 386 20 | - arm 21 | goarm: 22 | - 6 23 | - 7 24 | 25 | archives: 26 | - id: default 27 | builds: 28 | - upx 29 | format_overrides: 30 | - goos: windows 31 | format: zip 32 | checksum: 33 | name_template: "checksums.txt" 34 | publishers: 35 | - name: upyun 36 | checksum: true 37 | cmd: ../upx put {{ .ArtifactName }} /softwares/upx/ 38 | dir: "{{ dir .ArtifactPath }}" 39 | env: 40 | - HOME={{ .Env.HOME }} 41 | snapshot: 42 | name_template: "{{ .Tag }}-next" 43 | changelog: 44 | sort: asc 45 | filters: 46 | exclude: 47 | - "^docs:" 48 | - "^test:" 49 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:alpine 2 | COPY . /go/src/upx 3 | WORKDIR /go/src/upx 4 | RUN go get -d -v && go install -v 5 | CMD ["upx"] 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 UPYUN 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ifndef VER 2 | VER= latest 3 | endif 4 | 5 | APP= upx 6 | ROOT= $(shell echo $(GOPATH) | awk -F':' '{print $$1}') 7 | PROJ_DIR= $(ROOT)/src/upyun.com 8 | PWD= $(shell pwd) 9 | 10 | app: 11 | go build -o $(APP) ./cmd/upx/ 12 | 13 | test: 14 | go test -v . 15 | 16 | release: 17 | goreleaser --rm-dist 18 | 19 | upload: release 20 | ./upx pwd 21 | ./upx put dist/upx_darwin_amd64/upx /softwares/upx/upx_darwin_amd64_$(VER); \ 22 | 23 | for ARCH in amd64 386 arm64 arm_6 arm_7; do \ 24 | ./upx put dist/upx_linux_$$ARCH/upx /softwares/upx/upx_linux_$$ARCH_$(VER); \ 25 | done 26 | 27 | for ARCH in amd64 386; do \ 28 | ./upx put dist/upx_windows_$$ARCH/upx.exe /softwares/upx/upx_windows_$$ARCH_$(VER).exe; \ 29 | done 30 | 31 | .PHONY: app test release upload 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | > upx is a tool for managing files in UPYUN. Mac, Linux, Windows supported 2 | 3 | ![Test](https://github.com/upyun/upx/workflows/Test/badge.svg) 4 | ![Build](https://github.com/upyun/upx/workflows/Build/badge.svg) 5 | ![Lint](https://github.com/upyun/upx/workflows/Lint/badge.svg) 6 | [![Go Report Card](https://goreportcard.com/badge/github.com/upyun/upx)](https://goreportcard.com/report/github.com/upyun/upx) 7 | ![GitHub tag (latest by date)](https://img.shields.io/github/v/tag/upyun/upx?label=latest%20release) 8 | 9 | ## 基本功能 10 | 11 | - [x] 支持基本文件系统操作命令，如 `mkdir`, `cd`, `ls`, `rm`, `pwd` 12 | - [x] 支持上传文件或目录到又拍云存储 13 | - [x] 支持从又拍云存储下载文件或目录到本地 14 | - [x] 支持增量同步文件到又拍云存储 15 | - [x] 支持删除又拍云存储中的文件或目录，并且支持通配符 `*` 16 | - [x] 支持多用户，多操作系统 17 | - [x] 支持基于时间列目录以及删除文件 18 | - [x] 支持 `tree` 获取目录结构 19 | - [x] 支持提交异步处理任务 20 | - [x] 更加准确简洁的进度条 21 | - [x] 使用 UPYUN GoSDK v3 22 | - [x] 同步目录支持 --delete 23 | - [x] 支持 CDN 缓存刷新 24 | 25 | ## 安装 26 | 27 | ### 可执行程序二进制下载地址 28 | 29 | - [Windows amd64](https://collection.b0.upaiyun.com/softwares/upx/upx_0.4.9_windows_amd64.zip) 30 | - [Windows arm64](https://collection.b0.upaiyun.com/softwares/upx/upx_0.4.9_windows_arm64.zip) 31 | - [Windows armv6](https://collection.b0.upaiyun.com/softwares/upx/upx_0.4.9_windows_armv6.zip) 32 | - [Windows armv7](https://collection.b0.upaiyun.com/softwares/upx/upx_0.4.9_windows_armv7.zip) 33 | - [Mac amd64](https://collection.b0.upaiyun.com/softwares/upx/upx_0.4.9_darwin_amd64.tar.gz) 34 | - [Mac arm64](https://collection.b0.upaiyun.com/softwares/upx/upx_0.4.9_darwin_arm64.tar.gz) 35 | - [Linux amd64](https://collection.b0.upaiyun.com/softwares/upx/upx_0.4.9_linux_amd64.tar.gz) 36 | - [Linux i386](https://collection.b0.upaiyun.com/softwares/upx/upx_0.4.9_linux_386.tar.gz) 37 | - [Linux arm64](https://collection.b0.upaiyun.com/softwares/upx/upx_0.4.9_linux_arm64.tar.gz) 38 | - [Linux armv6](https://collection.b0.upaiyun.com/softwares/upx/upx_0.4.9_linux_armv6.tar.gz) 39 | - [Linux armv7](https://collection.b0.upaiyun.com/softwares/upx/upx_0.4.9_linux_armv7.tar.gz) 40 | 41 | ### 源码编译 42 | 43 | > 需要安装 [Golang 编译环境](https://golang.org/dl/) 44 | 45 | ``` 46 | $ git clone https://github.com/upyun/upx.git 47 | $ cd upx && make 48 | ``` 49 | or 50 | 51 | ``` 52 | $ go install github.com/upyun/upx/cmd/upx@master 53 | ``` 54 | 55 | ### Windows 56 | 57 | ``` 58 | PS> scoop bucket add carrot https://github.com/upyun/carrot.git 59 | Install upx from github or upyun cdn: 60 | PS> scoop install upx-github 61 | PS> scoop install upx-upcdn 62 | ``` 63 | 64 | ### Docker 65 | 66 | ```bash 67 | docker build -t upx . 68 | docker run --rm upx upx -v 69 | ``` 70 | 71 | --- 72 | 73 | ## 使用 74 | 75 | > 所有命令都支持 `-h` 查看使用方法 76 | 77 | | 命令 | 说明 | 78 | | -------- | ---- | 79 | | [login](#login) | 登录又拍云存储 | 80 | | [logout](#logout) | 退出帐号 | 81 | | [sessions](#sessions) | 查看所有的会话 | 82 | | [switch](#switch) | 切换会话 | 83 | | [info](#info) | 显示服务名、用户名等信息 | 84 | | [ls](#ls) | 显示当前目录下文件和目录信息 | 85 | | [cd](#cd) | 改变工作目录（进入一个目录）| 86 | | [pwd](#pwd) | 显示当前所在目录 | 87 | | [mkdir](#mkdir) | 创建目录 | 88 | | [tree](#tree) | 显示目录结构 | 89 | | [get](#get) | 下载一个文件或目录 | 90 | | [put](#put) | 上传一个文件或目录 | 91 | | [upload](#upload) | 上传多个文件或目录或 http(s) 文件, 支持 Glob 模式过滤上传文件| 92 | | [mv](#mv) | 在同一 bucket 内移动文件| 93 | | [cp](#cp) | 在同一 bucket 内复制文件 | 94 | | [rm](#rm) | 删除目录或文件 | 95 | | [sync](#sync) | 目录增量同步，类似 rsync | 96 | | [auth](#auth) | 生成包含空间名操作员密码信息的 auth 字符串 | 97 | | [post](#post) | 提交异步处理任务 | 98 | | [purge](#purge) | 提交 CDN 缓存刷新任务 | 99 | 100 | 101 | | global options | 说明 | 102 | | -------------- | ---- | 103 | | --quiet, -q | 不显示信息 | 104 | | --auth value | auth 字符串 | 105 | | --help, -h | 显示帮助信息 | 106 | | --version, -v | 显示版本号 | 107 | 108 | 109 | ## login 110 | > 使用又拍云操作员账号登录服务, 登录成功后将会保存会话，支持同时登录多个服务, 使用 `switch` 切换会话。 111 | 112 | 需要提供的验证字段 113 | + ServiceName: 服务(bucket)的名称 114 | + Operator: 操作员名 115 | + Password: 操作员密码 116 | 117 | 118 | #### 语法 119 | ```bash 120 | upx login 121 | ``` 122 | 123 | #### 示例 124 | ```bash 125 | upx login 126 | 127 | #ServiceName: testService 128 | #Operator: upx 129 | #Password: password 130 | ``` 131 | 132 | ## logout 133 | > 退出当前登录的会话，如果存在多个登录的会话，可以使用 `switch` 切换到需要退出的会话，然后退出。 134 | 135 | 136 | #### 语法 137 | ```bash 138 | upx logout 139 | ``` 140 | 141 | #### 示例 142 | ```bash 143 | upx logout 144 | 145 | # Goodbye upx/testService ~ 146 | ``` 147 | 148 | 149 | ## sessions 150 | > 列举出当前登录的所有会话 151 | 152 | #### 语法 153 | ```bash 154 | upx sessions 155 | ``` 156 | 157 | #### 示例 158 | ```bash 159 | upx sessions 160 | 161 | # > mybucket1 162 | # > mybucket2 163 | # > mybucket3 164 | ``` 165 | 166 | ## switch 167 | > 切换登录会话, 通过 `sessions` 命令可以查看所有的会话列表。 168 | 169 | | args | 说明 | 170 | | --------- | ---- | 171 | | service-name | 服务名称(bucket) | 172 | 173 | #### 语法 174 | ```bash 175 | upx switch 176 | ``` 177 | 178 | #### 示例 179 | ```bash 180 | upx switch mybucket3 181 | ``` 182 | 183 | ## info 184 | > 查看当前服务的状态。 185 | 186 | #### 语法 187 | ```bash 188 | upx info 189 | ``` 190 | 191 | #### 示例 192 | ```bash 193 | upx info 194 | > ServiceName: mybucket1 195 | > Operator: tester 196 | > CurrentDir: / 197 | > Usage: 2.69GB 198 | ``` 199 | 200 | ## ls 201 | > 默认按文件修改时间先后顺序输出 202 | 203 | | args | 说明 | 204 | | --------- | ---- | 205 | | remote-path | 远程路径 | 206 | 207 | | options | 说明 | 208 | | --------- | ---- | 209 | | -d | 仅显示目录 | 210 | | -r | 文件修改时间倒序输出 | 211 | | --color | 根据文件类型输出不同的颜色 | 212 | | -c v | 仅显示前 v 个文件或目录, 默认全部显示 | 213 | | --mtime v | 通过文件被修改的时间删选，参考 Linux `find` | 214 | 215 | #### 语法 216 | ```bash 217 | upx ls [options...] [remote-path] 218 | ``` 219 | 220 | #### 示例 221 | 222 | 查看根目录下的文件 223 | ```bash 224 | upx ls / 225 | ``` 226 | 227 | 只查看根目录下的目录 228 | ```bash 229 | upx ls -d / 230 | ``` 231 | 232 | 只查看根目录下的修改时间大于3天的文件 233 | ```bash 234 | upx ls --mtime +3 / 235 | ``` 236 | 237 | 只查看根目录下的修改时间小于1天的文件 238 | ```bash 239 | upx ls --mtime -1 / 240 | ``` 241 | 242 | ## cd 243 | > 改变当前的工作路径，默认工作路径为根目录, 工作路径影响到操作时的默认远程路径。 244 | 245 | | args | 说明 | 246 | | --------- | ---- | 247 | | remote-path | 远程路径 | 248 | 249 | #### 语法 250 | ```bash 251 | upx cd 252 | ``` 253 | 254 | #### 示例 255 | 将当前工作路径切换到 `/www` 256 | ``` 257 | upx cd /www 258 | ``` 259 | 260 | ## pwd 261 | > 显示当前所在的远程目录 262 | 263 | #### 语法 264 | ```bash 265 | upx pwd 266 | ``` 267 | 268 | #### 示例 269 | ```bash 270 | upx pwd 271 | 272 | > /www 273 | ``` 274 | 275 | ## mkdir 276 | > 创建远程目录 277 | 278 | | args | 说明 | 279 | | --------- | ---- | 280 | | remote-dir | 远程目录 | 281 | 282 | #### 语法 283 | ```bash 284 | upx mkdir 285 | ``` 286 | 287 | #### 示例 288 | 在当前工作目录下创建一个名为 mytestbucket 的目录 289 | ```bash 290 | upx mkdir mytestbucket 291 | ``` 292 | 293 | 在根目录下创建一个名为 mytestbucket 的目录 294 | ```bash 295 | upx mkdir /mytestbucket 296 | ``` 297 | 298 | ## tree 299 | > 显示目录结构，树形模式显示 300 | 301 | #### 语法 302 | ```bash 303 | upx tree 304 | ``` 305 | 306 | #### 示例 307 | 查看 `/ccc` 目录下的目录结构 308 | ```bash 309 | upx tree /ccc 310 | > |-- aaacd 311 | > ! |-- mail4788ca.png 312 | > |-- ccc 313 | > ! |-- Eroge de Subete wa Kaiketsu Dekiru! The Animation - 02 (2022) [1080p-HEVC-WEBRip][8D1929F5].mkv 314 | > ! |-- baima_text_auditer.tar 315 | > ! |-- linux-1.txt 316 | ``` 317 | 318 | ## get 319 | > 下载文件 320 | 321 | | args | 说明 | 322 | | --------- | ---- | 323 | | remote-path | 远程路径，支持文件或文件夹 | 324 | | saved-file | 需要保存到的本地目录，或指定完整的文件名 | 325 | 326 | | options | 说明 | 327 | |---------|-----------------------------| 328 | | -w | 多线程下载 (1-10) (default: 5) | 329 | | -c | 恢复中断的下载 | 330 | | --start | 只下载路径字典序大于等于 `start` 的文件或目录 | 331 | | --end | 只下载路径字典序小于 `end` 的文件或目录 | 332 | 333 | 334 | #### 语法 335 | ```bash 336 | upx get [options] [saved-file] 337 | ``` 338 | 339 | #### 示例 340 | 下载文件 341 | ```bash 342 | upx get /baima_text_auditer.tar 343 | ``` 344 | 345 | 下载文件时指定保存路径 346 | ```bash 347 | upx get /baima_text_auditer.tar ./baima_text_auditer2.tar 348 | ``` 349 | 350 | 多线程下载文件 351 | ```bash 352 | upx get -w 10 /baima_text_auditer.tar 353 | ``` 354 | 355 | 恢复中断的下载 356 | ```bash 357 | upx get -c /baima_text_auditer.tar 358 | ``` 359 | 360 | ## put 361 | > 上传文件或文件夹 362 | 363 | | args | 说明 | 364 | | --------- | ---- | 365 | | local-file | 本地的文件或文件夹 | 366 | | url | 远端 url 文件 | 367 | | remote-file | 需要保存到的远程文件路径或文件夹 | 368 | 369 | | options | 说明 | 370 | |---------|-----------------------------| 371 | | -w | 多线程下载 (1-10) (default: 5) | 372 | | -all | 上传包含目录下隐藏的文件和文件夹 | 373 | #### 语法 374 | ```bash 375 | upx put | [remote-file] 376 | ``` 377 | 378 | #### 示例 379 | 上传本地文件，到远程绝对路径 380 | ```bash 381 | upx put aaa.mp4 /video/aaa.mp4 382 | ``` 383 | 384 | 上传本地目录，到远程绝对路径 385 | ```bash 386 | upx put ./video /myfiles 387 | ``` 388 | 389 | 上传 url 文件，到远程绝对路径 390 | ```bash 391 | upx put https://xxxx.com/myfile.tar.gz /myfiles 392 | ``` 393 | 394 | 保存上传文件的错误日志 395 | ```bash 396 | upx put . --err-log=err.log 397 | ``` 398 | 399 | ### 上传大文件的过程中同时下载 400 | 401 | ``` 402 | upx put --in-progress ./file_1G ./inprogress/file_1G 403 | ``` 404 | 405 | ``` 406 | upx get --in-progress /inprogress/file_1G inprogress/file_1G 407 | ``` 408 | 409 | 410 | 411 | ## upload 412 | > 上传文件或目录，支持多文件，文件名匹配 413 | 414 | | args | 说明 | 415 | | --------- | ---- | 416 | | local-file | 本地的文件或文件夹, 或匹配文件规则 | 417 | | remote-path | 需要保存到的远程文件路径 | 418 | 419 | | options | 说明 | 420 | |---------|-----------------------------| 421 | | -w | 多线程下载 (1-10) (default: 5) | 422 | | -all | 上传包含目录下隐藏的文件和文件夹 | 423 | | --remote | 远程路径 | 424 | 425 | #### 语法 426 | ```bash 427 | upx upload [--remote remote-path] 428 | ``` 429 | 430 | #### 示例 431 | 432 | 上传当前路径下的所有 `jpg` 图片到 `/images` 目录 433 | ``` 434 | upx upload --remote /images ./*.jpg 435 | ``` 436 | 437 | 保存上传文件的错误日志 438 | ```bash 439 | upx upload . --err-log=1.log 440 | ``` 441 | 442 | ## rm 443 | 444 | > 默认不会删除目录，支持通配符 `*` 445 | 446 | | args | 说明 | 447 | | --------- | ---- | 448 | | remote-file | 远程文件 | 449 | 450 | | options | 说明 | 451 | | --------- | ---- | 452 | | -d | 仅删除目录 | 453 | | -a | 删除目录跟文件 | 454 | | --async | 异步删除，目录可能需要二次删除 | 455 | 456 | #### 语法 457 | ```bash 458 | upx rm [options] 459 | ``` 460 | 461 | #### 示例 462 | 删除目录 `/www` 463 | ```bash 464 | upx rm -d /www 465 | ``` 466 | 467 | 删除文件 `/aaa.png` 468 | ```bash 469 | upx rm /aaa.png 470 | ``` 471 | 472 | ## mv 473 | 474 | > 在 `bucket` 内部移动文件 475 | 476 | | args | 说明 | 477 | | --------- | ---- | 478 | | source-file | 需要移动的源文件 | 479 | | dest-file | 需要移动到的目标文件 | 480 | 481 | | options | 说明 | 482 | | --------- | ---- | 483 | | -f | 允许覆盖目标文件 | 484 | 485 | #### 语法 486 | ```bash 487 | upx mv [options] 488 | ``` 489 | 490 | #### 示例 491 | 移动文件 492 | ```bash 493 | upx mv /aaa.mp4 /abc/aaa.mp4 494 | ``` 495 | 496 | 移动文件，如果目标存在则强制覆盖 497 | ```bash 498 | upx mv -f /aaa.mp4 /abc/aaa.mp4 499 | ``` 500 | 501 | ## cp 502 | 503 | > 在 `bucket` 内部拷贝文件 504 | 505 | | args | 说明 | 506 | | --------- | ---- | 507 | | source-file | 需要复制的源文件 | 508 | | dest-file | 需要复制到的目标文件 | 509 | 510 | | options | 说明 | 511 | | --------- | ---- | 512 | | -f | 允许覆盖目标文件 | 513 | 514 | #### 语法 515 | ```bash 516 | upx mv [options] 517 | ``` 518 | 519 | #### 示例 520 | 移动文件 521 | ```bash 522 | upx cp /aaa.mp4 /abc/aaa.mp4 523 | ``` 524 | 525 | 复制文件，如果目标存在则强制覆盖 526 | ```bash 527 | upx cp -f /aaa.mp4 /abc/aaa.mp4 528 | ``` 529 | 530 | ## sync 531 | 532 | > sync 本地路径 存储路径 533 | 534 | | args | 说明 | 535 | | --------- | ---- | 536 | | local-path | 本地的路径 | 537 | | remote-path | 远程文件路径 | 538 | 539 | | options | 说明 | 540 | | -------- | ---- | 541 | | -w | 指定并发数，默认为 5 | 542 | | --delete | 删除上一次同步后本地删除的文件 | 543 | 544 | #### 语法 545 | ```bash 546 | upx sync 547 | ``` 548 | 549 | #### 示例 550 | 同步本地路径和远程路径 551 | ```bash 552 | upx sync ./workspace /workspace 553 | ``` 554 | 555 | ## auth 556 | 557 | > 生成包含空间名操作员密码信息, auth 空间名 操作员 密码 558 | 559 | #### 示例 560 | 当命令中包含 `--auth` 参数时，会忽略已登陆的信息。 561 | 562 | 生成 auth 字符串 563 | ```bash 564 | upx auth mybucket user password 565 | ``` 566 | 567 | 通过生成的 auth 字符串上传文件 568 | ```bash 569 | upx --auth=auth-string put temp.file 570 | ``` 571 | 572 | 573 | ## post 574 | 575 | | options | 说明 | 576 | | -------------- | ---- | 577 | | --app value | app 名称 | 578 | | --notify value | 回调地址 | 579 | | --task value | 任务文件名 | 580 | 581 | ## purge 582 | 583 | > purge url --list urls 584 | 585 | | options | 说明 | 586 | | -------------- | ---- | 587 | | --list value | 批量刷新文件名 | 588 | 589 | 590 | ## TODO 591 | 592 | - [x] put 支持断点续传 593 | - [ ] upx 支持指定 API 地址 594 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | v0.4.9 2 | -------------------------------------------------------------------------------- /basic_test.go: -------------------------------------------------------------------------------- 1 | package upx 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestLoginAndLogout(t *testing.T) { 13 | 14 | b, err := Upx("login", BUCKET_1, USERNAME, PASSWORD) 15 | assert.NoError(t, err) 16 | assert.Equal(t, string(b), fmt.Sprintf("Welcome to %s, %s!\n", BUCKET_1, USERNAME)) 17 | 18 | b, err = Upx("login", BUCKET_2, USERNAME, PASSWORD) 19 | assert.NoError(t, err) 20 | assert.Equal(t, string(b), fmt.Sprintf("Welcome to %s, %s!\n", BUCKET_2, USERNAME)) 21 | 22 | b, err = Upx("logout") 23 | assert.NoError(t, err) 24 | assert.Equal(t, string(b), fmt.Sprintf("Goodbye %s/%s ~~\n", USERNAME, BUCKET_2)) 25 | 26 | b, err = Upx("logout") 27 | assert.NoError(t, err) 28 | assert.Equal(t, string(b), fmt.Sprintf("Goodbye %s/%s ~~\n", USERNAME, BUCKET_1)) 29 | } 30 | 31 | func TestGetInfo(t *testing.T) { 32 | SetUp() 33 | defer TearDown() 34 | pwd, _ := Upx("pwd") 35 | b, err := Upx("info") 36 | assert.NoError(t, err) 37 | s := []string{ 38 | "ServiceName: " + BUCKET_1, 39 | "Operator: " + USERNAME, 40 | "CurrentDir: " + strings.TrimRight(string(pwd), "\n"), 41 | "Usage: ", 42 | } 43 | assert.Equal(t, strings.HasPrefix(string(b), strings.Join(s, "\n")), true) 44 | } 45 | 46 | func TestSessionsAndSwitch(t *testing.T) { 47 | SetUp() 48 | defer TearDown() 49 | b, err := Upx("sessions") 50 | assert.NoError(t, err) 51 | assert.Equal(t, string(b), fmt.Sprintf("> %s\n", BUCKET_1)) 52 | 53 | Upx("login", BUCKET_2, USERNAME, PASSWORD) 54 | b, err = Upx("sessions") 55 | assert.NoError(t, err) 56 | assert.Equal(t, string(b), fmt.Sprintf(" %s\n> %s\n", BUCKET_1, BUCKET_2)) 57 | 58 | Upx("switch", BUCKET_1) 59 | b, err = Upx("sessions") 60 | assert.NoError(t, err) 61 | assert.Equal(t, string(b), fmt.Sprintf("> %s\n %s\n", BUCKET_1, BUCKET_2)) 62 | 63 | pwd, _ := Upx("pwd") 64 | b, err = Upx("info") 65 | assert.NoError(t, err) 66 | s := []string{ 67 | "ServiceName: " + BUCKET_1, 68 | "Operator: " + USERNAME, 69 | "CurrentDir: " + strings.TrimRight(string(pwd), "\n"), 70 | "Usage: ", 71 | } 72 | assert.Equal(t, strings.HasPrefix(string(b), strings.Join(s, "\n")), true) 73 | } 74 | 75 | // TODO 76 | func TestAuth(t *testing.T) { 77 | } 78 | 79 | func TestPurge(t *testing.T) { 80 | SetUp() 81 | defer TearDown() 82 | b, err := Upx("purge", fmt.Sprintf("http://%s.b0.upaiyun.com/test.jpg", BUCKET_1)) 83 | assert.NoError(t, err) 84 | assert.Equal(t, len(b), 0) 85 | 86 | _, err = Upx("purge", "http://www.baidu.com") 87 | if !assert.Error(t, err) { 88 | assert.Fail(t, "purge not has return error") 89 | } 90 | assert.Equal(t, err.Error(), "Purge failed urls:\nhttp://www.baidu.com\ntoo many fails\n") 91 | 92 | fd, _ := os.Create("list") 93 | fd.WriteString(fmt.Sprintf("http://%s.b0.upaiyun.com/test.jpg\n", BUCKET_1)) 94 | fd.WriteString(fmt.Sprintf("http://%s.b0.upaiyun.com/测试.jpg\n", BUCKET_1)) 95 | fd.WriteString(fmt.Sprintf("http://%s.b0.upaiyun.com/%%E5%%8F%%88%%E6%%8B%%8D%%E4%%BA%%91.jpg\n", BUCKET_1)) 96 | fd.Close() 97 | 98 | b, err = Upx("purge", "--list", "list") 99 | assert.NoError(t, err) 100 | assert.Equal(t, len(b), 0) 101 | } 102 | -------------------------------------------------------------------------------- /cmd/upx/upx.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/upyun/upx" 7 | "github.com/upyun/upx/processbar" 8 | ) 9 | 10 | func main() { 11 | if upx.IsVerbose { 12 | processbar.ProcessBar.Enable() 13 | defer processbar.ProcessBar.Wait() 14 | } 15 | upx.CreateUpxApp().Run(os.Args) 16 | } 17 | -------------------------------------------------------------------------------- /commands.go: -------------------------------------------------------------------------------- 1 | package upx 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "log" 8 | "os" 9 | "path" 10 | "path/filepath" 11 | "strings" 12 | 13 | "github.com/fatih/color" 14 | "github.com/upyun/upx/xerrors" 15 | "github.com/urfave/cli" 16 | "golang.org/x/term" 17 | ) 18 | 19 | const ( 20 | NO_CHECK = false 21 | CHECK = true 22 | ) 23 | 24 | func InitAndCheck(login, check bool, c *cli.Context) (err error) { 25 | if login == LOGIN && session == nil { 26 | err = readConfigFromFile(LOGIN) 27 | } 28 | if login == NO_LOGIN { 29 | err = readConfigFromFile(NO_LOGIN) 30 | } 31 | 32 | if check && c.NArg() == 0 && c.NumFlags() == 0 { 33 | err = xerrors.ErrInvalidCommand 34 | } 35 | return 36 | } 37 | 38 | func CreateInitCheckFunc(login, check bool) cli.BeforeFunc { 39 | return func(ctx *cli.Context) error { 40 | if err := InitAndCheck(login, check, ctx); err != nil { 41 | if errors.Is(err, xerrors.ErrInvalidCommand) { 42 | cli.ShowCommandHelp(ctx, ctx.Command.Name) 43 | return &cli.ExitError{} 44 | } 45 | return cli.NewExitError(err.Error(), -1) 46 | } 47 | return nil 48 | } 49 | } 50 | 51 | func NewLoginCommand() cli.Command { 52 | return cli.Command{ 53 | Name: "login", 54 | Usage: "Log in to UpYun", 55 | Before: CreateInitCheckFunc(NO_LOGIN, NO_CHECK), 56 | Action: func(c *cli.Context) error { 57 | session = &Session{CWD: "/"} 58 | args := c.Args() 59 | if len(args) == 3 { 60 | session.Bucket = args.Get(0) 61 | session.Operator = args.Get(1) 62 | session.Password = args.Get(2) 63 | } else { 64 | fmt.Printf("ServiceName: ") 65 | fmt.Scanf("%s\n", &session.Bucket) 66 | fmt.Printf("Operator: ") 67 | fmt.Scanf("%s\n", &session.Operator) 68 | fmt.Printf("Password: ") 69 | b, err := term.ReadPassword(int(os.Stdin.Fd())) 70 | if err == nil { 71 | session.Password = string(b) 72 | } 73 | // TODO 74 | Print("") 75 | } 76 | 77 | if err := session.Init(); err != nil { 78 | PrintErrorAndExit("login failed: %v", err) 79 | } 80 | Print("Welcome to %s, %s!", session.Bucket, session.Operator) 81 | 82 | if config == nil { 83 | config = &Config{ 84 | SessionId: 0, 85 | Sessions: []*Session{session}, 86 | } 87 | } else { 88 | config.Insert(session) 89 | } 90 | saveConfigToFile() 91 | 92 | return nil 93 | }, 94 | } 95 | } 96 | 97 | func NewLogoutCommand() cli.Command { 98 | return cli.Command{ 99 | Name: "logout", 100 | Usage: "Log out of your UpYun account", 101 | Before: CreateInitCheckFunc(NO_LOGIN, NO_CHECK), 102 | Action: func(c *cli.Context) error { 103 | if session != nil { 104 | op, bucket := session.Operator, session.Bucket 105 | config.PopCurrent() 106 | saveConfigToFile() 107 | Print("Goodbye %s/%s ~~", op, bucket) 108 | } else { 109 | PrintErrorAndExit("nothing to do") 110 | } 111 | return nil 112 | }, 113 | } 114 | } 115 | 116 | func NewAuthCommand() cli.Command { 117 | return cli.Command{ 118 | Name: "auth", 119 | Usage: "Generate auth string", 120 | Action: func(c *cli.Context) error { 121 | if c.NArg() == 3 { 122 | s, err := makeAuthStr(c.Args()[0], c.Args()[1], c.Args()[2]) 123 | if err != nil { 124 | PrintErrorAndExit("auth: %v", err) 125 | } 126 | Print(s) 127 | } else { 128 | PrintErrorAndExit("auth: invalid parameters") 129 | } 130 | return nil 131 | }, 132 | } 133 | } 134 | 135 | func NewListSessionsCommand() cli.Command { 136 | return cli.Command{ 137 | Name: "sessions", 138 | Usage: "List all sessions", 139 | Before: CreateInitCheckFunc(NO_LOGIN, NO_CHECK), 140 | Action: func(c *cli.Context) error { 141 | for k, v := range config.Sessions { 142 | if k == config.SessionId { 143 | Print("> %s", color.YellowString(v.Bucket)) 144 | } else { 145 | Print(" %s", v.Bucket) 146 | } 147 | } 148 | return nil 149 | }, 150 | } 151 | } 152 | 153 | func NewSwitchSessionCommand() cli.Command { 154 | return cli.Command{ 155 | Name: "switch", 156 | Usage: "Switch to specific session", 157 | Before: CreateInitCheckFunc(NO_LOGIN, CHECK), 158 | Action: func(c *cli.Context) error { 159 | bucket := c.Args().First() 160 | for k, v := range config.Sessions { 161 | if bucket == v.Bucket { 162 | session = v 163 | config.SessionId = k 164 | saveConfigToFile() 165 | Print("Welcome to %s, %s!", session.Bucket, session.Operator) 166 | return nil 167 | } 168 | } 169 | PrintErrorAndExit("switch %s: No such session", bucket) 170 | return nil 171 | }, 172 | } 173 | } 174 | 175 | func NewInfoCommand() cli.Command { 176 | return cli.Command{ 177 | Name: "info", 178 | Usage: "Current session information", 179 | Before: CreateInitCheckFunc(LOGIN, NO_CHECK), 180 | Action: func(c *cli.Context) error { 181 | session.Info() 182 | return nil 183 | }, 184 | } 185 | } 186 | 187 | func NewMkdirCommand() cli.Command { 188 | return cli.Command{ 189 | Name: "mkdir", 190 | Usage: "Make directory", 191 | ArgsUsage: "", 192 | Before: CreateInitCheckFunc(LOGIN, CHECK), 193 | Action: func(c *cli.Context) error { 194 | session.Mkdir(c.Args()...) 195 | return nil 196 | }, 197 | } 198 | } 199 | 200 | func NewCdCommand() cli.Command { 201 | return cli.Command{ 202 | Name: "cd", 203 | Usage: "Change directory", 204 | ArgsUsage: "", 205 | Before: CreateInitCheckFunc(LOGIN, NO_CHECK), 206 | Action: func(c *cli.Context) error { 207 | fpath := "/" 208 | if c.NArg() > 0 { 209 | fpath = c.Args().First() 210 | } 211 | session.Cd(fpath) 212 | saveConfigToFile() 213 | return nil 214 | }, 215 | } 216 | } 217 | 218 | func NewPwdCommand() cli.Command { 219 | return cli.Command{ 220 | Name: "pwd", 221 | Usage: "Print working directory", 222 | Before: CreateInitCheckFunc(LOGIN, NO_CHECK), 223 | Action: func(c *cli.Context) error { 224 | session.Pwd() 225 | return nil 226 | }, 227 | } 228 | } 229 | 230 | func NewLsCommand() cli.Command { 231 | return cli.Command{ 232 | Name: "ls", 233 | Usage: "List directory or file", 234 | ArgsUsage: "", 235 | Before: CreateInitCheckFunc(LOGIN, NO_CHECK), 236 | Action: func(c *cli.Context) error { 237 | fpath := session.CWD 238 | if c.NArg() > 0 { 239 | fpath = c.Args().First() 240 | } 241 | mc := &MatchConfig{} 242 | if c.Bool("d") { 243 | mc.ItemType = DIR 244 | } 245 | base := path.Base(fpath) 246 | dir := path.Dir(fpath) 247 | if strings.Contains(base, "*") { 248 | mc.Wildcard = base 249 | fpath = dir 250 | } 251 | if c.String("mtime") != "" { 252 | err := parseMTime(c.String("mtime"), mc) 253 | if err != nil { 254 | PrintErrorAndExit("ls %s: parse mtime: %v", fpath, err) 255 | } 256 | } 257 | session.color = c.Bool("color") 258 | session.Ls(fpath, mc, c.Int("c"), c.Bool("r")) 259 | return nil 260 | }, 261 | Flags: []cli.Flag{ 262 | cli.BoolFlag{Name: "r", Usage: "reverse order"}, 263 | cli.BoolFlag{Name: "d", Usage: "only show directory"}, 264 | cli.BoolFlag{Name: "color", Usage: "colorful output"}, 265 | cli.IntFlag{Name: "c", Usage: "max items to list"}, 266 | cli.StringFlag{Name: "mtime", Usage: "file's data was last modified n*24 hours ago, same as linux find command."}, 267 | }, 268 | } 269 | } 270 | 271 | func NewGetCommand() cli.Command { 272 | return cli.Command{ 273 | Name: "get", 274 | Usage: "Get directory or file", 275 | ArgsUsage: "[-c] [save-path]", 276 | Before: CreateInitCheckFunc(LOGIN, CHECK), 277 | Action: func(c *cli.Context) error { 278 | upPath := c.Args().First() 279 | localPath := "." + string(filepath.Separator) 280 | 281 | if c.NArg() > 2 { 282 | PrintErrorAndExit("upx get args limit 2") 283 | } 284 | if c.NArg() > 1 { 285 | localPath = c.Args().Get(1) 286 | } 287 | 288 | mc := &MatchConfig{} 289 | base := path.Base(upPath) 290 | dir := path.Dir(upPath) 291 | if strings.Contains(base, "*") { 292 | mc.Wildcard, upPath = base, dir 293 | } 294 | if c.String("start") != "" { 295 | mc.Start = c.String("start") 296 | } 297 | if c.String("end") != "" { 298 | mc.End = c.String("end") 299 | } 300 | if c.String("mtime") != "" { 301 | err := parseMTime(c.String("mtime"), mc) 302 | if err != nil { 303 | PrintErrorAndExit("get %s: parse mtime: %v", upPath, err) 304 | } 305 | } 306 | if c.Int("w") > 10 || c.Int("w") < 1 { 307 | PrintErrorAndExit("max concurrent threads must between (1 - 10)") 308 | } 309 | if mc.Start != "" || mc.End != "" { 310 | if c.Bool("in-progress") { 311 | PrintErrorAndExit("get %s: --in-progress and -start/-end can't be used together", upPath) 312 | } 313 | session.GetStartBetweenEndFiles(upPath, localPath, mc, c.Int("w")) 314 | } else { 315 | session.Get(upPath, localPath, mc, c.Int("w"), c.Bool("c"), c.Bool("in-progress")) 316 | } 317 | return nil 318 | }, 319 | Flags: []cli.Flag{ 320 | cli.IntFlag{Name: "w", Usage: "max concurrent threads (1-10)", Value: 5}, 321 | cli.BoolFlag{Name: "c", Usage: "continue download, Resume Broken Download"}, 322 | cli.BoolFlag{Name: "in-progress", Usage: "download the file being uploaded"}, 323 | cli.StringFlag{Name: "mtime", Usage: "file's data was last modified n*24 hours ago, same as linux find command."}, 324 | cli.StringFlag{Name: "start", Usage: "file download range starting location"}, 325 | cli.StringFlag{Name: "end", Usage: "file download range ending location"}, 326 | }, 327 | } 328 | } 329 | 330 | func NewPutCommand() cli.Command { 331 | return cli.Command{ 332 | Name: "put", 333 | Usage: "Put directory or file", 334 | ArgsUsage: "| [remote-path]", 335 | Before: CreateInitCheckFunc(LOGIN, CHECK), 336 | Action: func(c *cli.Context) error { 337 | localPath := c.Args().First() 338 | upPath := "./" 339 | 340 | if c.NArg() > 2 { 341 | fmt.Println("Use the upload command instead of the put command for multiple file uploads") 342 | os.Exit(0) 343 | } 344 | 345 | if c.NArg() > 1 { 346 | upPath = c.Args().Get(1) 347 | } 348 | if c.Int("w") > 10 || c.Int("w") < 1 { 349 | PrintErrorAndExit("max concurrent threads must between (1 - 10)") 350 | } 351 | errLog := c.String("err-log") 352 | if errLog != "" { 353 | f, err := os.OpenFile(errLog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 354 | if err != nil { 355 | PrintErrorAndExit("open error log file: %v", err) 356 | } 357 | defer f.Close() 358 | log.SetOutput(f) 359 | } 360 | session.Put( 361 | localPath, 362 | upPath, 363 | c.Int("w"), 364 | c.Bool("all"), 365 | c.Bool("in-progress"), 366 | ) 367 | return nil 368 | }, 369 | Flags: []cli.Flag{ 370 | cli.IntFlag{Name: "w", Usage: "max concurrent threads", Value: 5}, 371 | cli.BoolFlag{Name: "in-progress", Usage: "upload a file that can be downloaded simultaneously"}, 372 | cli.BoolFlag{Name: "all", Usage: "upload all files including hidden files"}, 373 | cli.StringFlag{Name: "err-log", Usage: "upload file error log to file"}, 374 | }, 375 | } 376 | } 377 | 378 | func NewUploadCommand() cli.Command { 379 | return cli.Command{ 380 | Name: "upload", 381 | Usage: "upload multiple directory or file", 382 | ArgsUsage: "[local-path...] [--remote remote-path]", 383 | Before: CreateInitCheckFunc(LOGIN, CHECK), 384 | Action: func(c *cli.Context) error { 385 | if c.Int("w") > 10 || c.Int("w") < 1 { 386 | PrintErrorAndExit("max concurrent threads must between (1 - 10)") 387 | } 388 | filenames := c.Args() 389 | if isWindowsGOOS() { 390 | filenames = globFiles(filenames) 391 | } 392 | errLog := c.String("err-log") 393 | if errLog != "" { 394 | f, err := os.OpenFile(errLog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 395 | if err != nil { 396 | PrintErrorAndExit("open error log file: %v", err) 397 | } 398 | defer f.Close() 399 | log.SetOutput(f) 400 | } 401 | session.Upload( 402 | filenames, 403 | c.String("remote"), 404 | c.Int("w"), 405 | c.Bool("all"), 406 | ) 407 | return nil 408 | }, 409 | Flags: []cli.Flag{ 410 | cli.BoolFlag{Name: "all", Usage: "upload all files including hidden files"}, 411 | cli.IntFlag{Name: "w", Usage: "max concurrent threads", Value: 5}, 412 | cli.StringFlag{Name: "remote", Usage: "remote path", Value: "./"}, 413 | cli.StringFlag{Name: "err-log", Usage: "upload file error log to file"}, 414 | }, 415 | } 416 | } 417 | 418 | func NewRmCommand() cli.Command { 419 | return cli.Command{ 420 | Name: "rm", 421 | Usage: "Remove directory or file", 422 | ArgsUsage: "", 423 | Before: CreateInitCheckFunc(LOGIN, CHECK), 424 | Action: func(c *cli.Context) error { 425 | fpath := c.Args().First() 426 | base := path.Base(fpath) 427 | dir := path.Dir(fpath) 428 | mc := &MatchConfig{ 429 | ItemType: FILE, 430 | } 431 | if strings.Contains(base, "*") { 432 | mc.Wildcard, fpath = base, dir 433 | } 434 | 435 | if c.Bool("d") { 436 | mc.ItemType = DIR 437 | } 438 | if c.Bool("a") { 439 | mc.ItemType = ITEM_NOT_SET 440 | } 441 | 442 | // if c.String("mtime") != "" { 443 | // err := parseMTime(c.String("mtime"), mc) 444 | // if err != nil { 445 | // PrintErrorAndExit("rm %s: parse mtime: %v", fpath, err) 446 | // } 447 | // } 448 | 449 | session.Rm(fpath, mc, c.Bool("async")) 450 | return nil 451 | }, 452 | Flags: []cli.Flag{ 453 | cli.BoolFlag{Name: "d", Usage: "only remove directories"}, 454 | cli.BoolFlag{Name: "a", Usage: "remove files, directories and their contents recursively, never prompt"}, 455 | cli.BoolFlag{Name: "async", Usage: "remove asynchronously"}, 456 | // cli.StringFlag{Name: "mtime", Usage: "file's data was last modified n*24 hours ago, same as linux find command."}, 457 | }, 458 | } 459 | } 460 | 461 | func NewTreeCommand() cli.Command { 462 | return cli.Command{ 463 | Name: "tree", 464 | Usage: "List contents of directories in a tree-like format", 465 | ArgsUsage: "", 466 | Before: CreateInitCheckFunc(LOGIN, NO_CHECK), 467 | Action: func(c *cli.Context) error { 468 | fpath := session.CWD 469 | if c.NArg() > 0 { 470 | fpath = c.Args().First() 471 | } 472 | session.color = c.Bool("color") 473 | session.Tree(fpath) 474 | return nil 475 | }, 476 | Flags: []cli.Flag{ 477 | cli.BoolFlag{Name: "color", Usage: "colorful output"}, 478 | }, 479 | } 480 | } 481 | 482 | func NewSyncCommand() cli.Command { 483 | return cli.Command{ 484 | Name: "sync", 485 | Usage: "Sync local directory to UpYun", 486 | ArgsUsage: " [remote-path]", 487 | Before: CreateInitCheckFunc(LOGIN, CHECK), 488 | Action: func(c *cli.Context) error { 489 | localPath := c.Args().First() 490 | upPath := session.CWD 491 | if c.NArg() > 1 { 492 | upPath = c.Args().Get(1) 493 | } 494 | if c.Int("w") > 10 || c.Int("w") < 1 { 495 | PrintErrorAndExit("max concurrent threads must between (1 - 10)") 496 | } 497 | session.Sync(localPath, upPath, c.Int("w"), c.Bool("delete"), c.Bool("strong")) 498 | return nil 499 | }, 500 | Flags: []cli.Flag{ 501 | cli.IntFlag{Name: "w", Usage: "max concurrent threads", Value: 5}, 502 | cli.BoolFlag{Name: "delete", Usage: "delete extraneous files from last sync"}, 503 | cli.BoolFlag{Name: "strong", Usage: "strong consistency"}, 504 | }, 505 | } 506 | } 507 | 508 | func NewPostCommand() cli.Command { 509 | return cli.Command{ 510 | Name: "post", 511 | Usage: "Post async process task", 512 | Before: CreateInitCheckFunc(LOGIN, CHECK), 513 | Action: func(c *cli.Context) error { 514 | app := c.String("app") 515 | notify := c.String("notify") 516 | task := c.String("task") 517 | session.PostTask(app, notify, task) 518 | return nil 519 | }, 520 | Flags: []cli.Flag{ 521 | cli.StringFlag{Name: "app", Usage: "app name"}, 522 | cli.StringFlag{Name: "notify", Usage: "notify url"}, 523 | cli.StringFlag{Name: "task", Usage: "task file"}, 524 | }, 525 | } 526 | } 527 | 528 | func NewPurgeCommand() cli.Command { 529 | return cli.Command{ 530 | Name: "purge", 531 | Usage: "refresh CDN cache", 532 | Before: CreateInitCheckFunc(LOGIN, CHECK), 533 | Action: func(c *cli.Context) error { 534 | list := c.String("list") 535 | session.Purge(c.Args(), list) 536 | return nil 537 | }, 538 | Flags: []cli.Flag{ 539 | cli.StringFlag{Name: "list", Usage: "file which contains urls"}, 540 | }, 541 | } 542 | } 543 | 544 | func NewGetDBCommand() cli.Command { 545 | return cli.Command{ 546 | Name: "get-db", 547 | Usage: "get db value", 548 | Before: CreateInitCheckFunc(LOGIN, CHECK), 549 | Action: func(c *cli.Context) error { 550 | if c.NArg() != 2 { 551 | PrintErrorAndExit("get-db local remote") 552 | } 553 | if err := initDB(); err != nil { 554 | PrintErrorAndExit("get-db: init database: %v", err) 555 | } 556 | value, err := getDBValue(c.Args()[0], c.Args()[1]) 557 | if err != nil { 558 | PrintErrorAndExit("get-db: %v", err) 559 | } 560 | b, _ := json.MarshalIndent(value, "", " ") 561 | Print("%s", string(b)) 562 | return nil 563 | }, 564 | } 565 | } 566 | 567 | func NewCleanDBCommand() cli.Command { 568 | return cli.Command{ 569 | Name: "clean-db", 570 | Usage: "clean db by local_prefx and remote_prefix", 571 | Before: CreateInitCheckFunc(LOGIN, CHECK), 572 | Action: func(c *cli.Context) error { 573 | if c.NArg() != 2 { 574 | PrintErrorAndExit("clean-db local remote") 575 | } 576 | if err := initDB(); err != nil { 577 | PrintErrorAndExit("clean-db: init database: %v", err) 578 | } 579 | delDBValues(c.Args()[0], c.Args()[1]) 580 | return nil 581 | }, 582 | } 583 | } 584 | 585 | func NewUpgradeCommand() cli.Command { 586 | return cli.Command{ 587 | Name: "upgrade", 588 | Usage: "upgrade upx to latest version", 589 | Action: func(c *cli.Context) error { 590 | Upgrade() 591 | return nil 592 | }, 593 | } 594 | } 595 | 596 | func NewCopyCommand() cli.Command { 597 | return cli.Command{ 598 | Name: "cp", 599 | Usage: "copy files inside cloud storage", 600 | ArgsUsage: "[remote-source-path] [remote-target-path]", 601 | Before: CreateInitCheckFunc(LOGIN, CHECK), 602 | Action: func(c *cli.Context) error { 603 | if c.NArg() != 2 { 604 | PrintErrorAndExit("invalid command args") 605 | } 606 | if err := session.Copy(c.Args()[0], c.Args()[1], c.Bool("f")); err != nil { 607 | PrintErrorAndExit(err.Error()) 608 | } 609 | return nil 610 | }, 611 | Flags: []cli.Flag{ 612 | cli.BoolFlag{Name: "f", Usage: "Force overwrite existing files"}, 613 | }, 614 | } 615 | } 616 | 617 | func NewMoveCommand() cli.Command { 618 | return cli.Command{ 619 | Name: "mv", 620 | Usage: "move files inside cloud storage", 621 | ArgsUsage: "[remote-source-path] [remote-target-path]", 622 | Before: CreateInitCheckFunc(LOGIN, CHECK), 623 | Action: func(c *cli.Context) error { 624 | if c.NArg() != 2 { 625 | PrintErrorAndExit("invalid command args") 626 | } 627 | if err := session.Move(c.Args()[0], c.Args()[1], c.Bool("f")); err != nil { 628 | PrintErrorAndExit(err.Error()) 629 | } 630 | return nil 631 | }, 632 | Flags: []cli.Flag{ 633 | cli.BoolFlag{Name: "f", Usage: "Force overwrite existing files"}, 634 | }, 635 | } 636 | } 637 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package upx 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "os" 9 | "path/filepath" 10 | "runtime" 11 | 12 | "github.com/upyun/upx/xerrors" 13 | ) 14 | 15 | const ( 16 | LOGIN = true 17 | NO_LOGIN = false 18 | MinJitter = 1 19 | MaxJitter = 5 20 | MaxRetry = 5 21 | ) 22 | 23 | type Config struct { 24 | SessionId int `json:"user_idx"` 25 | Sessions []*Session `json:"users"` 26 | } 27 | 28 | func (c *Config) PopCurrent() { 29 | if c.SessionId == -1 { 30 | c.SessionId = 0 31 | } 32 | 33 | c.Sessions = append(c.Sessions[0:c.SessionId], c.Sessions[c.SessionId+1:]...) 34 | c.SessionId = 0 35 | } 36 | 37 | func (c *Config) Insert(sess *Session) { 38 | for idx, s := range c.Sessions { 39 | if s.Bucket == sess.Bucket && s.Operator == sess.Operator { 40 | c.Sessions[idx] = sess 41 | c.SessionId = idx 42 | return 43 | } 44 | } 45 | c.Sessions = append(c.Sessions, sess) 46 | c.SessionId = len(c.Sessions) - 1 47 | } 48 | 49 | var ( 50 | confname string 51 | config *Config 52 | ) 53 | 54 | func makeAuthStr(bucket, operator, password string) (string, error) { 55 | sess := &Session{ 56 | Bucket: bucket, 57 | Operator: operator, 58 | Password: password, 59 | CWD: "/", 60 | } 61 | if err := sess.Init(); err != nil { 62 | return "", err 63 | } 64 | 65 | s := []string{bucket, operator, password} 66 | 67 | b, err := json.Marshal(s) 68 | if err != nil { 69 | return "", err 70 | } 71 | return hashEncode(base64.StdEncoding.EncodeToString(b)), nil 72 | } 73 | 74 | func authStrToConfig(auth string) error { 75 | data, err := base64.StdEncoding.DecodeString(hashEncode(auth)) 76 | if err != nil { 77 | return err 78 | } 79 | ss := []string{} 80 | if err := json.Unmarshal(data, &ss); err != nil { 81 | return err 82 | } 83 | if len(ss) == 3 { 84 | session = &Session{ 85 | Bucket: ss[0], 86 | Operator: ss[1], 87 | Password: ss[2], 88 | CWD: "/", 89 | } 90 | if err := session.Init(); err != nil { 91 | return err 92 | } 93 | } else { 94 | return fmt.Errorf("invalid auth string") 95 | } 96 | return nil 97 | } 98 | 99 | func readConfigFromFile(login bool) error { 100 | if confname == "" { 101 | confname = getConfigName() 102 | } 103 | 104 | b, err := ioutil.ReadFile(confname) 105 | if err != nil { 106 | os.RemoveAll(confname) 107 | if os.IsNotExist(err) && login == NO_LOGIN { 108 | return nil 109 | } 110 | return err 111 | } 112 | 113 | data, err := base64.StdEncoding.DecodeString(hashEncode(string(b))) 114 | if err != nil { 115 | os.RemoveAll(confname) 116 | return err 117 | } 118 | 119 | config = &Config{SessionId: -1} 120 | if err := json.Unmarshal(data, config); err != nil { 121 | os.RemoveAll(confname) 122 | return err 123 | } 124 | 125 | if config.SessionId != -1 && config.SessionId < len(config.Sessions) { 126 | session = config.Sessions[config.SessionId] 127 | if login == LOGIN { 128 | if err := session.Init(); err != nil { 129 | config.PopCurrent() 130 | return err 131 | } 132 | } 133 | } else { 134 | if login == LOGIN { 135 | return xerrors.ErrRequireLogin 136 | } 137 | } 138 | return nil 139 | } 140 | 141 | func saveConfigToFile() { 142 | if confname == "" { 143 | confname = getConfigName() 144 | } 145 | 146 | b, err := json.Marshal(config) 147 | if err != nil { 148 | PrintErrorAndExit("save config: %v", err) 149 | } 150 | s := hashEncode(base64.StdEncoding.EncodeToString(b)) 151 | 152 | fd, err := os.Create(confname) 153 | if err != nil { 154 | PrintErrorAndExit("save config: %v", err) 155 | } 156 | defer fd.Close() 157 | _, err = fd.WriteString(s) 158 | 159 | if err != nil { 160 | PrintErrorAndExit("save config: %v", err) 161 | } 162 | } 163 | 164 | func getConfigName() string { 165 | if runtime.GOOS == "windows" { 166 | return filepath.Join(os.Getenv("USERPROFILE"), ".upx.cfg") 167 | } 168 | return filepath.Join(os.Getenv("HOME"), ".upx.cfg") 169 | } 170 | 171 | func hashEncode(s string) string { 172 | r := []rune(s) 173 | for i := 0; i < len(r); i++ { 174 | switch { 175 | case r[i] >= 'a' && r[i] <= 'z': 176 | r[i] += 'A' - 'a' 177 | case r[i] >= 'A' && r[i] <= 'Z': 178 | r[i] += 'a' - 'A' 179 | case r[i] >= '0' && r[i] <= '9': 180 | r[i] = '9' - r[i] + '0' 181 | } 182 | } 183 | return string(r) 184 | } 185 | -------------------------------------------------------------------------------- /copy_test.go: -------------------------------------------------------------------------------- 1 | package upx 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "path" 7 | "path/filepath" 8 | "testing" 9 | "time" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestCopy(t *testing.T) { 15 | SetUp() 16 | defer TearDown() 17 | 18 | upRootPath := path.Join(ROOT, "copy") 19 | Upx("mkdir", upRootPath) 20 | 21 | localRootPath, err := ioutil.TempDir("", "test") 22 | assert.NoError(t, err) 23 | localRootName := filepath.Base(localRootPath) 24 | 25 | CreateFile(path.Join(localRootPath, "FILE1")) 26 | CreateFile(path.Join(localRootPath, "FILE2")) 27 | 28 | // 上传文件 29 | _, err = Upx("put", localRootPath, upRootPath) 30 | assert.NoError(t, err) 31 | 32 | files, err := Ls(path.Join(upRootPath, localRootName)) 33 | assert.NoError(t, err) 34 | assert.Len(t, files, 2) 35 | assert.ElementsMatch( 36 | t, 37 | files, 38 | []string{"FILE1", "FILE2"}, 39 | ) 40 | 41 | time.Sleep(time.Second) 42 | 43 | // 正常复制文件 44 | _, err = Upx( 45 | "cp", 46 | path.Join(upRootPath, localRootName, "FILE1"), 47 | path.Join(upRootPath, localRootName, "FILE3"), 48 | ) 49 | assert.NoError(t, err) 50 | 51 | files, err = Ls(path.Join(upRootPath, localRootName)) 52 | assert.NoError(t, err) 53 | assert.Len(t, files, 3) 54 | assert.ElementsMatch( 55 | t, 56 | files, 57 | []string{"FILE1", "FILE2", "FILE3"}, 58 | ) 59 | 60 | time.Sleep(time.Second) 61 | 62 | // 目标文件已存在 63 | _, err = Upx( 64 | "cp", 65 | path.Join(upRootPath, localRootName, "FILE1"), 66 | path.Join(upRootPath, localRootName, "FILE2"), 67 | ) 68 | assert.Error(t, err) 69 | assert.Equal( 70 | t, 71 | err.Error(), 72 | fmt.Sprintf( 73 | "target path %s already exists use -f to force overwrite\n", 74 | path.Join(upRootPath, localRootName, "FILE2"), 75 | ), 76 | ) 77 | 78 | files, err = Ls(path.Join(upRootPath, localRootName)) 79 | assert.NoError(t, err) 80 | assert.Len(t, files, 3) 81 | assert.ElementsMatch( 82 | t, 83 | files, 84 | []string{"FILE1", "FILE2", "FILE3"}, 85 | ) 86 | 87 | time.Sleep(time.Second) 88 | 89 | // 目标文件已存在, 强制覆盖 90 | _, err = Upx( 91 | "cp", 92 | "-f", 93 | path.Join(upRootPath, localRootName, "FILE1"), 94 | path.Join(upRootPath, localRootName, "FILE2"), 95 | ) 96 | assert.NoError(t, err) 97 | 98 | files, err = Ls(path.Join(upRootPath, localRootName)) 99 | assert.NoError(t, err) 100 | assert.Len(t, files, 3) 101 | assert.ElementsMatch( 102 | t, 103 | files, 104 | []string{"FILE1", "FILE2", "FILE3"}, 105 | ) 106 | } 107 | -------------------------------------------------------------------------------- /db.go: -------------------------------------------------------------------------------- 1 | package upx 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | "os" 7 | "path" 8 | "path/filepath" 9 | "runtime" 10 | "strings" 11 | 12 | "github.com/syndtr/goleveldb/leveldb" 13 | ) 14 | 15 | var db *leveldb.DB 16 | 17 | type dbKey struct { 18 | SrcPath string `json:"src_path"` 19 | DstPath string `json:"dst_path"` 20 | } 21 | 22 | type fileMeta struct { 23 | Name string `json:"name"` 24 | IsDir bool `json:"isdir"` 25 | } 26 | 27 | type dbValue struct { 28 | ModifyTime int64 `json:"modify_time"` 29 | Md5 string `json:"md5"` 30 | IsDir string `json:"isdir"` 31 | Items []*fileMeta `json:"items"` 32 | } 33 | 34 | func getDBName() string { 35 | if runtime.GOOS == "windows" { 36 | return filepath.Join(os.Getenv("USERPROFILE"), ".upx.db") 37 | } 38 | return filepath.Join(os.Getenv("HOME"), ".upx.db") 39 | } 40 | 41 | func makeDBKey(src, dst string) ([]byte, error) { 42 | return json.Marshal(&dbKey{ 43 | SrcPath: src, 44 | DstPath: path.Join(session.Bucket, dst), 45 | }) 46 | } 47 | 48 | func makeDBValue(filename string, md5 bool) (*dbValue, error) { 49 | finfo, err := os.Stat(filename) 50 | if err != nil { 51 | return nil, err 52 | } 53 | 54 | dbV := &dbValue{ 55 | ModifyTime: finfo.ModTime().UnixNano(), 56 | } 57 | 58 | if !finfo.IsDir() { 59 | if md5 { 60 | md5Str, _ := md5File(filename) 61 | dbV.Md5 = md5Str 62 | } 63 | dbV.IsDir = "false" 64 | } else { 65 | dbV.IsDir = "true" 66 | } 67 | return dbV, nil 68 | } 69 | 70 | func getDBValue(src, dst string) (*dbValue, error) { 71 | key, err := makeDBKey(src, dst) 72 | if err != nil { 73 | return nil, err 74 | } 75 | 76 | raw, err := db.Get(key, nil) 77 | if err != nil { 78 | if err == leveldb.ErrNotFound { 79 | return nil, nil 80 | } 81 | return nil, err 82 | } 83 | 84 | var value dbValue 85 | if err = json.Unmarshal(raw, &value); err != nil { 86 | return nil, err 87 | } 88 | return &value, nil 89 | } 90 | 91 | func setDBValue(src, dst string, v *dbValue) error { 92 | key, err := makeDBKey(src, dst) 93 | if err != nil { 94 | return err 95 | } 96 | 97 | if v == nil { 98 | v, err = makeDBValue(src, true) 99 | if err != nil { 100 | return err 101 | } 102 | } 103 | 104 | b, err := json.Marshal(v) 105 | if err != nil { 106 | return err 107 | } 108 | 109 | return db.Put(key, b, nil) 110 | } 111 | 112 | func delDBValue(src, dst string) error { 113 | key, err := makeDBKey(src, dst) 114 | if err != nil { 115 | return err 116 | } 117 | 118 | return db.Delete(key, nil) 119 | } 120 | 121 | func delDBValues(srcPrefix, dstPrefix string) { 122 | dstPrefix = path.Join(session.Bucket, dstPrefix) 123 | iter := db.NewIterator(nil, nil) 124 | if ok := iter.First(); !ok { 125 | return 126 | } 127 | for { 128 | k := new(dbKey) 129 | key := iter.Key() 130 | err := json.Unmarshal(key, k) 131 | if err != nil { 132 | PrintError("decode %s: %v", string(key), err) 133 | } 134 | if strings.HasPrefix(k.SrcPath, srcPrefix) && strings.HasPrefix(k.DstPath, dstPrefix) { 135 | PrintOnlyVerbose("found %s => %s to delete", k.SrcPath, k.DstPath) 136 | db.Delete(iter.Key(), nil) 137 | } 138 | if ok := iter.Next(); !ok { 139 | break 140 | } 141 | } 142 | } 143 | 144 | func makeFileMetas(dirname string) ([]*fileMeta, error) { 145 | var res []*fileMeta 146 | fInfos, err := ioutil.ReadDir(dirname) 147 | if err != nil { 148 | return res, err 149 | } 150 | for _, fInfo := range fInfos { 151 | fpath := filepath.Join(dirname, fInfo.Name()) 152 | fi, _ := os.Stat(fpath) 153 | if fi != nil && fi.IsDir() { 154 | res = append(res, &fileMeta{fInfo.Name(), true}) 155 | } else { 156 | res = append(res, &fileMeta{fInfo.Name(), false}) 157 | } 158 | } 159 | return res, nil 160 | } 161 | 162 | func diffFileMetas(src []*fileMeta, dst []*fileMeta) []*fileMeta { 163 | i, j := 0, 0 164 | var res []*fileMeta 165 | for i < len(src) && j < len(dst) { 166 | if src[i].Name < dst[j].Name { 167 | res = append(res, src[i]) 168 | i++ 169 | } else if src[i].Name == dst[j].Name { 170 | if src[i].IsDir != dst[j].IsDir { 171 | res = append(res, src[i]) 172 | } 173 | i++ 174 | j++ 175 | } else { 176 | j++ 177 | } 178 | } 179 | 180 | res = append(res, src[i:]...) 181 | return res 182 | } 183 | 184 | func initDB() (err error) { 185 | db, err = leveldb.OpenFile(getDBName(), nil) 186 | if err != nil { 187 | Print("db %v %s", err, getDBName()) 188 | } 189 | return err 190 | } 191 | -------------------------------------------------------------------------------- /fscmds_test.go: -------------------------------------------------------------------------------- 1 | package upx 2 | 3 | import ( 4 | "fmt" 5 | "path" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | /* 13 | mkdir /path/to/mkdir/case1 14 | cd /path/to/mkdir 15 | mkdir case2 16 | cd case2 17 | mkdir ../case3 18 | cd ../case3 19 | ls /path/to/mkdir 20 | */ 21 | func TestMkdirAndCdAndPwd(t *testing.T) { 22 | SetUp() 23 | defer TearDown() 24 | 25 | base := path.Join(ROOT, "mkdir") 26 | 27 | case1 := path.Join(base, "case1") 28 | b, err := Upx("mkdir", case1) 29 | assert.NoError(t, err) 30 | assert.Equal(t, string(b), "") 31 | 32 | Upx("cd", base) 33 | b, _ = Upx("pwd") 34 | assert.Equal(t, string(b), base+"\n") 35 | 36 | case2 := path.Join(base, "case2") 37 | b, err = Upx("mkdir", "case2") 38 | assert.NoError(t, err) 39 | assert.Equal(t, string(b), "") 40 | 41 | Upx("cd", "case2") 42 | b, _ = Upx("pwd") 43 | assert.Equal(t, string(b), case2+"\n") 44 | 45 | case3 := path.Join(base, "case3") 46 | b, err = Upx("mkdir", "../case3") 47 | assert.NoError(t, err) 48 | assert.Equal(t, string(b), "") 49 | 50 | Upx("cd", "../case3") 51 | b, _ = Upx("pwd") 52 | assert.Equal(t, string(b), case3+"\n") 53 | 54 | // check 55 | b, err = Upx("ls", base) 56 | assert.NoError(t, err) 57 | output := string(b) 58 | lines := strings.Split(output, "\n") 59 | assert.Equal(t, len(lines), 4) 60 | assert.Equal(t, strings.Contains(output, " case1\n"), true) 61 | assert.Equal(t, strings.Contains(output, " case2\n"), true) 62 | assert.Equal(t, strings.Contains(output, " case3\n"), true) 63 | } 64 | 65 | /* 66 | ls /path/to/file 67 | ls -r /path/to/dir 68 | ls -c 10 /path/to/dir 69 | ls -d /path/to/dir 70 | ls -r -d -c 10 /path/to/dir 71 | */ 72 | func TestLs(t *testing.T) { 73 | base := ROOT + "/ls" 74 | dirs, files := []string{}, []string{} 75 | func() { 76 | SetUp() 77 | 78 | Upx("mkdir", base) 79 | Upx("cd", base) 80 | for i := 0; i < 11; i++ { 81 | Upx("mkdir", fmt.Sprintf("dir%d", i)) 82 | dirs = append(dirs, fmt.Sprintf("dir%d", i)) 83 | } 84 | 85 | CreateFile("FILE") 86 | for i := 0; i < 5; i++ { 87 | Upx("put", "FILE", fmt.Sprintf("FILE%d", i)) 88 | files = append(files, fmt.Sprintf("FILE%d", i)) 89 | } 90 | }() 91 | 92 | defer func() { 93 | for _, file := range files { 94 | Upx("rm", file) 95 | } 96 | for _, dir := range dirs { 97 | Upx("rm", dir) 98 | } 99 | TearDown() 100 | }() 101 | 102 | b, err := Upx("ls") 103 | assert.NoError(t, err) 104 | assert.Equal(t, len(strings.Split(string(b), "\n")), len(dirs)+len(files)+1) 105 | 106 | normal, err := Upx("ls", base) 107 | assert.NoError(t, err) 108 | assert.Equal(t, len(strings.Split(string(normal), "\n")), len(dirs)+len(files)+1) 109 | 110 | c := (len(dirs) + len(files)) - 1 111 | limited, err := Upx("ls", "-c", fmt.Sprint(c)) 112 | assert.NoError(t, err) 113 | assert.Equal(t, len(strings.Split(string(limited), "\n")), c+1) 114 | 115 | folders, err := Upx("ls", "-d") 116 | assert.NoError(t, err) 117 | assert.Equal(t, len(strings.Split(string(folders), "\n")), len(dirs)+1) 118 | 119 | c = len(dirs) - 1 120 | lfolders, err := Upx("ls", "-d", "-c", fmt.Sprint(c)) 121 | assert.NoError(t, err) 122 | assert.Equal(t, len(strings.Split(string(lfolders), "\n")), c+1) 123 | for _, line := range strings.Split(string(lfolders), "\n")[0:c] { 124 | assert.Equal(t, strings.HasPrefix(line, "drwxrwxrwx "), true) 125 | } 126 | 127 | lfiles, err := Upx("ls", "FILE*") 128 | assert.NoError(t, err) 129 | assert.Equal(t, len(strings.Split(string(lfiles), "\n")), 6) 130 | 131 | reversed, err := Upx("ls", "-r", base) 132 | assert.NoError(t, err) 133 | assert.NotEqual(t, string(reversed), string(normal)) 134 | } 135 | -------------------------------------------------------------------------------- /fsutil/ignore.go: -------------------------------------------------------------------------------- 1 | package fsutil 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | // 判断文件是否是 . 开头的 8 | func hasDotPrefix(filename string) bool { 9 | return strings.HasPrefix(filename, ".") 10 | } 11 | -------------------------------------------------------------------------------- /fsutil/ignore_unix.go: -------------------------------------------------------------------------------- 1 | //go:build linux || darwin 2 | 3 | package fsutil 4 | 5 | import ( 6 | "io/fs" 7 | "path/filepath" 8 | ) 9 | 10 | // 判断文件是否是需要忽略的文件 11 | func IsIgnoreFile(path string, fileInfo fs.FileInfo) bool { 12 | return hasDotPrefix(filepath.Base(path)) 13 | } 14 | -------------------------------------------------------------------------------- /fsutil/ignore_windows.go: -------------------------------------------------------------------------------- 1 | //go:build windows 2 | 3 | package fsutil 4 | 5 | import ( 6 | "io/fs" 7 | "path/filepath" 8 | "syscall" 9 | ) 10 | 11 | // 判断文件是否是需要忽略的文件 12 | func IsIgnoreFile(path string, fileInfo fs.FileInfo) bool { 13 | for hasDotPrefix(filepath.Base(path)) { 14 | return true 15 | } 16 | 17 | underlyingData := fileInfo.Sys().(*syscall.Win32FileAttributeData) 18 | if underlyingData != nil { 19 | return underlyingData.FileAttributes&syscall.FILE_ATTRIBUTE_HIDDEN != 0 20 | } 21 | 22 | return false 23 | } 24 | -------------------------------------------------------------------------------- /get_test.go: -------------------------------------------------------------------------------- 1 | package upx 2 | 3 | import ( 4 | "io/ioutil" 5 | "log" 6 | "os" 7 | "path" 8 | "path/filepath" 9 | "sort" 10 | "strings" 11 | "testing" 12 | 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | func GetStartBetweenEndFiles(t *testing.T, src, dst, correct, start, end string) { 17 | var err error 18 | src = AbsPath(src) 19 | 20 | if start != "" && start[0] != '/' { 21 | start = path.Join(src, start) 22 | } 23 | if end != "" && end[0] != '/' { 24 | end = path.Join(src, end) 25 | } 26 | if dst == "" { 27 | _, err = Upx("get", src, "--start="+start, "--end="+end) 28 | } else { 29 | _, err = Upx("get", src, dst, "--start="+start, "--end="+end) 30 | } 31 | assert.NoError(t, err) 32 | } 33 | 34 | /* 35 | 测试目录 test1:start=123 end=999 test2:start=111 end=666 36 | input: local: local: 37 | 38 | |-- 111 ├── 333 ├── 111 39 | |-- 333 ├── 666 ├── 333 40 | |-- 777 │ ├── 111 └── 444 41 | |-- 444 │ ├── 333 └── 666 42 | ! `-- 666 │ ├── 666 43 | `-- 666 │ └── 777 44 | |-- 111 └── 777 45 | |-- 333 46 | |-- 666 47 | `-- 777 48 | */ 49 | func TestGetStartBetweenEndFiles(t *testing.T) { 50 | nowpath, _ := os.Getwd() 51 | root := strings.Join(strings.Split(ROOT, " "), "-") 52 | base := root + "/get/" 53 | pwd, err := ioutil.TempDir("", "test") 54 | assert.NoError(t, err) 55 | localBase := filepath.Join(pwd, "get") 56 | 57 | func() { 58 | SetUp() 59 | err := os.MkdirAll(localBase, 0755) 60 | assert.NoError(t, err) 61 | }() 62 | defer TearDown() 63 | 64 | err = os.Chdir(localBase) 65 | assert.NoError(t, err) 66 | Upx("mkdir", base) 67 | Upx("cd", base) 68 | 69 | type uploadFiles []struct { 70 | name string 71 | file string 72 | dst string 73 | correct string 74 | } 75 | type uploadDirs []struct { 76 | dir string 77 | dst string 78 | correct string 79 | } 80 | //构造测试目录 81 | files := uploadFiles{ 82 | {name: "111", file: filepath.Join(localBase, "111"), dst: "", correct: path.Join(base, "111")}, 83 | {name: "333", file: filepath.Join(localBase, "333"), dst: "", correct: path.Join(base, "333")}, 84 | {name: "333", file: "333", dst: path.Join(base, "333"), correct: path.Join(base, "333")}, 85 | {name: "777", file: "777", dst: base, correct: path.Join(base, "777")}, 86 | {name: "666", file: "666", dst: base + "/444/", correct: path.Join(base, "444", "666")}, 87 | } 88 | for _, file := range files { 89 | CreateFile(file.name) 90 | putFile(t, file.file, file.dst, file.correct) 91 | } 92 | log.Println(122) 93 | 94 | dirs := uploadDirs{ 95 | {dir: localBase, dst: base + "/666/", correct: base + "/666/"}, 96 | } 97 | for _, dir := range dirs { 98 | putDir(t, dir.dir, dir.dst, dir.correct) 99 | } 100 | 101 | type list struct { 102 | start string 103 | end string 104 | testDir string 105 | } 106 | type test struct { 107 | input list 108 | real []string 109 | want []string 110 | } 111 | //构造测试 112 | tests := []test{ 113 | {input: list{start: "123", end: "999", testDir: filepath.Join(nowpath, "test1")}, real: localFile("test1", base), want: upFile(t, base, "123", "999")}, 114 | {input: list{start: "111", end: "666", testDir: filepath.Join(nowpath, "test2")}, real: localFile("test2", base), want: upFile(t, base, "444", "666")}, 115 | } 116 | for _, tc := range tests { 117 | input := tc.input 118 | 119 | err = os.MkdirAll(input.testDir, os.ModePerm) 120 | if err != nil { 121 | log.Println(err) 122 | } 123 | 124 | GetStartBetweenEndFiles(t, base, input.testDir, input.testDir, input.start, input.end) 125 | 126 | sort.Strings(tc.real) 127 | sort.Strings(tc.want) 128 | assert.Equal(t, len(tc.real), len(tc.want)) 129 | 130 | for i := 0; i < len(tc.real); i++ { 131 | log.Println("compare:", tc.real[i], " ", tc.want[i]) 132 | assert.Equal(t, tc.real[i], tc.want[i]) 133 | } 134 | } 135 | } 136 | 137 | // 递归获取下载到本地的文件 138 | func localFile(local, up string) []string { 139 | var locals []string 140 | localLen := len(local) 141 | fInfos, _ := ioutil.ReadDir(local + "/") 142 | for _, fInfo := range fInfos { 143 | fp := filepath.Join(local, fInfo.Name()) 144 | //使用云存储目录作为前缀方便比较 145 | locals = append(locals, up[:len(up)-1]+fp[localLen:]) 146 | if IsDir(fp) { 147 | localFile(fp, up) 148 | } 149 | } 150 | return locals 151 | } 152 | 153 | // 递归获取云存储目录文件 154 | func upFile(t *testing.T, up, start, end string) []string { 155 | b, err := Upx("ls", up) 156 | assert.NoError(t, err) 157 | 158 | var ups []string 159 | output := strings.TrimRight(string(b), "\n") 160 | for _, line := range strings.Split(output, "\n") { 161 | items := strings.Split(line, " ") 162 | fp := path.Join(up, items[len(items)-1]) 163 | ups = append(ups, fp) 164 | if items[0][0] == 'd' { 165 | upFile(t, fp, start, end) 166 | } 167 | } 168 | 169 | var upfiles []string 170 | for _, file := range ups { 171 | if file >= start && file < end { 172 | upfiles = append(upfiles, file) 173 | } 174 | } 175 | return upfiles 176 | } 177 | 178 | func IsDir(path string) bool { 179 | s, err := os.Stat(path) 180 | if err != nil { 181 | 182 | return false 183 | } 184 | 185 | return s.IsDir() 186 | } 187 | 188 | func AbsPath(upPath string) (ret string) { 189 | if strings.HasPrefix(upPath, "/") { 190 | ret = path.Join(upPath) 191 | } else { 192 | ret = path.Join("/", upPath) 193 | } 194 | if strings.HasSuffix(upPath, "/") && ret != "/" { 195 | ret += "/" 196 | } 197 | return 198 | } 199 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/upyun/upx 2 | 3 | go 1.23.0 4 | 5 | toolchain go1.23.8 6 | 7 | require ( 8 | github.com/fatih/color v1.15.0 9 | github.com/stretchr/testify v1.8.4 10 | github.com/syndtr/goleveldb v1.0.0 11 | github.com/upyun/go-sdk/v3 v3.0.5-0.20241031074256-0e762735b0db 12 | github.com/urfave/cli v1.22.12 13 | github.com/vbauerster/mpb/v8 v8.5.2 14 | golang.org/x/term v0.31.0 15 | ) 16 | 17 | require ( 18 | github.com/VividCortex/ewma v1.2.0 // indirect 19 | github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect 20 | github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect 21 | github.com/davecgh/go-spew v1.1.1 // indirect 22 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect 23 | github.com/kr/pretty v0.2.0 // indirect 24 | github.com/mattn/go-colorable v0.1.13 // indirect 25 | github.com/mattn/go-isatty v0.0.19 // indirect 26 | github.com/mattn/go-runewidth v0.0.14 // indirect 27 | github.com/pmezard/go-difflib v1.0.0 // indirect 28 | github.com/rivo/uniseg v0.4.4 // indirect 29 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 30 | golang.org/x/net v0.39.0 // indirect 31 | golang.org/x/sys v0.32.0 // indirect 32 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect 33 | gopkg.in/yaml.v3 v3.0.1 // indirect 34 | ) 35 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= 2 | github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= 3 | github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= 4 | github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= 5 | github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= 6 | github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= 7 | github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 8 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 10 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 11 | github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= 12 | github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= 13 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 14 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 15 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= 16 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 17 | github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= 18 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 19 | github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= 20 | github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 21 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 22 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 23 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 24 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 25 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 26 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 27 | github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= 28 | github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 29 | github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= 30 | github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 31 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 32 | github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= 33 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 34 | github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= 35 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 36 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 37 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 38 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 39 | github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= 40 | github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= 41 | github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= 42 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 43 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 44 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 45 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 46 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 47 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 48 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 49 | github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= 50 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 51 | github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= 52 | github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= 53 | github.com/upyun/go-sdk/v3 v3.0.5-0.20241031024504-de08aa91940c h1:Wer/AWhipz0U88vbivpDgrlWc0H1MgF5FOHu+rFnbG4= 54 | github.com/upyun/go-sdk/v3 v3.0.5-0.20241031024504-de08aa91940c/go.mod h1:xtmsshnvsTP8h8iqA3+L0NWqFEJc5tUJLGXHVUVzLs8= 55 | github.com/upyun/go-sdk/v3 v3.0.5-0.20241031074256-0e762735b0db h1:DQN6EEJS8lJzsW+IRkBL0NpTyLJMZt3WL16KFH1kzg8= 56 | github.com/upyun/go-sdk/v3 v3.0.5-0.20241031074256-0e762735b0db/go.mod h1:xtmsshnvsTP8h8iqA3+L0NWqFEJc5tUJLGXHVUVzLs8= 57 | github.com/urfave/cli v1.22.12 h1:igJgVw1JdKH+trcLWLeLwZjU9fEfPesQ+9/e4MQ44S8= 58 | github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= 59 | github.com/vbauerster/mpb/v8 v8.5.2 h1:zanzt1cZpSEG5uGNYKcv43+97f0IgEnXpuBFaMxKbM0= 60 | github.com/vbauerster/mpb/v8 v8.5.2/go.mod h1:YqKyR4ZR6Gd34yD3cDHPMmQxc+uUQMwjgO/LkxiJQ6I= 61 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 62 | golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= 63 | golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= 64 | golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= 65 | golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= 66 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 67 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 68 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 69 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 70 | golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= 71 | golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 72 | golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= 73 | golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 74 | golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= 75 | golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= 76 | golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= 77 | golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= 78 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 79 | golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= 80 | golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= 81 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 82 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 83 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 84 | gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= 85 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 86 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 87 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 88 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 89 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 90 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 91 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 92 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 93 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 94 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | # Code generated by godownloader on 2021-01-07T03:49:16Z. DO NOT EDIT. 4 | # 5 | 6 | usage() { 7 | this=$1 8 | cat </dev/null 139 | } 140 | echoerr() { 141 | echo "$@" 1>&2 142 | } 143 | log_prefix() { 144 | echo "$0" 145 | } 146 | _logp=6 147 | log_set_priority() { 148 | _logp="$1" 149 | } 150 | log_priority() { 151 | if test -z "$1"; then 152 | echo "$_logp" 153 | return 154 | fi 155 | [ "$1" -le "$_logp" ] 156 | } 157 | log_tag() { 158 | case $1 in 159 | 0) echo "emerg" ;; 160 | 1) echo "alert" ;; 161 | 2) echo "crit" ;; 162 | 3) echo "err" ;; 163 | 4) echo "warning" ;; 164 | 5) echo "notice" ;; 165 | 6) echo "info" ;; 166 | 7) echo "debug" ;; 167 | *) echo "$1" ;; 168 | esac 169 | } 170 | log_debug() { 171 | log_priority 7 || return 0 172 | echoerr "$(log_prefix)" "$(log_tag 7)" "$@" 173 | } 174 | log_info() { 175 | log_priority 6 || return 0 176 | echoerr "$(log_prefix)" "$(log_tag 6)" "$@" 177 | } 178 | log_err() { 179 | log_priority 3 || return 0 180 | echoerr "$(log_prefix)" "$(log_tag 3)" "$@" 181 | } 182 | log_crit() { 183 | log_priority 2 || return 0 184 | echoerr "$(log_prefix)" "$(log_tag 2)" "$@" 185 | } 186 | uname_os() { 187 | os=$(uname -s | tr '[:upper:]' '[:lower:]') 188 | case "$os" in 189 | cygwin_nt*) os="windows" ;; 190 | mingw*) os="windows" ;; 191 | msys_nt*) os="windows" ;; 192 | esac 193 | echo "$os" 194 | } 195 | uname_arch() { 196 | arch=$(uname -m) 197 | case $arch in 198 | x86_64) arch="amd64" ;; 199 | x86) arch="386" ;; 200 | i686) arch="386" ;; 201 | i386) arch="386" ;; 202 | aarch64) arch="arm64" ;; 203 | armv5*) arch="armv5" ;; 204 | armv6*) arch="armv6" ;; 205 | armv7*) arch="armv7" ;; 206 | esac 207 | echo ${arch} 208 | } 209 | uname_os_check() { 210 | os=$(uname_os) 211 | case "$os" in 212 | darwin) return 0 ;; 213 | dragonfly) return 0 ;; 214 | freebsd) return 0 ;; 215 | linux) return 0 ;; 216 | android) return 0 ;; 217 | nacl) return 0 ;; 218 | netbsd) return 0 ;; 219 | openbsd) return 0 ;; 220 | plan9) return 0 ;; 221 | solaris) return 0 ;; 222 | windows) return 0 ;; 223 | esac 224 | log_crit "uname_os_check '$(uname -s)' got converted to '$os' which is not a GOOS value. Please file bug at https://github.com/client9/shlib" 225 | return 1 226 | } 227 | uname_arch_check() { 228 | arch=$(uname_arch) 229 | case "$arch" in 230 | 386) return 0 ;; 231 | amd64) return 0 ;; 232 | arm64) return 0 ;; 233 | armv5) return 0 ;; 234 | armv6) return 0 ;; 235 | armv7) return 0 ;; 236 | ppc64) return 0 ;; 237 | ppc64le) return 0 ;; 238 | mips) return 0 ;; 239 | mipsle) return 0 ;; 240 | mips64) return 0 ;; 241 | mips64le) return 0 ;; 242 | s390x) return 0 ;; 243 | amd64p32) return 0 ;; 244 | esac 245 | log_crit "uname_arch_check '$(uname -m)' got converted to '$arch' which is not a GOARCH value. Please file bug report at https://github.com/client9/shlib" 246 | return 1 247 | } 248 | untar() { 249 | tarball=$1 250 | case "${tarball}" in 251 | *.tar.gz | *.tgz) tar --no-same-owner -xzf "${tarball}" ;; 252 | *.tar) tar --no-same-owner -xf "${tarball}" ;; 253 | *.zip) unzip "${tarball}" ;; 254 | *) 255 | log_err "untar unknown archive format for ${tarball}" 256 | return 1 257 | ;; 258 | esac 259 | } 260 | http_download_curl() { 261 | local_file=$1 262 | source_url=$2 263 | header=$3 264 | if [ -z "$header" ]; then 265 | code=$(curl -w '%{http_code}' -sL -o "$local_file" "$source_url") 266 | else 267 | code=$(curl -w '%{http_code}' -sL -H "$header" -o "$local_file" "$source_url") 268 | fi 269 | if [ "$code" != "200" ]; then 270 | log_debug "http_download_curl received HTTP status $code" 271 | return 1 272 | fi 273 | return 0 274 | } 275 | http_download_wget() { 276 | local_file=$1 277 | source_url=$2 278 | header=$3 279 | if [ -z "$header" ]; then 280 | wget -q -O "$local_file" "$source_url" 281 | else 282 | wget -q --header "$header" -O "$local_file" "$source_url" 283 | fi 284 | } 285 | http_download() { 286 | log_debug "http_download $2" 287 | if is_command curl; then 288 | http_download_curl "$@" 289 | return 290 | elif is_command wget; then 291 | http_download_wget "$@" 292 | return 293 | fi 294 | log_crit "http_download unable to find wget or curl" 295 | return 1 296 | } 297 | http_copy() { 298 | tmp=$(mktemp) 299 | http_download "${tmp}" "$1" "$2" || return 1 300 | body=$(cat "$tmp") 301 | rm -f "${tmp}" 302 | echo "$body" 303 | } 304 | github_release() { 305 | owner_repo=$1 306 | version=$2 307 | test -z "$version" && version="latest" 308 | giturl="https://github.com/${owner_repo}/releases/${version}" 309 | json=$(http_copy "$giturl" "Accept:application/json") 310 | test -z "$json" && return 1 311 | version=$(echo "$json" | tr -s '\n' ' ' | sed 's/.*"tag_name":"//' | sed 's/".*//') 312 | test -z "$version" && return 1 313 | echo "$version" 314 | } 315 | hash_sha256() { 316 | TARGET=${1:-/dev/stdin} 317 | if is_command gsha256sum; then 318 | hash=$(gsha256sum "$TARGET") || return 1 319 | echo "$hash" | cut -d ' ' -f 1 320 | elif is_command sha256sum; then 321 | hash=$(sha256sum "$TARGET") || return 1 322 | echo "$hash" | cut -d ' ' -f 1 323 | elif is_command shasum; then 324 | hash=$(shasum -a 256 "$TARGET" 2>/dev/null) || return 1 325 | echo "$hash" | cut -d ' ' -f 1 326 | elif is_command openssl; then 327 | hash=$(openssl -dst openssl dgst -sha256 "$TARGET") || return 1 328 | echo "$hash" | cut -d ' ' -f a 329 | else 330 | log_crit "hash_sha256 unable to find command to compute sha-256 hash" 331 | return 1 332 | fi 333 | } 334 | hash_sha256_verify() { 335 | TARGET=$1 336 | checksums=$2 337 | if [ -z "$checksums" ]; then 338 | log_err "hash_sha256_verify checksum file not specified in arg2" 339 | return 1 340 | fi 341 | BASENAME=${TARGET##*/} 342 | want=$(grep "${BASENAME}" "${checksums}" 2>/dev/null | tr '\t' ' ' | cut -d ' ' -f 1) 343 | if [ -z "$want" ]; then 344 | log_err "hash_sha256_verify unable to find checksum for '${TARGET}' in '${checksums}'" 345 | return 1 346 | fi 347 | got=$(hash_sha256 "$TARGET") 348 | if [ "$want" != "$got" ]; then 349 | log_err "hash_sha256_verify checksum for '$TARGET' did not verify ${want} vs $got" 350 | return 1 351 | fi 352 | } 353 | cat /dev/null < 0 { 52 | s = fmt.Sprintf(arg0, args...) 53 | } 54 | if !strings.HasSuffix(s, "\n") { 55 | s += "\n" 56 | } 57 | mu.Lock() 58 | os.Stdout.WriteString(s) 59 | mu.Unlock() 60 | } 61 | 62 | func PrintOnlyVerbose(arg0 string, args ...interface{}) { 63 | if IsVerbose { 64 | Print(arg0, args...) 65 | } 66 | } 67 | 68 | func PrintError(arg0 string, args ...interface{}) { 69 | s := fmt.Sprintf(arg0, args...) 70 | if !strings.HasSuffix(s, "\n") { 71 | s += "\n" 72 | } 73 | mu.Lock() 74 | os.Stderr.WriteString(s) 75 | mu.Unlock() 76 | } 77 | 78 | func PrintErrorAndExit(arg0 string, args ...interface{}) { 79 | PrintError(arg0, args...) 80 | os.Exit(-1) 81 | } 82 | -------------------------------------------------------------------------------- /match.go: -------------------------------------------------------------------------------- 1 | package upx 2 | 3 | import ( 4 | "path/filepath" 5 | "time" 6 | 7 | "github.com/upyun/go-sdk/v3/upyun" 8 | ) 9 | 10 | const ( 11 | TIME_NOT_SET = iota 12 | TIME_BEFORE 13 | TIME_AFTER 14 | TIME_INTERVAL 15 | ) 16 | 17 | const ( 18 | ITEM_NOT_SET = iota 19 | DIR 20 | FILE 21 | ) 22 | 23 | type MatchConfig struct { 24 | Wildcard string 25 | 26 | TimeType int 27 | Before time.Time 28 | After time.Time 29 | 30 | Start string 31 | End string 32 | 33 | ItemType int 34 | } 35 | 36 | func IsMatched(upInfo *upyun.FileInfo, mc *MatchConfig) bool { 37 | if mc.Wildcard != "" { 38 | if same, _ := filepath.Match(mc.Wildcard, upInfo.Name); !same { 39 | return false 40 | } 41 | } 42 | 43 | switch mc.TimeType { 44 | case TIME_BEFORE: 45 | if !upInfo.Time.Before(mc.Before) { 46 | return false 47 | } 48 | case TIME_AFTER: 49 | if !upInfo.Time.After(mc.After) { 50 | return false 51 | } 52 | case TIME_INTERVAL: 53 | if !upInfo.Time.Before(mc.Before) { 54 | return false 55 | } 56 | if !upInfo.Time.After(mc.After) { 57 | return false 58 | } 59 | } 60 | 61 | switch mc.ItemType { 62 | case DIR: 63 | if !upInfo.IsDir { 64 | return false 65 | } 66 | case FILE: 67 | if upInfo.IsDir { 68 | return false 69 | } 70 | } 71 | 72 | return true 73 | } 74 | -------------------------------------------------------------------------------- /move_test.go: -------------------------------------------------------------------------------- 1 | package upx 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "path" 7 | "path/filepath" 8 | "testing" 9 | "time" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestMove(t *testing.T) { 15 | SetUp() 16 | defer TearDown() 17 | 18 | upRootPath := path.Join(ROOT, "move") 19 | Upx("mkdir", upRootPath) 20 | 21 | localRootPath, err := ioutil.TempDir("", "test") 22 | assert.NoError(t, err) 23 | localRootName := filepath.Base(localRootPath) 24 | 25 | CreateFile(path.Join(localRootPath, "FILE1")) 26 | CreateFile(path.Join(localRootPath, "FILE2")) 27 | 28 | // 上传文件 29 | Upx("put", localRootPath, upRootPath) 30 | files, err := Ls(path.Join(upRootPath, localRootName)) 31 | 32 | assert.NoError(t, err) 33 | assert.Len(t, files, 2) 34 | assert.ElementsMatch( 35 | t, 36 | files, 37 | []string{"FILE1", "FILE2"}, 38 | ) 39 | 40 | time.Sleep(time.Second) 41 | 42 | // 正常移动文件 43 | _, err = Upx( 44 | "mv", 45 | path.Join(upRootPath, localRootName, "FILE1"), 46 | path.Join(upRootPath, localRootName, "FILE3"), 47 | ) 48 | assert.NoError(t, err) 49 | 50 | files, err = Ls(path.Join(upRootPath, localRootName)) 51 | assert.NoError(t, err) 52 | assert.Len(t, files, 2) 53 | assert.ElementsMatch( 54 | t, 55 | files, 56 | []string{"FILE2", "FILE3"}, 57 | ) 58 | 59 | time.Sleep(time.Second) 60 | 61 | // 目标文件已存在 62 | _, err = Upx( 63 | "mv", 64 | path.Join(upRootPath, localRootName, "FILE2"), 65 | path.Join(upRootPath, localRootName, "FILE3"), 66 | ) 67 | assert.Equal( 68 | t, 69 | err.Error(), 70 | fmt.Sprintf( 71 | "target path %s already exists use -f to force overwrite\n", 72 | path.Join(upRootPath, localRootName, "FILE3"), 73 | ), 74 | ) 75 | 76 | files, err = Ls(path.Join(upRootPath, localRootName)) 77 | assert.NoError(t, err) 78 | assert.Len(t, files, 2) 79 | assert.ElementsMatch( 80 | t, 81 | files, 82 | []string{"FILE2", "FILE3"}, 83 | ) 84 | 85 | time.Sleep(time.Second) 86 | 87 | // 目标文件已存在, 强制覆盖 88 | _, err = Upx( 89 | "mv", 90 | "-f", 91 | path.Join(upRootPath, localRootName, "FILE2"), 92 | path.Join(upRootPath, localRootName, "FILE3"), 93 | ) 94 | assert.NoError(t, err) 95 | 96 | files, err = Ls(path.Join(upRootPath, localRootName)) 97 | assert.NoError(t, err) 98 | assert.Len(t, files, 1) 99 | assert.ElementsMatch( 100 | t, 101 | files, 102 | []string{"FILE3"}, 103 | ) 104 | } 105 | -------------------------------------------------------------------------------- /partial/chunk.go: -------------------------------------------------------------------------------- 1 | package partial 2 | 3 | import ( 4 | "sync/atomic" 5 | ) 6 | 7 | type Chunk struct { 8 | // 切片的顺序 9 | index int64 10 | 11 | // 切片内容的在源文件的开始地址 12 | start int64 13 | 14 | // 切片内容在源文件的结束地址 15 | end int64 16 | 17 | // 切片任务的下载错误 18 | err error 19 | 20 | // 下载完的切片的具体内容 21 | buffer []byte 22 | } 23 | 24 | func NewChunk(index, start, end int64) *Chunk { 25 | chunk := &Chunk{ 26 | start: start, 27 | end: end, 28 | index: index, 29 | } 30 | return chunk 31 | } 32 | 33 | func (p *Chunk) SetData(bytes []byte) { 34 | p.buffer = bytes 35 | } 36 | 37 | func (p *Chunk) SetError(err error) { 38 | p.err = err 39 | } 40 | 41 | func (p *Chunk) Error() error { 42 | return p.err 43 | } 44 | 45 | func (p *Chunk) Data() []byte { 46 | return p.buffer 47 | } 48 | 49 | // 切片乱序写入后，将切片顺序读取 50 | type ChunksSorter struct { 51 | // 已经读取的切片数量 52 | readCount int64 53 | 54 | // 切片的所有总数 55 | chunkCount int64 56 | 57 | // 线程数，用于阻塞写入 58 | works int64 59 | 60 | // 存储切片的缓存区 61 | chunks []chan *Chunk 62 | } 63 | 64 | func NewChunksSorter(chunkCount int64, works int) *ChunksSorter { 65 | chunks := make([]chan *Chunk, works) 66 | for i := 0; i < len(chunks); i++ { 67 | chunks[i] = make(chan *Chunk) 68 | } 69 | 70 | return &ChunksSorter{ 71 | chunkCount: chunkCount, 72 | works: int64(works), 73 | chunks: chunks, 74 | } 75 | } 76 | 77 | // 将数据写入到缓存区，如果该缓存已满，则会被阻塞 78 | func (p *ChunksSorter) Write(chunk *Chunk) { 79 | p.chunks[chunk.index%p.works] <- chunk 80 | } 81 | 82 | // 关闭 workId 下的通道 83 | func (p *ChunksSorter) Close(workId int) { 84 | if (len(p.chunks) - 1) >= workId { 85 | close(p.chunks[workId]) 86 | } 87 | } 88 | 89 | // 顺序读取切片，如果下一个切片没有下载完，则会被阻塞 90 | func (p *ChunksSorter) Read() *Chunk { 91 | if p.chunkCount == 0 { 92 | return nil 93 | } 94 | i := atomic.AddInt64(&p.readCount, 1) 95 | chunk := <-p.chunks[(i-1)%p.works] 96 | return chunk 97 | } 98 | -------------------------------------------------------------------------------- /partial/downloader.go: -------------------------------------------------------------------------------- 1 | package partial 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "io" 7 | "log" 8 | "os" 9 | "sync" 10 | ) 11 | 12 | const DefaultChunkSize = 1024 * 1024 * 10 13 | 14 | type ChunkDownFunc func(start, end int64) ([]byte, error) 15 | 16 | type MultiPartialDownloader struct { 17 | 18 | // 文件路径 19 | filePath string 20 | 21 | // 最终文件大小 22 | finalSize int64 23 | 24 | // 本地文件大小 25 | localSize int64 26 | 27 | //分片大小 28 | chunkSize int64 29 | 30 | writer io.Writer 31 | works int 32 | downFunc ChunkDownFunc 33 | } 34 | 35 | func NewMultiPartialDownloader(filePath string, finalSize, chunkSize int64, writer io.Writer, works int, fn ChunkDownFunc) *MultiPartialDownloader { 36 | return &MultiPartialDownloader{ 37 | filePath: filePath, 38 | finalSize: finalSize, 39 | works: works, 40 | writer: writer, 41 | chunkSize: chunkSize, 42 | downFunc: fn, 43 | } 44 | } 45 | 46 | func (p *MultiPartialDownloader) Download() error { 47 | fileinfo, err := os.Stat(p.filePath) 48 | 49 | // 如果异常 50 | // - 文件不存在异常: localSize 默认值 0 51 | // - 不是文件不存在异常: 报错 52 | if err != nil && !os.IsNotExist(err) { 53 | return err 54 | } 55 | if err == nil { 56 | p.localSize = fileinfo.Size() 57 | } 58 | 59 | // 计算需要下载的块数 60 | needDownSize := p.finalSize - p.localSize 61 | chunkCount := needDownSize / p.chunkSize 62 | if needDownSize%p.chunkSize != 0 { 63 | chunkCount++ 64 | } 65 | 66 | chunksSorter := NewChunksSorter( 67 | chunkCount, 68 | p.works, 69 | ) 70 | 71 | // 下载切片任务 72 | var wg sync.WaitGroup 73 | ctx, cancel := context.WithCancel(context.Background()) 74 | defer func() { 75 | // 取消切片下载任务，并等待 76 | cancel() 77 | wg.Wait() 78 | }() 79 | 80 | for i := 0; i < p.works; i++ { 81 | wg.Add(1) 82 | go func(ctx context.Context, workId int) { 83 | defer func() { 84 | // 关闭 workId 下的接收通道 85 | chunksSorter.Close(workId) 86 | wg.Done() 87 | }() 88 | 89 | // 每个 work 取自己倍数的 chunk 90 | for j := workId; j < int(chunkCount); j += p.works { 91 | select { 92 | case <-ctx.Done(): 93 | return 94 | default: 95 | var ( 96 | err error 97 | buffer []byte 98 | ) 99 | start := p.localSize + int64(j)*p.chunkSize 100 | end := p.localSize + int64(j+1)*p.chunkSize 101 | if end > p.finalSize { 102 | end = p.finalSize 103 | } 104 | chunk := NewChunk(int64(j), start, end) 105 | 106 | // 重试三次 107 | for t := 0; t < 3; t++ { 108 | // ? 由于长度是从1开始，而数据是从0地址开始 109 | // ? 计算字节时容量会多出开头的一位，所以末尾需要减少一位 110 | buffer, err = p.downFunc(chunk.start, chunk.end-1) 111 | if err == nil { 112 | break 113 | } 114 | } 115 | chunk.SetData(buffer) 116 | chunk.SetError(err) 117 | chunksSorter.Write(chunk) 118 | 119 | if err != nil { 120 | log.Printf("part %d, error: %s", chunk.index, err) 121 | return 122 | } 123 | } 124 | } 125 | }(ctx, i) 126 | } 127 | 128 | // 将分片顺序写入到文件 129 | for { 130 | chunk := chunksSorter.Read() 131 | if chunk == nil { 132 | break 133 | } 134 | if chunk.Error() != nil { 135 | return chunk.Error() 136 | } 137 | if len(chunk.Data()) == 0 { 138 | return errors.New("chunk buffer download but size is 0") 139 | } 140 | _, err := p.writer.Write(chunk.Data()) 141 | if err != nil { 142 | log.Printf("part %d, error: %s", chunk.index, err) 143 | return err 144 | } 145 | } 146 | return nil 147 | } 148 | -------------------------------------------------------------------------------- /partial/downloader_test.go: -------------------------------------------------------------------------------- 1 | package partial 2 | 3 | import ( 4 | "bytes" 5 | "crypto/md5" 6 | "strings" 7 | "testing" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | var buffer bytes.Buffer 12 | 13 | filedata := []byte(strings.Repeat("hello world", 1024*100)) 14 | download := NewMultiPartialDownloader( 15 | "myTestfile", 16 | int64(len(filedata)), 17 | 1024, 18 | &buffer, 19 | 3, 20 | func(start, end int64) ([]byte, error) { 21 | return filedata[start : end+1], nil 22 | }, 23 | ) 24 | 25 | err := download.Download() 26 | if err != nil { 27 | t.Fatal(err.Error()) 28 | } 29 | if md5.Sum(buffer.Bytes()) != md5.Sum(filedata) { 30 | t.Fatal("download file has diff MD5") 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /processbar/bar.go: -------------------------------------------------------------------------------- 1 | package processbar 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | "github.com/vbauerster/mpb/v8" 8 | "github.com/vbauerster/mpb/v8/decor" 9 | ) 10 | 11 | type UpxProcessBar struct { 12 | process *mpb.Progress 13 | enable bool 14 | } 15 | 16 | var ProcessBar = &UpxProcessBar{ 17 | process: mpb.New( 18 | mpb.WithWidth(100), 19 | mpb.WithRefreshRate(180*time.Millisecond), 20 | mpb.WithWaitGroup(&sync.WaitGroup{}), 21 | ), 22 | enable: false, 23 | } 24 | 25 | func (p *UpxProcessBar) Enable() { 26 | p.enable = true 27 | } 28 | 29 | func (p *UpxProcessBar) AddBar(name string, total int64) *mpb.Bar { 30 | if !p.enable { 31 | return nil 32 | } 33 | 34 | bar := p.process.AddBar(0, 35 | mpb.PrependDecorators( 36 | decor.Name(leftAlign(shortPath(name, 30), 30), decor.WCSyncWidth), 37 | decor.Counters(decor.SizeB1024(0), "%.2f / %.2f", decor.WCSyncWidth), 38 | ), 39 | mpb.AppendDecorators( 40 | decor.NewPercentage("%d", decor.WCSyncWidth), 41 | decor.OnComplete( 42 | decor.Name("...", decor.WCSyncWidth), " done", 43 | ), 44 | decor.AverageSpeed(decor.SizeB1024(0), " %.1f", decor.WCSyncWidth), 45 | )) 46 | 47 | bar.SetTotal(total, false) 48 | bar.DecoratorAverageAdjust(time.Now()) 49 | return bar 50 | } 51 | 52 | func (p *UpxProcessBar) Wait() { 53 | if p.enable { 54 | p.process.Wait() 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /processbar/fmt.go: -------------------------------------------------------------------------------- 1 | package processbar 2 | 3 | import "strings" 4 | 5 | func shortPath(s string, width int) string { 6 | if slen(s) <= width { 7 | return s 8 | } 9 | 10 | dotLen := 3 11 | headLen := (width - dotLen) / 2 12 | tailLen := width - dotLen - headLen 13 | 14 | st := 1 15 | for ; st < len(s); st++ { 16 | if slen(s[0:st]) > headLen { 17 | break 18 | } 19 | } 20 | 21 | ed := len(s) - 1 22 | for ; ed >= 0; ed-- { 23 | if slen(s[ed:]) > tailLen { 24 | break 25 | } 26 | } 27 | 28 | return s[0:st-1] + strings.Repeat(".", dotLen) + s[ed+1:] 29 | } 30 | 31 | func leftAlign(s string, width int) string { 32 | l := slen(s) 33 | for i := 0; i < width-l; i++ { 34 | s += " " 35 | } 36 | return s 37 | } 38 | func rightAlign(s string, width int) string { 39 | l := slen(s) 40 | for i := 0; i < width-l; i++ { 41 | s = " " + s 42 | } 43 | return s 44 | } 45 | 46 | func slen(s string) int { 47 | l, rl := len(s), len([]rune(s)) 48 | return (l-rl)/2 + rl 49 | } 50 | -------------------------------------------------------------------------------- /putgetrm_test.go: -------------------------------------------------------------------------------- 1 | package upx 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path" 7 | "path/filepath" 8 | "sort" 9 | "strings" 10 | "testing" 11 | 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func putFile(t *testing.T, src, dst, correct string) { 16 | var err error 17 | if dst == "" { 18 | _, err = Upx("put", src) 19 | } else { 20 | _, err = Upx("put", src, dst) 21 | } 22 | assert.NoError(t, err) 23 | 24 | b, err := Upx("ls", correct) 25 | assert.NoError(t, err) 26 | assert.Equal(t, strings.HasSuffix(string(b), " "+correct+"\n"), true) 27 | } 28 | 29 | func compare(t *testing.T, local, up string) { 30 | locals := []string{} 31 | ups := []string{} 32 | fInfos, _ := ioutil.ReadDir(local) 33 | for _, fInfo := range fInfos { 34 | locals = append(locals, fInfo.Name()) 35 | } 36 | 37 | b, err := Upx("ls", up) 38 | assert.NoError(t, err) 39 | output := strings.TrimRight(string(b), "\n") 40 | for _, line := range strings.Split(output, "\n") { 41 | items := strings.Split(line, " ") 42 | ups = append(ups, items[len(items)-1]) 43 | } 44 | 45 | sort.Strings(locals) 46 | sort.Strings(ups) 47 | 48 | assert.Equal(t, len(locals), len(ups)) 49 | for i := 0; i < len(locals); i++ { 50 | assert.Equal(t, locals[i], ups[i]) 51 | } 52 | } 53 | 54 | func putDir(t *testing.T, src, dst, correct string) { 55 | var err error 56 | if dst == "" { 57 | _, err = Upx("put", src) 58 | } else { 59 | _, err = Upx("put", src, dst) 60 | } 61 | assert.NoError(t, err) 62 | 63 | compare(t, src, correct) 64 | } 65 | 66 | func getFile(t *testing.T, src, dst, correct string) { 67 | var err error 68 | if dst == "" { 69 | _, err = Upx("get", src) 70 | } else { 71 | _, err = Upx("get", src, dst) 72 | } 73 | assert.NoError(t, err) 74 | 75 | _, err = os.Stat(correct) 76 | assert.NoError(t, err) 77 | } 78 | 79 | func getDir(t *testing.T, src, dst, correct string) { 80 | var err error 81 | if dst == "" { 82 | _, err = Upx("get", src) 83 | } else { 84 | _, err = Upx("get", src, dst) 85 | } 86 | assert.NoError(t, err) 87 | 88 | compare(t, correct, src) 89 | } 90 | 91 | func TestPutAndGet(t *testing.T) { 92 | base := ROOT + "/put/" 93 | pwd, err := ioutil.TempDir("", "test") 94 | assert.NoError(t, err) 95 | localBase := filepath.Join(pwd, "put") 96 | func() { 97 | SetUp() 98 | err := os.MkdirAll(localBase, 0755) 99 | assert.NoError(t, err) 100 | }() 101 | defer TearDown() 102 | 103 | err = os.Chdir(localBase) 104 | assert.NoError(t, err) 105 | Upx("mkdir", base) 106 | Upx("cd", base) 107 | 108 | // upx put localBase/FILE upBase/FILE 109 | CreateFile("FILE") 110 | putFile(t, filepath.Join(localBase, "FILE"), "", path.Join(base, "FILE")) 111 | 112 | // upx put ../put/FILE2 113 | CreateFile("FILE2") 114 | localPath := ".." + string(filepath.Separator) + filepath.Join("put", "FILE2") 115 | putFile(t, localPath, "", path.Join(base, "FILE2")) 116 | 117 | // upx put /path/to/file /path/to/file 118 | putFile(t, "FILE", path.Join(base, "FILE4"), path.Join(base, "FILE4")) 119 | 120 | // upx put /path/to/file /path/to/dir 121 | CreateFile("FILE3") 122 | putFile(t, "FILE3", base, path.Join(base, "FILE3")) 123 | 124 | // upx put /path/to/file ../path/to/dir/ 125 | putFile(t, "FILE", base+"/putfile/", path.Join(base, "putfile", "FILE")) 126 | 127 | // upx put ../path/to/dir 128 | localPath = ".." + string(filepath.Separator) + "put" 129 | putDir(t, localPath, "", path.Join(base, "put")) 130 | 131 | // upx put /path/to/dir /path/to/dir/ 132 | putDir(t, localBase, base+"/putdir/", base+"/putdir/") 133 | 134 | _, err = Upx("put", localBase, path.Join(base, "FILE")) 135 | assert.Error(t, err) 136 | 137 | localBase = filepath.Join(pwd, "get") 138 | os.MkdirAll(localBase, 0755) 139 | err = os.Chdir(localBase) 140 | assert.NoError(t, err) 141 | 142 | // upx get /path/to/file 143 | getFile(t, path.Join(base, "FILE"), "", filepath.Join(localBase, "FILE")) 144 | 145 | // upx get ../path/to/file 146 | getFile(t, "../put/FILE2", "", filepath.Join(localBase, "FILE2")) 147 | 148 | // upx get /path/to/file /path/to/file 149 | getFile(t, "FILE4", filepath.Join(localBase, "FILE5"), filepath.Join(localBase, "FILE5")) 150 | 151 | // upx get /path/to/file /path/to/dir 152 | getFile(t, "FILE3", localBase, filepath.Join(localBase, "FILE3")) 153 | 154 | // upx get /path/to/file /path/to/dir/ 155 | localPath = filepath.Join(localBase, "getfile") + string(filepath.Separator) 156 | os.MkdirAll(localPath, 0755) 157 | getFile(t, "FILE", localPath, filepath.Join(localPath, "FILE")) 158 | 159 | // upx get ../path/to/dir 160 | getDir(t, "../put", "", filepath.Join(localBase, "put")) 161 | 162 | // upx get /path/to/dir /path/to/dir/ 163 | localPath = filepath.Join(localBase, "getdir") + string(filepath.Separator) 164 | getDir(t, "../put", localPath, localPath) 165 | 166 | _, err = Upx("get", base, filepath.Join(localBase, "FILE")) 167 | assert.Error(t, err) 168 | 169 | // upx get FILE* 170 | localPath = filepath.Join(localBase, "wildcard") + string(filepath.Separator) 171 | _, err = Upx("get", "FILE*", localPath) 172 | assert.NoError(t, err) 173 | files, _ := Upx("ls", "FILE*") 174 | lfiles, _ := ioutil.ReadDir(localPath) 175 | assert.NotEqual(t, len(lfiles), 0) 176 | assert.Equal(t, len(lfiles)+1, len(strings.Split(string(files), "\n"))) 177 | } 178 | 179 | func TestRm(t *testing.T) { 180 | SetUp() 181 | defer TearDown() 182 | base := ROOT + "/put/" 183 | Upx("cd", base) 184 | _, err := Upx("rm", "put") 185 | assert.Error(t, err) 186 | 187 | _, err = Upx("rm", "put/FILE") 188 | assert.NoError(t, err) 189 | _, err = Upx("ls", "put/FILE") 190 | assert.Error(t, err) 191 | 192 | _, err = Upx("rm", "put/FILE*") 193 | assert.NoError(t, err) 194 | _, err = Upx("ls", "put/FILE*") 195 | assert.Error(t, err) 196 | 197 | _, err = Upx("rm", "-d", "put/*") 198 | assert.NoError(t, err) 199 | _, err = Upx("ls", "-d", "put/*") 200 | assert.Error(t, err) 201 | 202 | _, err = Upx("rm", "-a", "put") 203 | assert.NoError(t, err) 204 | _, err = Upx("ls", "put") 205 | assert.Error(t, err) 206 | } 207 | -------------------------------------------------------------------------------- /putiginore_test.go: -------------------------------------------------------------------------------- 1 | package upx 2 | 3 | import ( 4 | "io/ioutil" 5 | "path" 6 | "path/filepath" 7 | "strings" 8 | "testing" 9 | "time" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func Ls(up string) ([]string, error) { 15 | b, err := Upx("ls", up) 16 | if err != nil { 17 | return nil, err 18 | } 19 | 20 | var ups = make([]string, 0) 21 | output := strings.TrimRight(string(b), "\n") 22 | for _, line := range strings.Split(output, "\n") { 23 | items := strings.Split(line, " ") 24 | ups = append(ups, items[len(items)-1]) 25 | } 26 | return ups, nil 27 | } 28 | 29 | func TestPutIgnore(t *testing.T) { 30 | SetUp() 31 | defer TearDown() 32 | 33 | upRootPath := path.Join(ROOT, "iginore") 34 | Upx("mkdir", upRootPath) 35 | 36 | localRootPath, err := ioutil.TempDir("", "test") 37 | assert.NoError(t, err) 38 | localRootName := filepath.Base(localRootPath) 39 | 40 | CreateFile(path.Join(localRootPath, "FILE1")) 41 | CreateFile(path.Join(localRootPath, "FILE2")) 42 | CreateFile(path.Join(localRootPath, ".FILE3")) 43 | CreateFile(path.Join(localRootPath, ".FILES/FILE")) 44 | 45 | // 上传文件夹 46 | // 不包含隐藏的文件，所以只有FILE1和FILE2 47 | Upx("put", localRootPath, upRootPath) 48 | files, err := Ls(path.Join(upRootPath, localRootName)) 49 | 50 | assert.NoError(t, err) 51 | assert.Len(t, files, 2) 52 | assert.ElementsMatch( 53 | t, 54 | files, 55 | []string{"FILE1", "FILE2"}, 56 | ) 57 | 58 | time.Sleep(time.Second) 59 | 60 | // 上传隐藏的文件夹, 无all，上传失效 61 | Upx( 62 | "put", 63 | path.Join(localRootPath, ".FILES"), 64 | path.Join(upRootPath, localRootName, ".FILES"), 65 | ) 66 | files, err = Ls(path.Join(upRootPath, localRootName)) 67 | assert.NoError(t, err) 68 | 69 | assert.Len(t, files, 2) 70 | assert.ElementsMatch( 71 | t, 72 | files, 73 | []string{"FILE1", "FILE2"}, 74 | ) 75 | 76 | time.Sleep(time.Second) 77 | 78 | // 上传隐藏的文件夹, 有all，上传成功 79 | Upx( 80 | "put", 81 | "-all", 82 | path.Join(localRootPath, ".FILES"), 83 | path.Join(upRootPath, localRootName, ".FILES"), 84 | ) 85 | files, err = Ls(path.Join(upRootPath, localRootName)) 86 | assert.NoError(t, err) 87 | assert.Len(t, files, 3) 88 | assert.ElementsMatch( 89 | t, 90 | files, 91 | []string{"FILE1", "FILE2", ".FILES"}, 92 | ) 93 | 94 | time.Sleep(time.Second) 95 | 96 | // 上传所有文件 97 | Upx("put", "-all", localRootPath, upRootPath) 98 | files, err = Ls(path.Join(upRootPath, localRootName)) 99 | assert.NoError(t, err) 100 | assert.Len(t, files, 4) 101 | assert.ElementsMatch( 102 | t, 103 | files, 104 | []string{"FILE1", "FILE2", ".FILE3", ".FILES"}, 105 | ) 106 | } 107 | -------------------------------------------------------------------------------- /session.go: -------------------------------------------------------------------------------- 1 | package upx 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "io/fs" 9 | "io/ioutil" 10 | "log" 11 | "math/rand" 12 | "net/http" 13 | "net/url" 14 | "os" 15 | "os/signal" 16 | "path" 17 | "path/filepath" 18 | "strings" 19 | "sync" 20 | "time" 21 | 22 | "github.com/fatih/color" 23 | "github.com/upyun/go-sdk/v3/upyun" 24 | "github.com/upyun/upx/fsutil" 25 | "github.com/upyun/upx/partial" 26 | "github.com/upyun/upx/processbar" 27 | "github.com/vbauerster/mpb/v8" 28 | ) 29 | 30 | const ( 31 | SYNC_EXISTS = iota 32 | SYNC_OK 33 | SYNC_FAIL 34 | SYNC_NOT_FOUND 35 | DELETE_OK 36 | DELETE_FAIL 37 | 38 | MinResumePutFileSize = 100 * 1024 * 1024 39 | DefaultBlockSize = 10 * 1024 * 1024 40 | DefaultResumeRetry = 10 41 | ) 42 | 43 | type Session struct { 44 | Bucket string `json:"bucket"` 45 | Operator string `json:"username"` 46 | Password string `json:"password"` 47 | CWD string `json:"cwd"` 48 | 49 | updriver *upyun.UpYun 50 | color bool 51 | 52 | scores map[int]int 53 | smu sync.RWMutex 54 | multipart bool 55 | 56 | taskChan chan interface{} 57 | } 58 | 59 | type syncTask struct { 60 | src, dest string 61 | isdir bool 62 | } 63 | 64 | type delTask struct { 65 | src, dest string 66 | isdir bool 67 | } 68 | 69 | type UploadedFile struct { 70 | barId int 71 | LocalPath string 72 | UpPath string 73 | LocalInfo os.FileInfo 74 | } 75 | 76 | var ( 77 | session *Session 78 | ) 79 | 80 | func (sess *Session) update(key int) { 81 | sess.smu.Lock() 82 | sess.scores[key]++ 83 | sess.smu.Unlock() 84 | } 85 | 86 | func (sess *Session) dump() string { 87 | s := make(map[string]string) 88 | titles := []string{"SYNC_EXISTS", "SYNC_OK", "SYNC_FAIL", "SYNC_NOT_FOUND", "DELETE_OK", "DELETE_FAIL"} 89 | for i, title := range titles { 90 | v := fmt.Sprint(sess.scores[i]) 91 | if len(v) > len(title) { 92 | title = strings.Repeat(" ", len(v)-len(title)) + title 93 | } else { 94 | v = strings.Repeat(" ", len(title)-len(v)) + v 95 | } 96 | s[title] = v 97 | } 98 | header := "+" 99 | for _, title := range titles { 100 | header += strings.Repeat("=", len(s[title])+2) + "+" 101 | } 102 | header += "\n" 103 | footer := strings.Replace(header, "=", "-", -1) 104 | 105 | ret := "\n\n" + header 106 | ret += "|" 107 | for _, title := range titles { 108 | ret += " " + title + " |" 109 | } 110 | ret += "\n" + footer 111 | 112 | ret += "|" 113 | for _, title := range titles { 114 | ret += " " + s[title] + " |" 115 | } 116 | return ret + "\n" + footer 117 | } 118 | 119 | func (sess *Session) AbsPath(upPath string) (ret string) { 120 | if strings.HasPrefix(upPath, "/") { 121 | ret = path.Join(upPath) 122 | } else { 123 | ret = path.Join(sess.CWD, upPath) 124 | } 125 | 126 | if strings.HasSuffix(upPath, "/") && ret != "/" { 127 | ret += "/" 128 | } 129 | return 130 | } 131 | 132 | func (sess *Session) IsUpYunDir(upPath string) (isDir bool, exist bool) { 133 | upInfo, err := sess.updriver.GetInfo(sess.AbsPath(upPath)) 134 | if err != nil { 135 | return false, false 136 | } 137 | return upInfo.IsDir, true 138 | } 139 | 140 | func (sess *Session) IsLocalDir(localPath string) (isDir bool, exist bool) { 141 | fInfo, err := os.Stat(localPath) 142 | if err != nil { 143 | return false, false 144 | } 145 | return fInfo.IsDir(), true 146 | } 147 | 148 | func (sess *Session) FormatUpInfo(upInfo *upyun.FileInfo) string { 149 | s := "drwxrwxrwx" 150 | if !upInfo.IsDir { 151 | s = "-rw-rw-rw-" 152 | } 153 | s += fmt.Sprintf(" 1 %s %s %12d", sess.Operator, sess.Bucket, upInfo.Size) 154 | if upInfo.Time.Year() != time.Now().Year() { 155 | s += " " + upInfo.Time.Format("Jan 02 2006") 156 | } else { 157 | s += " " + upInfo.Time.Format("Jan 02 03:04") 158 | } 159 | if upInfo.IsDir && sess.color { 160 | s += " " + color.BlueString(upInfo.Name) 161 | } else { 162 | s += " " + upInfo.Name 163 | } 164 | return s 165 | } 166 | 167 | func (sess *Session) Init() error { 168 | sess.scores = make(map[int]int) 169 | sess.updriver = upyun.NewUpYun(&upyun.UpYunConfig{ 170 | Bucket: sess.Bucket, 171 | Operator: sess.Operator, 172 | Password: sess.Password, 173 | UserAgent: fmt.Sprintf("upx/%s", VERSION), 174 | }) 175 | _, err := sess.updriver.Usage() 176 | return err 177 | } 178 | 179 | func (sess *Session) Info() { 180 | n, err := sess.updriver.Usage() 181 | if err != nil { 182 | PrintErrorAndExit("usage: %v", err) 183 | } 184 | 185 | tmp := []string{ 186 | fmt.Sprintf("ServiceName: %s", sess.Bucket), 187 | fmt.Sprintf("Operator: %s", sess.Operator), 188 | fmt.Sprintf("CurrentDir: %s", sess.CWD), 189 | fmt.Sprintf("Usage: %s", humanizeSize(n)), 190 | } 191 | 192 | Print(strings.Join(tmp, "\n")) 193 | } 194 | 195 | func (sess *Session) Pwd() { 196 | Print("%s", sess.CWD) 197 | } 198 | 199 | func (sess *Session) Mkdir(upPaths ...string) { 200 | for _, upPath := range upPaths { 201 | fpath := sess.AbsPath(upPath) 202 | for fpath != "/" { 203 | if err := sess.updriver.Mkdir(fpath); err != nil { 204 | PrintErrorAndExit("mkdir %s: %v", fpath, err) 205 | } 206 | fpath = path.Dir(fpath) 207 | } 208 | } 209 | } 210 | 211 | func (sess *Session) Cd(upPath string) { 212 | fpath := sess.AbsPath(upPath) 213 | if isDir, _ := sess.IsUpYunDir(fpath); isDir { 214 | sess.CWD = fpath 215 | Print(sess.CWD) 216 | } else { 217 | PrintErrorAndExit("cd: %s: Not a directory", fpath) 218 | } 219 | } 220 | 221 | func (sess *Session) Ls(upPath string, match *MatchConfig, maxItems int, isDesc bool) { 222 | fpath := sess.AbsPath(upPath) 223 | isDir, exist := sess.IsUpYunDir(fpath) 224 | if !exist { 225 | PrintErrorAndExit("ls: cannot access %s: No such file or directory", fpath) 226 | } 227 | 228 | if !isDir { 229 | fInfo, err := sess.updriver.GetInfo(fpath) 230 | if err != nil { 231 | PrintErrorAndExit("ls %s: %v", fpath, err) 232 | } 233 | if IsMatched(fInfo, match) { 234 | Print(sess.FormatUpInfo(fInfo)) 235 | } else { 236 | PrintErrorAndExit("ls: cannot access %s: No such file or directory", fpath) 237 | } 238 | return 239 | } 240 | 241 | fInfoChan := make(chan *upyun.FileInfo, 50) 242 | go func() { 243 | err := sess.updriver.List(&upyun.GetObjectsConfig{ 244 | Path: fpath, 245 | ObjectsChan: fInfoChan, 246 | DescOrder: isDesc, 247 | }) 248 | if err != nil { 249 | PrintErrorAndExit("ls %s: %v", fpath, err) 250 | } 251 | }() 252 | 253 | objs := 0 254 | for fInfo := range fInfoChan { 255 | if IsMatched(fInfo, match) { 256 | Print(sess.FormatUpInfo(fInfo)) 257 | objs++ 258 | } 259 | if maxItems > 0 && objs >= maxItems { 260 | break 261 | } 262 | } 263 | if objs == 0 && (match.Wildcard != "" || match.TimeType != TIME_NOT_SET) { 264 | msg := fpath 265 | if match.Wildcard != "" { 266 | msg = fpath + "/" + match.Wildcard 267 | } 268 | if match.TimeType != TIME_NOT_SET { 269 | msg += " timestamp@" 270 | if match.TimeType == TIME_AFTER || match.TimeType == TIME_INTERVAL { 271 | msg += "[" + match.After.Format("2006-01-02 15:04:05") + "," 272 | } else { 273 | msg += "[-oo," 274 | } 275 | if match.TimeType == TIME_BEFORE || match.TimeType == TIME_INTERVAL { 276 | msg += match.Before.Format("2006-01-02 15:04:05") + "]" 277 | } else { 278 | msg += "+oo]" 279 | } 280 | } 281 | PrintErrorAndExit("ls: cannot access %s: No such file or directory", msg) 282 | } 283 | } 284 | 285 | func (sess *Session) getDir(upPath, localPath string, match *MatchConfig, workers int, resume bool) error { 286 | if err := os.MkdirAll(localPath, 0755); err != nil { 287 | return err 288 | } 289 | 290 | var wg sync.WaitGroup 291 | 292 | fInfoChan := make(chan *upyun.FileInfo, workers*2) 293 | wg.Add(workers) 294 | for w := 0; w < workers; w++ { 295 | go func() { 296 | defer wg.Done() 297 | var e error 298 | for fInfo := range fInfoChan { 299 | if IsMatched(fInfo, match) { 300 | fpath := path.Join(upPath, fInfo.Name) 301 | lpath := filepath.Join(localPath, filepath.FromSlash(cleanFilename(fInfo.Name))) 302 | if fInfo.IsDir { 303 | os.MkdirAll(lpath, 0755) 304 | } else { 305 | isContinue := resume 306 | 307 | // 判断本地文件是否存在 308 | // 如果存在，大小一致 并且本地文件的最后修改时间大于云端文件的最后修改时间 则跳过该下载 309 | // 如果云端文件最后的修改时间大于本地文件的创建时间，则强制重新下载 310 | stat, err := os.Stat(lpath) 311 | if err == nil { 312 | if stat.Size() == fInfo.Size && stat.ModTime().After(fInfo.Time) { 313 | continue 314 | } 315 | if stat.Size() > fInfo.Size { 316 | isContinue = false 317 | } 318 | if fInfo.Time.After(stat.ModTime()) { 319 | isContinue = false 320 | } 321 | } 322 | 323 | for i := 1; i <= MaxRetry; i++ { 324 | e = sess.getFileWithProgress(fpath, lpath, fInfo, 1, isContinue, false) 325 | if e == nil { 326 | break 327 | } 328 | if upyun.IsNotExist(e) { 329 | e = nil 330 | break 331 | } 332 | 333 | time.Sleep(time.Duration(i*(rand.Intn(MaxJitter-MinJitter)+MinJitter)) * time.Second) 334 | } 335 | } 336 | if e != nil { 337 | return 338 | } 339 | } 340 | } 341 | }() 342 | } 343 | 344 | err := sess.updriver.List(&upyun.GetObjectsConfig{ 345 | Path: upPath, 346 | ObjectsChan: fInfoChan, 347 | MaxListTries: 3, 348 | MaxListLevel: -1, 349 | }) 350 | wg.Wait() 351 | return err 352 | } 353 | 354 | func (sess *Session) getFileWithProgress(upPath, localPath string, upInfo *upyun.FileInfo, works int, resume, inprogress bool) error { 355 | var err error 356 | 357 | var bar *mpb.Bar 358 | if upInfo.Size > 0 { 359 | bar = processbar.ProcessBar.AddBar(localPath, upInfo.Size) 360 | } 361 | 362 | dir := filepath.Dir(localPath) 363 | if err = os.MkdirAll(dir, 0755); err != nil { 364 | return err 365 | } 366 | 367 | w, err := NewFileWrappedWriter(localPath, bar, resume) 368 | if err != nil { 369 | return err 370 | } 371 | defer w.Close() 372 | 373 | downloader := partial.NewMultiPartialDownloader( 374 | localPath, 375 | upInfo.Size, 376 | partial.DefaultChunkSize, 377 | w, 378 | works, 379 | func(start, end int64) ([]byte, error) { 380 | var buffer bytes.Buffer 381 | headers := map[string]string{ 382 | "Range": fmt.Sprintf("bytes=%d-%d", start, end), 383 | } 384 | if inprogress { 385 | headers["X-Upyun-Multi-In-Progress"] = "true" 386 | } 387 | _, err = sess.updriver.Get(&upyun.GetObjectConfig{ 388 | Path: sess.AbsPath(upPath), 389 | Writer: &buffer, 390 | Headers: headers, 391 | }) 392 | return buffer.Bytes(), err 393 | }, 394 | ) 395 | err = downloader.Download() 396 | if bar != nil { 397 | bar.EnableTriggerComplete() 398 | if err != nil { 399 | bar.Abort(false) 400 | } 401 | } 402 | return err 403 | } 404 | 405 | func (sess *Session) Get(upPath, localPath string, match *MatchConfig, workers int, resume, inprogress bool) { 406 | upPath = sess.AbsPath(upPath) 407 | headers := map[string]string{} 408 | if inprogress { 409 | headers["X-Upyun-Multi-In-Progress"] = "true" 410 | resume = true 411 | } 412 | upInfo, err := sess.updriver.GetInfoWithHeaders(upPath, headers) 413 | if err != nil { 414 | PrintErrorAndExit("getinfo %s: %v", upPath, err) 415 | } 416 | 417 | exist, isDir := false, false 418 | if localInfo, _ := os.Stat(localPath); localInfo != nil { 419 | exist = true 420 | isDir = localInfo.IsDir() 421 | } else { 422 | if strings.HasSuffix(localPath, "/") { 423 | isDir = true 424 | } 425 | } 426 | 427 | if upInfo.IsDir { 428 | if inprogress { 429 | PrintErrorAndExit("get: %s is a directory", localPath) 430 | } 431 | if exist { 432 | if !isDir { 433 | PrintErrorAndExit("get: %s Not a directory", localPath) 434 | } else { 435 | if match.Wildcard == "" { 436 | localPath = filepath.Join(localPath, path.Base(upPath)) 437 | } 438 | } 439 | } 440 | if err := sess.getDir(upPath, localPath, match, workers, resume); err != nil { 441 | PrintErrorAndExit(err.Error()) 442 | } 443 | } else { 444 | if isDir { 445 | localPath = filepath.Join(localPath, cleanFilename(path.Base(upPath))) 446 | } 447 | 448 | // 小于 100M 不开启多线程 449 | if upInfo.Size < 1024*1024*100 || inprogress { 450 | workers = 1 451 | } 452 | err := sess.getFileWithProgress(upPath, localPath, upInfo, workers, resume, inprogress) 453 | if err != nil { 454 | PrintErrorAndExit(err.Error()) 455 | } 456 | } 457 | } 458 | 459 | func (sess *Session) GetStartBetweenEndFiles(upPath, localPath string, match *MatchConfig, workers int) { 460 | fpath := sess.AbsPath(upPath) 461 | isDir, exist := sess.IsUpYunDir(fpath) 462 | if !exist { 463 | if match.ItemType == DIR { 464 | isDir = true 465 | } else { 466 | PrintErrorAndExit("get: cannot down %s:No such file or directory", fpath) 467 | } 468 | } 469 | if isDir && match != nil && match.Wildcard == "" { 470 | if match.ItemType == FILE { 471 | PrintErrorAndExit("get: cannot down %s: Is a directory", fpath) 472 | } 473 | } 474 | 475 | fInfoChan := make(chan *upyun.FileInfo, 1) 476 | objectsConfig := &upyun.GetObjectsConfig{ 477 | Path: fpath, 478 | ObjectsChan: fInfoChan, 479 | QuitChan: make(chan bool, 1), 480 | } 481 | go func() { 482 | err := sess.updriver.List(objectsConfig) 483 | if err != nil { 484 | PrintErrorAndExit("ls %s: %v", fpath, err) 485 | } 486 | }() 487 | 488 | startList := match.Start 489 | if startList != "" && startList[0] != '/' { 490 | startList = filepath.Join(fpath, startList) 491 | } 492 | endList := match.End 493 | if endList != "" && endList[0] != '/' { 494 | endList = filepath.Join(fpath, endList) 495 | } 496 | 497 | for fInfo := range fInfoChan { 498 | fp := filepath.Join(fpath, fInfo.Name) 499 | if (fp >= startList || startList == "") && (fp < endList || endList == "") { 500 | sess.Get(fp, localPath, match, workers, false, false) 501 | } else if strings.HasPrefix(startList, fp) { 502 | //前缀相同进入下一级文件夹，继续递归判断 503 | if fInfo.IsDir { 504 | sess.GetStartBetweenEndFiles(fp, localPath+fInfo.Name+"/", match, workers) 505 | } 506 | } 507 | if fp >= endList && endList != "" && fInfo.IsDir { 508 | close(objectsConfig.QuitChan) 509 | break 510 | } 511 | } 512 | } 513 | 514 | func (sess *Session) putFileWithProgress(localPath, upPath string, localInfo os.FileInfo) error { 515 | var err error 516 | fd, err := os.Open(localPath) 517 | if err != nil { 518 | return err 519 | } 520 | defer fd.Close() 521 | cfg := &upyun.PutObjectConfig{ 522 | Path: upPath, 523 | Headers: map[string]string{ 524 | "Content-Length": fmt.Sprint(localInfo.Size()), 525 | }, 526 | Reader: fd, 527 | } 528 | 529 | var bar *mpb.Bar 530 | if IsVerbose { 531 | if localInfo.Size() > 0 { 532 | bar = processbar.ProcessBar.AddBar(upPath, localInfo.Size()) 533 | cfg.ProxyReader = func(offset int64, r io.Reader) io.Reader { 534 | if offset > 0 { 535 | bar.SetCurrent(offset) 536 | } 537 | return bar.ProxyReader(r) 538 | } 539 | } 540 | } else { 541 | log.Printf("file: %s, Start\n", upPath) 542 | } 543 | if localInfo.Size() >= MinResumePutFileSize || sess.multipart { 544 | cfg.UseResumeUpload = true 545 | cfg.ResumePartSize = ResumePartSize(localInfo.Size()) 546 | cfg.MaxResumePutTries = DefaultResumeRetry 547 | } 548 | 549 | err = sess.updriver.Put(cfg) 550 | if bar != nil { 551 | bar.EnableTriggerComplete() 552 | if err != nil { 553 | bar.Abort(false) 554 | } 555 | } 556 | if !IsVerbose { 557 | log.Printf("file: %s, Done\n", upPath) 558 | } 559 | return err 560 | } 561 | 562 | func (sess *Session) putRemoteFileWithProgress(rawURL, upPath string) error { 563 | var size int64 564 | 565 | // 如果可以的话，先从 Head 请求中获取文件长度 566 | resp, err := http.Head(rawURL) 567 | if err == nil && resp.ContentLength > 0 { 568 | size = resp.ContentLength 569 | } 570 | resp.Body.Close() 571 | 572 | // 通过get方法获取文件，如果get头中包含Content-Length，则使用get头中的Content-Length 573 | resp, err = http.Get(rawURL) 574 | if err != nil { 575 | return fmt.Errorf("http Get %s error: %v", rawURL, err) 576 | } 577 | defer resp.Body.Close() 578 | 579 | if resp.ContentLength > 0 { 580 | size = resp.ContentLength 581 | } 582 | 583 | // 如果无法获取 Content-Length 则报错 584 | if size == 0 { 585 | return fmt.Errorf("get http file Content-Length error: response headers not has Content-Length") 586 | } 587 | 588 | // 创建进度条 589 | bar := processbar.ProcessBar.AddBar(upPath, size) 590 | reader := NewFileWrappedReader(bar, resp.Body) 591 | 592 | // 上传文件 593 | err = sess.updriver.Put(&upyun.PutObjectConfig{ 594 | Path: upPath, 595 | Reader: reader, 596 | UseMD5: false, 597 | Headers: map[string]string{ 598 | "Content-Length": fmt.Sprint(size), 599 | }, 600 | }) 601 | if bar != nil { 602 | bar.EnableTriggerComplete() 603 | if err != nil { 604 | bar.Abort(false) 605 | } 606 | } 607 | if err != nil { 608 | PrintErrorAndExit("put file error: %v", err) 609 | } 610 | 611 | return nil 612 | } 613 | 614 | func (sess *Session) putFilesWitchProgress(localFiles []*UploadedFile, workers int) { 615 | var wg sync.WaitGroup 616 | 617 | tasks := make(chan *UploadedFile, workers*2) 618 | for w := 0; w < workers; w++ { 619 | wg.Add(1) 620 | go func() { 621 | defer wg.Done() 622 | for task := range tasks { 623 | err := sess.putFileWithProgress( 624 | task.LocalPath, 625 | task.UpPath, 626 | task.LocalInfo, 627 | ) 628 | if err != nil { 629 | fmt.Println("putFileWithProgress error: ", err.Error()) 630 | return 631 | } 632 | } 633 | }() 634 | } 635 | 636 | for _, f := range localFiles { 637 | tasks <- f 638 | } 639 | 640 | close(tasks) 641 | wg.Wait() 642 | } 643 | 644 | func (sess *Session) putDir(localPath, upPath string, workers int, withIgnore bool) { 645 | localAbsPath, err := filepath.Abs(localPath) 646 | if err != nil { 647 | PrintErrorAndExit(err.Error()) 648 | } 649 | // 如果上传的是目录，并且是隐藏的目录，则触发提示 650 | rootDirInfo, err := os.Stat(localAbsPath) 651 | if err != nil { 652 | PrintErrorAndExit(err.Error()) 653 | } 654 | if !withIgnore && fsutil.IsIgnoreFile(localAbsPath, rootDirInfo) { 655 | PrintErrorAndExit("%s is a ignore dir, use `-all` to force put all files", localAbsPath) 656 | } 657 | 658 | type FileInfo struct { 659 | fpath string 660 | fInfo os.FileInfo 661 | } 662 | localFiles := make(chan *FileInfo, workers*2) 663 | var wg sync.WaitGroup 664 | wg.Add(workers) 665 | for w := 0; w < workers; w++ { 666 | go func() { 667 | defer wg.Done() 668 | for info := range localFiles { 669 | rel, _ := filepath.Rel(localAbsPath, info.fpath) 670 | desPath := path.Join(upPath, filepath.ToSlash(rel)) 671 | fInfo, err := os.Stat(info.fpath) 672 | if err == nil && fInfo.IsDir() { 673 | err = sess.updriver.Mkdir(desPath) 674 | } else { 675 | err = sess.putFileWithProgress(info.fpath, desPath, info.fInfo) 676 | } 677 | if err != nil { 678 | log.Printf("put %s to %s error: %s", info.fpath, desPath, err) 679 | if upyun.IsTooManyRequests(err) { 680 | time.Sleep(time.Second) 681 | continue 682 | } 683 | return 684 | } 685 | } 686 | }() 687 | } 688 | 689 | filepath.Walk(localAbsPath, func(path string, info fs.FileInfo, err error) error { 690 | if err != nil { 691 | return err 692 | } 693 | if !withIgnore && fsutil.IsIgnoreFile(path, info) { 694 | if info.IsDir() { 695 | return filepath.SkipDir 696 | } 697 | } else { 698 | localFiles <- &FileInfo{ 699 | fpath: path, 700 | fInfo: info, 701 | } 702 | } 703 | return nil 704 | }) 705 | 706 | close(localFiles) 707 | wg.Wait() 708 | } 709 | 710 | // / Put 上传单文件或单目录 711 | func (sess *Session) Put(localPath, upPath string, workers int, withIgnore, inprogress bool) { 712 | upPath = sess.AbsPath(upPath) 713 | if inprogress { 714 | sess.multipart = true 715 | } 716 | exist, isDir := false, false 717 | if upInfo, _ := sess.updriver.GetInfo(upPath); upInfo != nil { 718 | exist = true 719 | isDir = upInfo.IsDir 720 | } 721 | // 如果指定了是远程的目录 但是实际在远程的目录是文件类型则报错 722 | if exist && !isDir && strings.HasSuffix(upPath, "/") { 723 | PrintErrorAndExit("cant put to %s: path is not a directory, maybe a file", upPath) 724 | } 725 | if !exist && strings.HasSuffix(upPath, "/") { 726 | isDir = true 727 | } 728 | 729 | // 如果需要上传的文件是URL链接 730 | fileURL, _ := url.ParseRequestURI(localPath) 731 | if fileURL != nil && fileURL.Scheme != "" && fileURL.Host != "" { 732 | if !contains([]string{"http", "https"}, fileURL.Scheme) { 733 | PrintErrorAndExit("Invalid URL %s", localPath) 734 | } 735 | 736 | // 如果指定的远程路径 upPath 是目录 737 | // 则从 url 中获取文件名，获取文件名失败则报错 738 | if isDir { 739 | if spaces := strings.Split(fileURL.Path, "/"); len(spaces) > 0 { 740 | upPath = path.Join(upPath, spaces[len(spaces)-1]) 741 | } else { 742 | PrintErrorAndExit("missing file name in the url, must has remote path name") 743 | } 744 | } 745 | err := sess.putRemoteFileWithProgress(localPath, upPath) 746 | if err != nil { 747 | PrintErrorAndExit(err.Error()) 748 | } 749 | return 750 | } 751 | 752 | localInfo, err := os.Stat(localPath) 753 | if err != nil { 754 | PrintErrorAndExit("stat %s: %v", localPath, err) 755 | } 756 | 757 | if localInfo.IsDir() { 758 | if exist { 759 | if !isDir { 760 | PrintErrorAndExit("put: %s: Not a directory", upPath) 761 | } else { 762 | upPath = path.Join(upPath, filepath.Base(localPath)) 763 | } 764 | } 765 | sess.putDir(localPath, upPath, workers, withIgnore) 766 | } else { 767 | if isDir { 768 | upPath = path.Join(upPath, filepath.Base(localPath)) 769 | } 770 | sess.putFileWithProgress(localPath, upPath, localInfo) 771 | } 772 | } 773 | 774 | // put 的升级版命令, 支持多文件上传 775 | func (sess *Session) Upload(filenames []string, upPath string, workers int, withIgnore bool) { 776 | upPath = sess.AbsPath(upPath) 777 | 778 | // 检测云端的目的地目录 779 | upPathExist, upPathIsDir := false, false 780 | if upInfo, _ := sess.updriver.GetInfo(upPath); upInfo != nil { 781 | upPathExist = true 782 | upPathIsDir = upInfo.IsDir 783 | } 784 | // 多文件上传 upPath 如果存在则只能是目录 785 | if upPathExist && !upPathIsDir { 786 | PrintErrorAndExit("upload: %s: Not a directory", upPath) 787 | } 788 | 789 | var ( 790 | dirs []string 791 | uploadedFile []*UploadedFile 792 | ) 793 | for _, filename := range filenames { 794 | localInfo, err := os.Stat(filename) 795 | if err != nil { 796 | PrintErrorAndExit(err.Error()) 797 | } 798 | 799 | if localInfo.IsDir() { 800 | dirs = append(dirs, filename) 801 | } else { 802 | uploadedFile = append(uploadedFile, &UploadedFile{ 803 | barId: -1, 804 | LocalPath: filename, 805 | UpPath: path.Join(upPath, filepath.Base(filename)), 806 | LocalInfo: localInfo, 807 | }) 808 | } 809 | } 810 | 811 | // 上传目录 812 | for _, localPath := range dirs { 813 | sess.putDir( 814 | localPath, 815 | path.Join(upPath, filepath.Base(localPath)), 816 | workers, 817 | withIgnore, 818 | ) 819 | } 820 | 821 | // 上传文件 822 | sess.putFilesWitchProgress(uploadedFile, workers) 823 | } 824 | 825 | func (sess *Session) rm(fpath string, isAsync bool, isFolder bool) { 826 | err := sess.updriver.Delete(&upyun.DeleteObjectConfig{ 827 | Path: fpath, 828 | Async: isAsync, 829 | Folder: isFolder, 830 | }) 831 | if err == nil || upyun.IsNotExist(err) { 832 | sess.update(DELETE_OK) 833 | PrintOnlyVerbose("DELETE %s OK", fpath) 834 | } else { 835 | sess.update(DELETE_FAIL) 836 | PrintError("DELETE %s FAIL %v", fpath, err) 837 | } 838 | } 839 | func (sess *Session) rmFile(fpath string, isAsync bool) { 840 | sess.rm(fpath, isAsync, false) 841 | } 842 | 843 | func (sess *Session) rmEmptyDir(fpath string, isAsync bool) { 844 | sess.rm(fpath, isAsync, true) 845 | } 846 | 847 | func (sess *Session) rmDir(fpath string, isAsync bool) { 848 | fInfoChan := make(chan *upyun.FileInfo, 50) 849 | go func() { 850 | err := sess.updriver.List(&upyun.GetObjectsConfig{ 851 | Path: fpath, 852 | ObjectsChan: fInfoChan, 853 | }) 854 | if err != nil { 855 | if upyun.IsNotExist(err) { 856 | return 857 | } else { 858 | PrintErrorAndExit("ls %s: %v", fpath, err) 859 | } 860 | } 861 | }() 862 | 863 | for fInfo := range fInfoChan { 864 | fp := path.Join(fpath, fInfo.Name) 865 | if fInfo.IsDir { 866 | sess.rmDir(fp, isAsync) 867 | } else { 868 | sess.rmFile(fp, isAsync) 869 | } 870 | } 871 | sess.rmEmptyDir(fpath, isAsync) 872 | } 873 | 874 | func (sess *Session) Rm(upPath string, match *MatchConfig, isAsync bool) { 875 | fpath := sess.AbsPath(upPath) 876 | isDir, exist := sess.IsUpYunDir(fpath) 877 | if !exist { 878 | if match.ItemType == DIR { 879 | isDir = true 880 | } else { 881 | PrintErrorAndExit("rm: cannot remove %s: No such file or directory", fpath) 882 | } 883 | } 884 | 885 | if isDir && match != nil && match.Wildcard == "" { 886 | if match.ItemType == FILE { 887 | PrintErrorAndExit("rm: cannot remove %s: Is a directory, add -d/-a flag", fpath) 888 | } 889 | sess.rmDir(fpath, isAsync) 890 | return 891 | } 892 | 893 | if !isDir { 894 | fInfo, err := sess.updriver.GetInfo(fpath) 895 | if err != nil { 896 | PrintErrorAndExit("getinfo %s: %v", fpath, err) 897 | } 898 | if IsMatched(fInfo, match) { 899 | sess.rmFile(fpath, isAsync) 900 | } 901 | return 902 | } 903 | 904 | fInfoChan := make(chan *upyun.FileInfo, 50) 905 | go func() { 906 | err := sess.updriver.List(&upyun.GetObjectsConfig{ 907 | Path: fpath, 908 | ObjectsChan: fInfoChan, 909 | }) 910 | if err != nil { 911 | PrintErrorAndExit("ls %s: %v", fpath, err) 912 | } 913 | }() 914 | 915 | for fInfo := range fInfoChan { 916 | fp := path.Join(fpath, fInfo.Name) 917 | if IsMatched(fInfo, match) { 918 | if fInfo.IsDir { 919 | sess.rmDir(fp, isAsync) 920 | } else { 921 | sess.rmFile(fp, isAsync) 922 | } 923 | } 924 | } 925 | } 926 | 927 | func (sess *Session) tree(upPath, prefix string, output chan string) (folders, files int, err error) { 928 | upInfos := make(chan *upyun.FileInfo, 50) 929 | fpath := sess.AbsPath(upPath) 930 | wg := sync.WaitGroup{} 931 | wg.Add(1) 932 | 933 | go func() { 934 | defer wg.Done() 935 | prevInfo := <-upInfos 936 | for fInfo := range upInfos { 937 | p := prefix + "|-- " 938 | if prevInfo.IsDir { 939 | if sess.color { 940 | output <- p + color.BlueString("%s", prevInfo.Name) 941 | } else { 942 | output <- p + prevInfo.Name 943 | } 944 | folders++ 945 | d, f, _ := sess.tree(path.Join(fpath, prevInfo.Name), prefix+"! ", output) 946 | folders += d 947 | files += f 948 | } else { 949 | output <- p + prevInfo.Name 950 | files++ 951 | } 952 | prevInfo = fInfo 953 | } 954 | if prevInfo == nil { 955 | return 956 | } 957 | p := prefix + "`-- " 958 | if prevInfo.IsDir { 959 | if sess.color { 960 | output <- p + color.BlueString("%s", prevInfo.Name) 961 | } else { 962 | output <- p + prevInfo.Name 963 | } 964 | folders++ 965 | d, f, _ := sess.tree(path.Join(fpath, prevInfo.Name), prefix+" ", output) 966 | folders += d 967 | files += f 968 | } else { 969 | output <- p + prevInfo.Name 970 | files++ 971 | } 972 | }() 973 | 974 | err = sess.updriver.List(&upyun.GetObjectsConfig{ 975 | Path: fpath, 976 | ObjectsChan: upInfos, 977 | }) 978 | wg.Wait() 979 | return 980 | } 981 | 982 | func (sess *Session) Tree(upPath string) { 983 | fpath := sess.AbsPath(upPath) 984 | files, folders := 0, 0 985 | defer func() { 986 | Print("\n%d directories, %d files", folders, files) 987 | }() 988 | 989 | if isDir, _ := sess.IsUpYunDir(fpath); !isDir { 990 | PrintErrorAndExit("%s [error opening dir]", fpath) 991 | } 992 | Print("%s", fpath) 993 | 994 | output := make(chan string, 50) 995 | go func() { 996 | folders, files, _ = sess.tree(fpath, "", output) 997 | close(output) 998 | }() 999 | 1000 | for s := range output { 1001 | Print(s) 1002 | } 1003 | return 1004 | } 1005 | 1006 | func (sess *Session) syncFile(localPath, upPath string, strongCheck bool) (status int, err error) { 1007 | curMeta, err := makeDBValue(localPath, false) 1008 | if err != nil { 1009 | if os.IsNotExist(err) { 1010 | return SYNC_NOT_FOUND, err 1011 | } 1012 | return SYNC_FAIL, err 1013 | } 1014 | if curMeta.IsDir == "true" { 1015 | return SYNC_FAIL, fmt.Errorf("file type changed") 1016 | } 1017 | 1018 | if strongCheck { 1019 | upInfo, _ := sess.updriver.GetInfo(upPath) 1020 | if upInfo != nil { 1021 | curMeta.Md5, _ = md5File(localPath) 1022 | if curMeta.Md5 == upInfo.MD5 { 1023 | setDBValue(localPath, upPath, curMeta) 1024 | return SYNC_EXISTS, nil 1025 | } 1026 | } 1027 | } else { 1028 | prevMeta, err := getDBValue(localPath, upPath) 1029 | if err != nil { 1030 | return SYNC_FAIL, err 1031 | } 1032 | 1033 | if prevMeta != nil { 1034 | if curMeta.ModifyTime == prevMeta.ModifyTime { 1035 | return SYNC_EXISTS, nil 1036 | } 1037 | curMeta.Md5, _ = md5File(localPath) 1038 | if curMeta.Md5 == prevMeta.Md5 { 1039 | setDBValue(localPath, upPath, curMeta) 1040 | return SYNC_EXISTS, nil 1041 | } 1042 | } 1043 | } 1044 | for i := 1; i <= MaxRetry; i++ { 1045 | err = sess.updriver.Put(&upyun.PutObjectConfig{Path: upPath, LocalPath: localPath}) 1046 | if err == nil { 1047 | break 1048 | } 1049 | time.Sleep(time.Duration(i*(rand.Intn(MaxJitter-MinJitter)+MinJitter)) * time.Second) 1050 | } 1051 | if err != nil { 1052 | return SYNC_FAIL, err 1053 | } 1054 | setDBValue(localPath, upPath, curMeta) 1055 | return SYNC_OK, nil 1056 | } 1057 | 1058 | func (sess *Session) syncObject(localPath, upPath string, isDir bool) { 1059 | if isDir { 1060 | status, err := sess.syncDirectory(localPath, upPath) 1061 | switch status { 1062 | case SYNC_OK: 1063 | PrintOnlyVerbose("sync %s to %s OK", localPath, upPath) 1064 | case SYNC_EXISTS: 1065 | PrintOnlyVerbose("sync %s to %s EXISTS", localPath, upPath) 1066 | case SYNC_FAIL, SYNC_NOT_FOUND: 1067 | PrintError("sync %s to %s FAIL %v", localPath, upPath, err) 1068 | } 1069 | sess.update(status) 1070 | } else { 1071 | sess.taskChan <- &syncTask{src: localPath, dest: upPath} 1072 | } 1073 | } 1074 | 1075 | func (sess *Session) syncDirectory(localPath, upPath string) (int, error) { 1076 | delFunc := func(prevMeta *fileMeta) { 1077 | sess.taskChan <- &delTask{ 1078 | src: filepath.Join(localPath, prevMeta.Name), 1079 | dest: path.Join(upPath, prevMeta.Name), 1080 | isdir: prevMeta.IsDir, 1081 | } 1082 | } 1083 | syncFunc := func(curMeta *fileMeta) { 1084 | src := filepath.Join(localPath, curMeta.Name) 1085 | dest := path.Join(upPath, curMeta.Name) 1086 | sess.syncObject(src, dest, curMeta.IsDir) 1087 | } 1088 | 1089 | dbVal, err := getDBValue(localPath, upPath) 1090 | if err != nil { 1091 | return SYNC_FAIL, err 1092 | } 1093 | 1094 | curMetas, err := makeFileMetas(localPath) 1095 | if err != nil { 1096 | // if not exist, should sync next time 1097 | if os.IsNotExist(err) { 1098 | return SYNC_NOT_FOUND, err 1099 | } 1100 | return SYNC_FAIL, err 1101 | } 1102 | 1103 | status := SYNC_EXISTS 1104 | var prevMetas []*fileMeta 1105 | if dbVal != nil && dbVal.IsDir == "true" { 1106 | prevMetas = dbVal.Items 1107 | } else { 1108 | if err = sess.updriver.Mkdir(upPath); err != nil { 1109 | return SYNC_FAIL, err 1110 | } 1111 | status = SYNC_OK 1112 | } 1113 | 1114 | cur, curSize, prev, prevSize := 0, len(curMetas), 0, len(prevMetas) 1115 | for cur < curSize && prev < prevSize { 1116 | curMeta, prevMeta := curMetas[cur], prevMetas[prev] 1117 | if curMeta.Name == prevMeta.Name { 1118 | if curMeta.IsDir != prevMeta.IsDir { 1119 | delFunc(prevMeta) 1120 | } 1121 | syncFunc(curMeta) 1122 | prev++ 1123 | cur++ 1124 | } else if curMeta.Name > prevMeta.Name { 1125 | delFunc(prevMeta) 1126 | prev++ 1127 | } else { 1128 | syncFunc(curMeta) 1129 | cur++ 1130 | } 1131 | } 1132 | for ; cur < curSize; cur++ { 1133 | syncFunc(curMetas[cur]) 1134 | } 1135 | for ; prev < prevSize; prev++ { 1136 | delFunc(prevMetas[prev]) 1137 | } 1138 | 1139 | setDBValue(localPath, upPath, &dbValue{IsDir: "true", Items: curMetas}) 1140 | return status, nil 1141 | } 1142 | 1143 | func (sess *Session) Sync(localPath, upPath string, workers int, delete, strong bool) { 1144 | var wg sync.WaitGroup 1145 | sess.taskChan = make(chan interface{}, workers*2) 1146 | stopChan := make(chan bool, 1) 1147 | sigChan := make(chan os.Signal, 1) 1148 | signal.Notify(sigChan, os.Interrupt) 1149 | 1150 | upPath = sess.AbsPath(upPath) 1151 | localPath, _ = filepath.Abs(localPath) 1152 | 1153 | if err := initDB(); err != nil { 1154 | PrintErrorAndExit("sync: init database: %v", err) 1155 | } 1156 | 1157 | var delLock sync.Mutex 1158 | for w := 0; w < workers; w++ { 1159 | wg.Add(1) 1160 | go func() { 1161 | defer wg.Done() 1162 | for task := range sess.taskChan { 1163 | switch v := task.(type) { 1164 | case *syncTask: 1165 | stat, err := sess.syncFile(v.src, v.dest, strong) 1166 | switch stat { 1167 | case SYNC_OK: 1168 | PrintOnlyVerbose("sync %s to %s OK", v.src, v.dest) 1169 | case SYNC_EXISTS: 1170 | PrintOnlyVerbose("sync %s to %s EXISTS", v.src, v.dest) 1171 | case SYNC_FAIL, SYNC_NOT_FOUND: 1172 | PrintError("sync %s to %s FAIL %v", v.src, v.dest, err) 1173 | } 1174 | sess.update(stat) 1175 | case *delTask: 1176 | if delete { 1177 | delDBValue(v.src, v.dest) 1178 | delLock.Lock() 1179 | if v.isdir { 1180 | sess.rmDir(v.dest, false) 1181 | } else { 1182 | sess.rmFile(v.dest, false) 1183 | } 1184 | delLock.Unlock() 1185 | } 1186 | } 1187 | } 1188 | }() 1189 | } 1190 | 1191 | go func() { 1192 | wg.Wait() 1193 | close(stopChan) 1194 | }() 1195 | 1196 | go func() { 1197 | isDir, _ := sess.IsLocalDir(localPath) 1198 | sess.syncObject(localPath, upPath, isDir) 1199 | close(sess.taskChan) 1200 | }() 1201 | 1202 | select { 1203 | case <-sigChan: 1204 | PrintErrorAndExit("%s", sess.dump()) 1205 | case <-stopChan: 1206 | if sess.scores[SYNC_FAIL] > 0 || sess.scores[DELETE_FAIL] > 0 { 1207 | PrintErrorAndExit("%s", sess.dump()) 1208 | } else { 1209 | Print("%s", sess.dump()) 1210 | } 1211 | } 1212 | } 1213 | func (sess *Session) PostTask(app, notify, taskFile string) { 1214 | fd, err := os.Open(taskFile) 1215 | if err != nil { 1216 | PrintErrorAndExit("open %s: %v", taskFile, err) 1217 | } 1218 | 1219 | body, err := ioutil.ReadAll(fd) 1220 | fd.Close() 1221 | if err != nil { 1222 | PrintErrorAndExit("read %s: %v", taskFile, err) 1223 | } 1224 | 1225 | var tasks []interface{} 1226 | if err = json.Unmarshal(body, &tasks); err != nil { 1227 | PrintErrorAndExit("json Unmarshal: %v", err) 1228 | } 1229 | 1230 | if notify == "" { 1231 | notify = "https://httpbin.org/post" 1232 | } 1233 | ids, err := sess.updriver.CommitTasks(&upyun.CommitTasksConfig{ 1234 | AppName: app, 1235 | NotifyUrl: notify, 1236 | Tasks: tasks, 1237 | }) 1238 | if err != nil { 1239 | PrintErrorAndExit("commit tasks: %v", err) 1240 | } 1241 | Print("%v", ids) 1242 | } 1243 | 1244 | func (sess *Session) Purge(urls []string, file string) { 1245 | if urls == nil { 1246 | urls = make([]string, 0) 1247 | } 1248 | if file != "" { 1249 | fd, err := os.Open(file) 1250 | if err != nil { 1251 | PrintErrorAndExit("open %s: %v", file, err) 1252 | } 1253 | body, err := ioutil.ReadAll(fd) 1254 | fd.Close() 1255 | if err != nil { 1256 | PrintErrorAndExit("read %s: %v", file, err) 1257 | } 1258 | for _, line := range strings.Split(string(body), "\n") { 1259 | if line == "" { 1260 | continue 1261 | } 1262 | urls = append(urls, line) 1263 | } 1264 | } 1265 | for idx := range urls { 1266 | if !strings.HasPrefix(urls[idx], "http") { 1267 | urls[idx] = "http://" + urls[idx] 1268 | } 1269 | } 1270 | if len(urls) == 0 { 1271 | return 1272 | } 1273 | 1274 | fails, err := sess.updriver.Purge(urls) 1275 | if fails != nil && len(fails) != 0 { 1276 | PrintError("Purge failed urls:") 1277 | for _, url := range fails { 1278 | PrintError("%s", url) 1279 | } 1280 | PrintErrorAndExit("too many fails") 1281 | } 1282 | if err != nil { 1283 | PrintErrorAndExit("purge error: %v", err) 1284 | } 1285 | } 1286 | 1287 | func (sess *Session) Copy(srcPath, destPath string, force bool) error { 1288 | return sess.copyMove(srcPath, destPath, "copy", force) 1289 | } 1290 | 1291 | func (sess *Session) Move(srcPath, destPath string, force bool) error { 1292 | return sess.copyMove(srcPath, destPath, "move", force) 1293 | } 1294 | 1295 | // 移动或者复制 1296 | // method: "move" | "copy" 1297 | // force: 是否覆盖目标文件 1298 | func (sess *Session) copyMove(srcPath, destPath, method string, force bool) error { 1299 | // 将源文件路径转化为绝对路径 1300 | srcPath = sess.AbsPath(srcPath) 1301 | 1302 | // 检测源文件 1303 | sourceFileInfo, err := sess.updriver.GetInfo(srcPath) 1304 | if err != nil { 1305 | if upyun.IsNotExist(err) { 1306 | return fmt.Errorf("source file %s is not exist", srcPath) 1307 | } 1308 | return err 1309 | } 1310 | if sourceFileInfo.IsDir { 1311 | return fmt.Errorf("not support dir, %s is dir", srcPath) 1312 | } 1313 | 1314 | // 将目标路径转化为绝对路径 1315 | destPath = sess.AbsPath(destPath) 1316 | 1317 | destFileInfo, err := sess.updriver.GetInfo(destPath) 1318 | // 如果返回的错误不是文件不存在错误，则返回错误 1319 | if err != nil && !upyun.IsNotExist(err) { 1320 | return err 1321 | } 1322 | // 如果没有错误，表示文件存在，则检测文件类型，并判断是否允许覆盖 1323 | if err == nil { 1324 | if !destFileInfo.IsDir { 1325 | // 如果目标文件是文件类型，则需要使用强制覆盖 1326 | if !force { 1327 | return fmt.Errorf( 1328 | "target path %s already exists use -f to force overwrite", 1329 | destPath, 1330 | ) 1331 | } 1332 | } else { 1333 | // 补全文件名后，再次检测文件存不存在 1334 | destPath = path.Join(destPath, path.Base(srcPath)) 1335 | destFileInfo, err := sess.updriver.GetInfo(destPath) 1336 | if err == nil { 1337 | if destFileInfo.IsDir { 1338 | return fmt.Errorf( 1339 | "target file %s already exists and is dir", 1340 | destPath, 1341 | ) 1342 | } 1343 | if !force { 1344 | return fmt.Errorf( 1345 | "target file %s already exists use -f to force overwrite", 1346 | destPath, 1347 | ) 1348 | } 1349 | } 1350 | } 1351 | } 1352 | 1353 | if srcPath == destPath { 1354 | return fmt.Errorf( 1355 | "source and target are the same %s => %s", 1356 | srcPath, 1357 | destPath, 1358 | ) 1359 | } 1360 | 1361 | switch method { 1362 | case "copy": 1363 | return sess.updriver.Copy(&upyun.CopyObjectConfig{ 1364 | SrcPath: srcPath, 1365 | DestPath: destPath, 1366 | }) 1367 | case "move": 1368 | return sess.updriver.Move(&upyun.MoveObjectConfig{ 1369 | SrcPath: srcPath, 1370 | DestPath: destPath, 1371 | }) 1372 | default: 1373 | return fmt.Errorf("not support method") 1374 | } 1375 | } 1376 | -------------------------------------------------------------------------------- /tree_test.go: -------------------------------------------------------------------------------- 1 | package upx 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestTree(t *testing.T) { 12 | base := ROOT + "/ls" 13 | dirs, files := []string{}, []string{} 14 | 15 | func() { 16 | SetUp() 17 | Upx("mkdir", base) 18 | Upx("cd", base) 19 | 20 | for i := 0; i < 11; i++ { 21 | Upx("mkdir", fmt.Sprintf("dir%d", i)) 22 | dirs = append(dirs, fmt.Sprintf("dir%d", i)) 23 | } 24 | 25 | CreateFile("FILE") 26 | for i := 0; i < 5; i++ { 27 | Upx("put", "FILE", fmt.Sprintf("FILE%d", i)) 28 | files = append(files, fmt.Sprintf("FILE%d", i)) 29 | } 30 | }() 31 | 32 | defer func() { 33 | TearDown() 34 | }() 35 | 36 | tree1, err := Upx("tree") 37 | assert.NoError(t, err) 38 | tree1s := string(tree1) 39 | arr := strings.Split(tree1s, "\n") 40 | assert.Equal(t, len(arr), len(dirs)+len(files)+4) 41 | pwd, _ := Upx("pwd") 42 | assert.Equal(t, arr[0]+"\n", string(pwd)) 43 | assert.Equal(t, arr[len(arr)-3], "") 44 | assert.Equal(t, arr[len(arr)-2], fmt.Sprintf("%d directories, %d files", len(dirs), len(files))) 45 | 46 | tree2, err := Upx("tree", base) 47 | assert.NoError(t, err) 48 | assert.Equal(t, string(tree2), string(tree1)) 49 | } 50 | -------------------------------------------------------------------------------- /upgrade.go: -------------------------------------------------------------------------------- 1 | package upx 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "net/http" 8 | "os" 9 | "runtime" 10 | "strings" 11 | ) 12 | 13 | const ( 14 | VERSION_URL = "https://raw.githubusercontent.com/upyun/upx/master/VERSION" 15 | DOWNLOAD_URL_PREFIX = "http://collection.b0.upaiyun.com/softwares/upx/upx-" 16 | ) 17 | 18 | func GetCurrentBinPath() string { 19 | p, _ := os.Executable() 20 | return p 21 | } 22 | 23 | func GetLatestVersion() (string, error) { 24 | url := VERSION_URL 25 | resp, err := http.Get(url) 26 | if err != nil { 27 | return "", fmt.Errorf("GetVersion: %v", err) 28 | } 29 | content, _ := ioutil.ReadAll(resp.Body) 30 | resp.Body.Close() 31 | 32 | if resp.StatusCode/100 != 2 { 33 | return "", fmt.Errorf("Get %s: %d", url, resp.StatusCode) 34 | } 35 | 36 | return strings.TrimSpace(string(content)), nil 37 | } 38 | 39 | func DownloadBin(version, binPath string) error { 40 | fd, err := os.Create(binPath) 41 | if err != nil { 42 | return fmt.Errorf("Create %s: %v", binPath, err) 43 | } 44 | defer fd.Close() 45 | 46 | url := DOWNLOAD_URL_PREFIX + fmt.Sprintf("%s-%s-%s", runtime.GOOS, runtime.GOARCH, version) 47 | if runtime.GOOS == "windows" { 48 | url += ".exe" 49 | } 50 | fmt.Print(url) 51 | resp, err := http.Get(url) 52 | if err != nil { 53 | return fmt.Errorf("Download %s: %v", url, err) 54 | } 55 | defer resp.Body.Close() 56 | 57 | if resp.StatusCode != 200 { 58 | content, _ := ioutil.ReadAll(resp.Body) 59 | return fmt.Errorf("Download %s %d: %s", url, resp.StatusCode, string(content)) 60 | } 61 | 62 | _, err = io.Copy(fd, resp.Body) 63 | if err != nil { 64 | return fmt.Errorf("Download %s: copy %v", url, err) 65 | } 66 | return nil 67 | } 68 | 69 | func ChmodAndRename(src, dst string) error { 70 | err := os.Chmod(src, 0755) 71 | if err != nil { 72 | return fmt.Errorf("chmod %s: %v", src, err) 73 | } 74 | err = os.Rename(src, dst) 75 | if err != nil { 76 | return fmt.Errorf("rename %s %s: %v", src, dst, err) 77 | } 78 | return nil 79 | } 80 | 81 | func Upgrade() { 82 | lv, err := GetLatestVersion() 83 | 84 | if err != nil { 85 | PrintErrorAndExit("Find Latest Version: %v", err) 86 | } 87 | 88 | Print("Find Latest Version: %s", lv) 89 | Print("Current Version: %s", VERSION) 90 | 91 | if lv == VERSION { 92 | return 93 | } 94 | 95 | binPath := GetCurrentBinPath() 96 | tmpBinPath := binPath + ".upgrade" 97 | err = DownloadBin(lv, tmpBinPath) 98 | if err != nil { 99 | PrintErrorAndExit("Download Binary %s: %v", VERSION, err) 100 | } 101 | Print("Download Binary %s: OK", VERSION) 102 | 103 | err = ChmodAndRename(tmpBinPath, binPath) 104 | if err != nil { 105 | PrintErrorAndExit("Chmod %s: %v", binPath, err) 106 | } 107 | PrintErrorAndExit("Chmod %s: OK", binPath) 108 | 109 | return 110 | } 111 | -------------------------------------------------------------------------------- /upx.go: -------------------------------------------------------------------------------- 1 | package upx 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | "time" 7 | 8 | "github.com/urfave/cli" 9 | ) 10 | 11 | const VERSION = "v0.4.9" 12 | 13 | func CreateUpxApp() *cli.App { 14 | app := cli.NewApp() 15 | app.Name = "upx" 16 | app.Usage = "a tool for driving UpYun Storage" 17 | app.Author = "Hongbo.Mo" 18 | app.Email = "zjutpolym@gmail.com" 19 | app.Version = fmt.Sprintf("%s %s/%s %s", VERSION, 20 | runtime.GOOS, runtime.GOARCH, runtime.Version()) 21 | app.EnableBashCompletion = true 22 | app.Compiled = time.Now() 23 | app.Flags = []cli.Flag{ 24 | cli.BoolFlag{Name: "quiet, q", Usage: "not verbose"}, 25 | cli.StringFlag{Name: "auth", Usage: "auth string"}, 26 | } 27 | app.Before = func(c *cli.Context) error { 28 | if c.Bool("q") { 29 | IsVerbose = false 30 | } 31 | if c.String("auth") != "" { 32 | err := authStrToConfig(c.String("auth")) 33 | if err != nil { 34 | PrintErrorAndExit("%s: invalid auth string", c.Command.FullName()) 35 | } 36 | } 37 | return nil 38 | } 39 | app.Commands = []cli.Command{ 40 | NewLoginCommand(), 41 | NewLogoutCommand(), 42 | NewListSessionsCommand(), 43 | NewSwitchSessionCommand(), 44 | NewInfoCommand(), 45 | NewCdCommand(), 46 | NewPwdCommand(), 47 | NewMkdirCommand(), 48 | NewLsCommand(), 49 | NewTreeCommand(), 50 | NewGetCommand(), 51 | NewPutCommand(), 52 | NewUploadCommand(), 53 | NewRmCommand(), 54 | NewSyncCommand(), 55 | NewAuthCommand(), 56 | NewPostCommand(), 57 | NewPurgeCommand(), 58 | NewGetDBCommand(), 59 | NewCleanDBCommand(), 60 | NewUpgradeCommand(), 61 | NewCopyCommand(), 62 | NewMoveCommand(), 63 | } 64 | return app 65 | } 66 | -------------------------------------------------------------------------------- /upx_test.go: -------------------------------------------------------------------------------- 1 | package upx 2 | 3 | import ( 4 | "bytes" 5 | "flag" 6 | "fmt" 7 | "io/ioutil" 8 | "os" 9 | "os/exec" 10 | "path/filepath" 11 | "testing" 12 | "time" 13 | ) 14 | 15 | var ( 16 | ROOT = fmt.Sprintf("/upx-test/%s", time.Now()) 17 | BUCKET_1 = os.Getenv("UPYUN_BUCKET1") 18 | BUCKET_2 = os.Getenv("UPYUN_BUCKET2") 19 | USERNAME = os.Getenv("UPYUN_USERNAME") 20 | PASSWORD = os.Getenv("UPYUN_PASSWORD") 21 | ) 22 | 23 | func SetUp() { 24 | Upx("login", BUCKET_1, USERNAME, PASSWORD) 25 | } 26 | 27 | func TearDown() { 28 | for { 29 | b, err := Upx("logout") 30 | if err != nil || string(b) == "Nothing to do ~\n" { 31 | break 32 | } 33 | } 34 | } 35 | 36 | func CreateFile(fpath string) { 37 | os.MkdirAll(filepath.Dir(fpath), 0755) 38 | fd, _ := os.Create(fpath) 39 | fd.WriteString("UPX") 40 | fd.Close() 41 | } 42 | 43 | func Upx(args ...string) ([]byte, error) { 44 | cmd := exec.Command("upx", args...) 45 | var obuf, ebuf bytes.Buffer 46 | cmd.Stdout, cmd.Stderr = &obuf, &ebuf 47 | if err := cmd.Start(); err != nil { 48 | return nil, err 49 | } 50 | err := cmd.Wait() 51 | ob, _ := ioutil.ReadAll(&obuf) 52 | eb, _ := ioutil.ReadAll(&ebuf) 53 | if err != nil { 54 | return ob, fmt.Errorf("%s", string(eb)) 55 | } 56 | return ob, nil 57 | } 58 | 59 | func TestMain(m *testing.M) { 60 | pwd, _ := os.Getwd() 61 | os.Setenv("PATH", pwd) 62 | flag.Parse() 63 | code := m.Run() 64 | os.Exit(code) 65 | } 66 | -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- 1 | package upx 2 | 3 | import ( 4 | "crypto/md5" 5 | "fmt" 6 | "io" 7 | "os" 8 | "path/filepath" 9 | "runtime" 10 | "strconv" 11 | "strings" 12 | "time" 13 | ) 14 | 15 | func parseMTime(value string, match *MatchConfig) error { 16 | if value == "" { 17 | return nil 18 | } 19 | 20 | v, err := strconv.ParseInt(value, 10, 64) 21 | if err != nil { 22 | return err 23 | } 24 | 25 | if v < 0 { 26 | match.After = time.Now().Add(time.Duration(v) * time.Hour * 24) 27 | match.TimeType = TIME_AFTER 28 | } else { 29 | if strings.HasPrefix(value, "+") { 30 | match.Before = time.Now().Add(time.Duration(-1*(v+1)) * time.Hour * 24) 31 | match.TimeType = TIME_BEFORE 32 | } else { 33 | match.Before = time.Now().Add(time.Duration(-1*v) * time.Hour * 24) 34 | match.After = time.Now().Add(time.Duration(-1*(v+1)) * time.Hour * 24) 35 | match.TimeType = TIME_INTERVAL 36 | } 37 | } 38 | return nil 39 | } 40 | 41 | func humanizeSize(b int64) string { 42 | unit := []string{"B", "KB", "MB", "GB", "TB"} 43 | u, v, s := 0, float64(b), "" 44 | for { 45 | if v < 1024.0 { 46 | switch { 47 | case v < 10: 48 | s = fmt.Sprintf("%.3f", v) 49 | case v < 100: 50 | s = fmt.Sprintf("%.2f", v) 51 | case v < 1000: 52 | s = fmt.Sprintf("%.1f", v) 53 | default: 54 | s = fmt.Sprintf("%.0f", v) 55 | } 56 | break 57 | } 58 | v /= 1024 59 | u++ 60 | } 61 | 62 | if strings.Contains(s, ".") { 63 | ed := len(s) - 1 64 | for ; ed > 0; ed-- { 65 | if s[ed] == '.' { 66 | ed-- 67 | break 68 | } 69 | if s[ed] != '0' { 70 | break 71 | } 72 | } 73 | s = s[:ed+1] 74 | } 75 | return s + unit[u] 76 | } 77 | 78 | func md5File(fpath string) (string, error) { 79 | fd, err := os.Open(fpath) 80 | if err != nil { 81 | return "", err 82 | } 83 | defer fd.Close() 84 | hash := md5.New() 85 | if _, err := io.Copy(hash, fd); err != nil { 86 | return "", err 87 | } 88 | return fmt.Sprintf("%x", hash.Sum(nil)), nil 89 | } 90 | 91 | func contains(slice []string, item string) bool { 92 | for _, s := range slice { 93 | if s == item { 94 | return true 95 | } 96 | } 97 | return false 98 | } 99 | 100 | func globFiles(patterns []string) []string { 101 | filenames := make([]string, 0) 102 | for _, filename := range patterns { 103 | matches, err := filepath.Glob(filename) 104 | if err == nil { 105 | filenames = append(filenames, matches...) 106 | } 107 | } 108 | return filenames 109 | } 110 | 111 | func isWindowsGOOS() bool { 112 | return runtime.GOOS == "windows" 113 | } 114 | 115 | func ResumePartSize(size int64) int64 { 116 | if size < 50*1024*1024 { 117 | return 1024 * 1024 118 | } 119 | 120 | if size < 1024*1024*1024 { 121 | return 10 * 1024 * 1024 122 | } 123 | 124 | if size < 100*1024*1024*1024 { 125 | return 50 * 1024 * 1024 126 | } 127 | 128 | return 100 * 1024 * 1024 129 | } 130 | 131 | func cleanFilename(name string) string { 132 | if !isWindowsGOOS() { 133 | return name 134 | } 135 | var name2 string 136 | if strings.HasPrefix(name, `\\?\`) { 137 | name2 = `\\?\` 138 | name = strings.TrimPrefix(name, `\\?\`) 139 | } 140 | if strings.HasPrefix(name, `//?/`) { 141 | name2 = `//?/` 142 | name = strings.TrimPrefix(name, `//?/`) 143 | } 144 | name2 += strings.Map(func(r rune) rune { 145 | switch r { 146 | case '<', '>', '"', '|', '?', '*', ':': 147 | return '_' 148 | } 149 | return r 150 | }, name) 151 | return name2 152 | } 153 | -------------------------------------------------------------------------------- /xerrors/errors.go: -------------------------------------------------------------------------------- 1 | package xerrors 2 | 3 | import "errors" 4 | 5 | var ( 6 | ErrInvalidCommand = errors.New("invalid command") 7 | ErrRequireLogin = errors.New("log in to UpYun first") 8 | ) 9 | --------------------------------------------------------------------------------